Merge remote-tracking branch 'origin/026' into stable

* origin/026:
  docs: update Changelog for release.
  Bump version for release.
  issue #555: ansible: workaround ancient reload(sys) hack.
  issue #554: mitogen_action_script fix
  issue #554: fix Ansible 2.4 compatibility
  issue #554: don't rely on tmp_path autoremoval in test.
  issue #554: track and remove multiple make_tmp_path() calls.
  docs: update Changelog.
  docs: drastically simplify install/changelog.
  issue #552: include process identity in log messages.
  issue #550: update Changelog.
  issue #550: parent: add explanatory comment.
  issue #550: fix up TTY ioctls on WSL 2016 Anniversary Update
  docs: update Changelog.
  service: make service list optional.
  docs: update Changelog; closes #548.
  issue #548: always treat transport=smart as 'ssh' for mitogen_via=.
  docs: better intro paragraph.
  .ci: copy private key file to tempdir.
  os_fork: more doc tweaks
  os_fork: more doc tweaks
  os_fork: yet more doc tidyup
  os_fork: more doc tweaks
  os_fork: clean up docs
  .ci: import soak scripts.
  .ci: allow containers for different jobs to run simultaneously
  os_fork: python 3 fixes and tests.
  issue #535: activate Corker on 2.4 in master too.
  issue #535: update Changelog.
  issue #535: wire mitogen.os_fork into Broker and Pool.
  issue #535: parent: add create_socketpair(size=..) parameter.
  issue #535: introduce mitogen.os_fork module and Corker class.
  issue #535: docs: update Changelog
  issue #535: service: support Pool.defer() like Broker.defer()
  issue #535: core: unicode.encode() may take importer lock on 2.x
  issue #535: docs: fix up Select doc
  issue #535: docs: update Changelog.
  issue #535: core/select: support selecting from Latches.
  core: increase cookie field lengths to 64-bit; closes #545.
  tests: ensure serialization restrictions are in effect
  tests/bench: set process affinity in throughput.py.
  docs: update copyright year.
  docs: update Changelog.
  core: Make Latch.put(obj=) optional.
  docs: change 'unreleased' Changelog format and add a hint.
  docs: update Changelog; closes #542.
  issue #542: return of select poller, new selection logic
  issue #542: .ci: move some tests to Azure and enable Mac job.
  ansible: create stub __init__.py for sdist.

.ci/azure-pipelines-steps.yml

@@ -0,0 +1,20 @@
+
+parameters:
+  name: ''
+  pool: ''
+  sign: false
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+    architecture: 'x64'
+
+- script: .ci/prep_azure.py
+  displayName: "Install requirements."
+
+- script: .ci/$(MODE)_install.py
+  displayName: "Install requirements."
+
+- script: .ci/$(MODE)_tests.py
+  displayName: Run tests.

.ci/azure-pipelines.yml

@@ -5,79 +5,85 @@
 
 jobs:
 
-- job: 'MitogenTests'
+- job: Mac
+  steps:
+  - template: azure-pipelines-steps.yml
   pool:
-    vmImage: 'Ubuntu 16.04'
+    vmImage: macOS-10.13
   strategy:
     matrix:
-      Mitogen27Debian_27:
+      Mito27_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: debian
 
-      MitogenPy27CentOS6_26:
+
+- job: Linux
+  pool:
+    vmImage: "Ubuntu 16.04"
+  steps:
+  - template: azure-pipelines-steps.yml
+  strategy:
+    matrix:
+      #
+      # Confirmed working
+      #
+      Mito27Debian_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: centos6
+        DISTRO: debian
 
-      #Py26CentOS7:
+      #MitoPy27CentOS6_26:
         #python.version: '2.7'
         #MODE: mitogen
         #DISTRO: centos6
 
-      Mitogen36CentOS6_26:
+      Mito36CentOS6_26:
         python.version: '3.6'
         MODE: mitogen
         DISTRO: centos6
 
-      DebOps_2460_27_27:
-        python.version: '2.7'
-        MODE: debops_common
-        VER: 2.4.6.0
-
-      DebOps_262_36_27:
-        python.version: '3.6'
-        MODE: debops_common
-        VER: 2.6.2
-
-      Ansible_2460_26:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.4.6.0
+      #
+      #
+      #
 
-      Ansible_262_26:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.6.2
+      #Py26CentOS7:
+        #python.version: '2.7'
+        #MODE: mitogen
+        #DISTRO: centos6
 
-      Ansible_2460_36:
-        python.version: '3.6'
-        MODE: ansible
-        VER: 2.4.6.0
+      #DebOps_2460_27_27:
+        #python.version: '2.7'
+        #MODE: debops_common
+        #VER: 2.4.6.0
 
-      Ansible_262_36:
-        python.version: '3.6'
-        MODE: ansible
-        VER: 2.6.2
+      #DebOps_262_36_27:
+        #python.version: '3.6'
+        #MODE: debops_common
+        #VER: 2.6.2
 
-      Vanilla_262_27:
-        python.version: '2.7'
-        MODE: ansible
-        VER: 2.6.2
-        DISTROS: debian
-        STRATEGY: linear
+      #Ansible_2460_26:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.4.6.0
 
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
-      architecture: 'x64'
+      #Ansible_262_26:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.6.2
 
-  - script: .ci/prep_azure.py
-    displayName: "Install requirements."
+      #Ansible_2460_36:
+        #python.version: '3.6'
+        #MODE: ansible
+        #VER: 2.4.6.0
 
-  - script: .ci/$(MODE)_install.py
-    displayName: "Install requirements."
+      #Ansible_262_36:
+        #python.version: '3.6'
+        #MODE: ansible
+        #VER: 2.6.2
 
-  - script: .ci/$(MODE)_tests.py
-    displayName: Run tests.
+      #Vanilla_262_27:
+        #python.version: '2.7'
+        #MODE: ansible
+        #VER: 2.6.2
+        #DISTROS: debian
+        #STRATEGY: linear

.ci/ci_lib.py

@@ -43,6 +43,18 @@ def subprocess__check_output(*popenargs, **kwargs):
     subprocess.check_output = subprocess__check_output
 
 
+# ------------------
+
+def have_apt():
+    proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
+
+def have_docker():
+    proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)
+    return proc.wait() == 0
+
+
 # -----------------
 
 # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars.
@@ -134,6 +146,17 @@ def __exit__(self, _1, _2, _3):
 BASE_PORT = 2200
 TMP = TempDir().path
 
+
+# We copy this out of the way to avoid random stuff modifying perms in the Git
+# tree (like git pull).
+src_key_file = os.path.join(GIT_ROOT,
+    'tests/data/docker/mitogen__has_sudo_pubkey.key')
+key_file = os.path.join(TMP,
+    'mitogen__has_sudo_pubkey.key')
+shutil.copyfile(src_key_file, key_file)
+os.chmod(key_file, int('0600', 8))
+
+
 os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
 os.environ['PYTHONPATH'] = '%s:%s' % (
     os.environ.get('PYTHONPATH', ''),
@@ -153,7 +176,7 @@ def image_for_distro(distro):
     return 'mitogen/%s-test' % (distro.partition('-')[0],)
 
 
-def make_containers():
+def make_containers(name_prefix='', port_offset=0):
     docker_hostname = get_docker_hostname()
     firstbit = lambda s: (s+'-').split('-')[0]
     secondbit = lambda s: (s+'-').split('-')[1]
@@ -171,9 +194,9 @@ def make_containers():
         for x in range(count):
             lst.append({
                 "distro": firstbit(distro),
-                "name": "target-%s-%s" % (distro, i),
+                "name": name_prefix + ("target-%s-%s" % (distro, i)),
                 "hostname": docker_hostname,
-                "port": BASE_PORT + i,
+                "port": BASE_PORT + i + port_offset,
                 "python_path": (
                     '/usr/bin/python3'
                     if secondbit(distro) == 'py3'
@@ -195,6 +218,8 @@ def start_containers(containers):
             "docker run "
                 "--rm "
                 "--detach "
+                "--privileged "
+                "--cap-add=SYS_PTRACE "
                 "--publish 0.0.0.0:%(port)s:22/tcp "
                 "--hostname=%(name)s "
                 "--name=%(name)s "

.ci/debops_common_tests.py

@@ -2,6 +2,7 @@
 
 from __future__ import print_function
 import os
+import shutil
 
 import ci_lib
 
@@ -10,17 +11,13 @@
 ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
 
 project_dir = os.path.join(ci_lib.TMP, 'project')
-key_file = os.path.join(
-    ci_lib.GIT_ROOT,
-    'tests/data/docker/mitogen__has_sudo_pubkey.key',
-)
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
 docker_hostname = ci_lib.get_docker_hostname()
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
     ci_lib.start_containers(containers)
 
 
@@ -36,7 +33,6 @@
             % (ci_lib.GIT_ROOT,)
         )
 
-    ci_lib.run('chmod go= %s', key_file)
     with open(vars_path, 'w') as fp:
         fp.write(
             "ansible_python_interpreter: /usr/bin/python2.7\n"
@@ -47,7 +43,7 @@
             "\n"
             # Speed up slow DH generation.
             "dhparam__bits: ['128', '64']\n"
-            % (key_file,)
+            % (ci_lib.key_file,)
         )
 
     with open(inventory_path, 'a') as fp:

.ci/mitogen_install.py

@@ -6,10 +6,12 @@
     [
         'pip install "pycparser<2.19" "idna<2.7"',
         'pip install -r tests/requirements.txt',
-    ],
-    [
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ]
 ]
 
+if ci_lib.have_docker():
+    batches.append([
+        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
+    ])
+
 ci_lib.run_batches(batches)

.ci/mitogen_tests.py

@@ -11,4 +11,7 @@
     'SKIP_ANSIBLE': '1',
 })
 
+if not ci_lib.have_docker():
+    os.environ['SKIP_DOCKER_TESTS'] = '1'
+
 ci_lib.run('./run_tests -v')

.ci/prep_azure.py

@@ -1,22 +1,30 @@
 #!/usr/bin/env python
 
+import os
+import sys
+
 import ci_lib
 
 batches = []
-batches.append([
-    'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
-    'sudo add-apt-repository ppa:deadsnakes/ppa',
-    'sudo apt-get update',
-    'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
-])
-
-batches.append([
-    'pip install -r dev_requirements.txt',
-])
-
-batches.extend(
-    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
-    for distro in ci_lib.DISTROS
-)
+
+if ci_lib.have_apt():
+    batches.append([
+        'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
+        'sudo add-apt-repository ppa:deadsnakes/ppa',
+        'sudo apt-get update',
+        'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
+    ])
+
+
+#batches.append([
+    #'pip install -r dev_requirements.txt',
+#])
+
+if ci_lib.have_docker():
+    batches.extend(
+        ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
+        for distro in ci_lib.DISTROS
+    )
+
 
 ci_lib.run_batches(batches)

.ci/soak/debops_common.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+
+# Make Docker containers once.
+/usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
+export KEEP=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
+done
+
+echo $i

.ci/soak/mitogen.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+export DISTROS="debian*4"
+
+# Make Docker containers once.
+/usr/bin/time -v ./.ci/ansible_tests.py "$@"
+export KEEP=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/ansible_tests.py "$@" || break
+done
+
+echo $i

.ci/soak/mitogen_py24.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/mitogen_py24_tests.py "$@" || break
+done
+
+echo $i