Merge pull request mitogen-hq#800 from moreati/0.2-backport

Prepare 0.2.10rc1

.ci/ansible_install.py

@@ -16,7 +16,7 @@
 ]
 
 batches.extend(
-    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
+    ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1']
     for distro in ci_lib.DISTROS
 )
 

.ci/ansible_tests.py

@@ -37,9 +37,6 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chdir(TESTS_DIR)
     os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
 
@@ -69,8 +66,6 @@ def pause_if_interactive():
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')

.ci/azure-pipelines-steps.yml

@@ -14,17 +14,25 @@ steps:
 # stuff into. The virtualenv can probably be removed again, but this was a
 # hard-fought battle and for now I am tired of this crap.
 - script: |
-    # need wheel before building virtualenv because of bdist_wheel and setuptools deps
-    # Mac's System Integrity Protection prevents symlinking /usr/bin
-    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
-    # the || will activate when running python3 tests
     # TODO: get python3 tests passing
-    (sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python &&
-      /usr/bin/python -m pip install -U pip wheel setuptools &&
-      /usr/bin/python -m pip install -U virtualenv &&
-      /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)) ||
-    (sudo /usr/bin/python$(python.version) -m pip install -U pip wheel setuptools &&
-      /usr/bin/python$(python.version) -m venv /tmp/venv)
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+    set -x
+
+    # Ensure virtualenv is in system-wide $PATH on OSX, for Ansible test tasks
+    # that use it, e.g. regression/issue_152__virtualenv_python_fails.yml
+    if [[ -d /Library/Frameworks/Python.framework/Versions/2.7/bin ]]; then
+        echo $PATH
+        which python
+        python -m pip install -U pip virtualenv
+        ln -s  /Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv /usr/local/bin/virtualenv
+        virtualenv --version
+    fi
+
+    # Create virtualenv using desired Python version, to run the testcases
+    python$(python.version) -m pip install -U pip wheel setuptools virtualenv
+    python$(python.version) -m virtualenv /tmp/venv -p python$(python.version)
     echo "##vso[task.prependpath]/tmp/venv/bin"
 
   displayName: activate venv
@@ -37,3 +45,5 @@ steps:
 
 - script: .ci/$(MODE)_tests.py
   displayName: "Run $(MODE)_tests.py"
+  env:
+    NOCOVERAGE: 1

.ci/azure-pipelines.yml

@@ -13,10 +13,10 @@ jobs:
   strategy:
     matrix:
       Mito27_27:
-        python.version: '2.7.18'
+        python.version: '2.7'
         MODE: mitogen
       Ans288_27:
-        python.version: '2.7.18'
+        python.version: '2.7'
         MODE: localhost_ansible
         VER: 2.8.8
 

.ci/ci_lib.py

@@ -80,12 +80,28 @@ def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc):
 # -----------------
 
 def _argv(s, *args):
+    """Interpolate a command line using *args, return an argv style list.
+
+    >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234)
+    ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)']
+    """
     if args:
         s %= args
     return shlex.split(s)
 
 
 def run(s, *args, **kwargs):
+    """ Run a command, with arguments, and print timing information
+
+    >>> rc = run('echo "%s %s"', 'foo', 'bar')
+    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    foo bar
+    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
+    0inputs+0outputs (0major+71minor)pagefaults 0swaps
+    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    >>> rc
+    0
+    """
     argv = ['/usr/bin/time', '--'] + _argv(s, *args)
     print('Running: %s' % (argv,))
     try:
@@ -98,12 +114,36 @@ def run(s, *args, **kwargs):
     return ret
 
 
-def run_batches(batches):
-    combine = lambda batch: 'set -x; ' + (' && '.join(
+def combine(batch):
+    """
+    >>> combine(['ls -l', 'echo foo'])
+    'set -x; ( ls -l; ) && ( echo foo; )'
+    """
+    return 'set -x; ' + (' && '.join(
         '( %s; )' % (cmd,)
         for cmd in batch
     ))
 
+
+def run_batches(batches):
+    """ Run shell commands grouped into batches, showing an execution trace.
+
+    Raise AssertionError if any command has exits with a non-zero status.
+
+    >>> run_batches([['echo foo', 'true']])
+    + echo foo
+    foo
+    + true
+    >>> run_batches([['true', 'echo foo'], ['false']])
+    + true
+    + echo foo
+    foo
+    + false
+    Traceback (most recent call last):
+    File "...", line ..., in <module>
+    File "...", line ..., in run_batches
+    AssertionError
+    """
     procs = [
         subprocess.Popen(combine(batch), shell=True)
         for batch in batches
@@ -112,12 +152,28 @@ def run_batches(batches):
 
 
 def get_output(s, *args, **kwargs):
+    """
+    Print and run command line s, %-interopolated using *args. Return stdout.
+
+    >>> s = get_output('echo "%s %s"', 'foo', 'bar')
+    Running: ['echo', 'foo bar']
+    >>> s
+    'foo bar\n'
+    """
     argv = _argv(s, *args)
     print('Running: %s' % (argv,))
     return subprocess.check_output(argv, **kwargs)
 
 
 def exists_in_path(progname):
+    """
+    Return True if proganme exists in $PATH.
+
+    >>> exists_in_path('echo')
+    True
+    >>> exists_in_path('kwyjibo') # Only found in North American cartoons
+    False
+    """
     return any(os.path.exists(os.path.join(dirname, progname))
                for dirname in os.environ['PATH'].split(os.pathsep))
 
@@ -132,6 +188,18 @@ def destroy(self, rmtree=shutil.rmtree):
 
 
 class Fold(object):
+    """
+    Bracket a section of stdout with travis_fold markers.
+
+    This allows the section to be collapsed or expanded in Travis CI web UI.
+
+    >>> with Fold('stage 1'):
+    ...     print('Frobnicate the frobnitz')
+    ...
+    travis_fold:start:stage 1
+    Frobnicate the frobnitz
+    travis_fold:end:stage 1
+    """
     def __init__(self, name):
         self.name = name
 
@@ -171,6 +239,8 @@ def __exit__(self, _1, _2, _3):
 )
 
 def get_docker_hostname():
+    """Return the hostname where the docker daemon is running.
+    """
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -180,10 +250,34 @@ def get_docker_hostname():
 
 
 def image_for_distro(distro):
-    return 'mitogen/%s-test' % (distro.partition('-')[0],)
+    """Return the container image name or path for a test distro name.
+
+    The returned value is suitable for use with `docker pull`.
+
+    >>> image_for_distro('centos5')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    >>> image_for_distro('centos5-something_custom')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    """
+    return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],)
 
 
 def make_containers(name_prefix='', port_offset=0):
+    """
+    >>> import pprint
+    >>> BASE_PORT=2200; DISTROS=['debian', 'centos6']
+    >>> pprint.pprint(make_containers())
+    [{'distro': 'debian',
+      'hostname': 'localhost',
+      'name': 'target-debian-1',
+      'port': 2201,
+      'python_path': '/usr/bin/python'},
+     {'distro': 'centos6',
+      'hostname': 'localhost',
+      'name': 'target-centos6-2',
+      'port': 2202,
+      'python_path': '/usr/bin/python'}]
+    """
     docker_hostname = get_docker_hostname()
     firstbit = lambda s: (s+'-').split('-')[0]
     secondbit = lambda s: (s+'-').split('-')[1]
@@ -256,6 +350,14 @@ def get_interesting_procs(container_name=None):
 
 
 def start_containers(containers):
+    """Run docker containers in the background, with sshd on specified ports.
+
+    >>> containers = start_containers([
+    ...     {'distro': 'debian', 'hostname': 'localhost',
+    ...      'name': 'target-debian-1', 'port': 2201,
+    ...      'python_path': '/usr/bin/python'},
+    ... ])
+    """
     if os.environ.get('KEEP'):
         return
 

.ci/localhost_ansible_tests.py

@@ -20,12 +20,15 @@
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chmod(KEY_PATH, int('0600', 8))
+    # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", 
+    # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
     if not ci_lib.exists_in_path('sshpass'):
-        run("brew install http://git.io/sshpass.rb")
+        os.system("curl -O -L  https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
+                tar xvf sshpass-1.05.tar.gz && \
+                cd sshpass-1.05 && \
+                ./configure && \
+                sudo make install")
 
 
 with ci_lib.Fold('machine_prep'):

.ci/prep_azure.py

@@ -43,12 +43,4 @@
             .format(pv=os.environ['PYTHONVERSION'])
     ])
 
-
-if ci_lib.have_docker():
-    batches.extend(
-        ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
-        for distro in ci_lib.DISTROS
-    )
-
-
 ci_lib.run_batches(batches)

.travis.yml

@@ -16,6 +16,9 @@ cache:
 - directories:
   - /home/travis/virtualenv
 
+env:
+  - NOCOVERAGE=1
+
 install:
 - grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
 - .ci/${MODE}_install.py

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2019, David Wilson
+Copyright 2021, the Mitogen authors
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

README.md

@@ -5,8 +5,8 @@
 
 ![](https://i.imgur.com/eBM6LhJ.gif)
 
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/dw/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/dw/mitogen/alerts/)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/)
 
-[![Build Status](https://travis-ci.org/dw/mitogen.svg?branch=master)](https://travis-ci.org/dw/mitogen)
+[![Build Status](https://api.travis-ci.com/mitogen-hq/mitogen.svg?branch=master)](https://api.travis-ci.com/mitogen-hq/mitogen)
 
-[![Pipelines Status](https://dev.azure.com/dw-mitogen/Mitogen/_apis/build/status/dw.mitogen?branchName=master)](https://dev.azure.com/dw-mitogen/Mitogen/_build/latest?definitionId=1?branchName=master)
+[![Pipelines Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master)

docs/ansible_detailed.rst

@@ -145,7 +145,7 @@ Testimonials
 Noteworthy Differences
 ----------------------
 
-* Ansible 2.3-2.8 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify
+* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify
   your installation is running one of these versions by checking ``ansible
   --version`` output.
 

docs/changelog.rst

@@ -14,18 +14,23 @@ Release Notes
         }
     </style>
 
+To avail of fixes in an unreleased version, please download a ZIP file
+`directly from GitHub <https://github.com/dw/mitogen/>`_.
 
 v0.2.10 (unreleased)
 --------------------
 
-To avail of fixes in an unreleased version, please download a ZIP file
-`directly from GitHub <https://github.com/dw/mitogen/>`_.
-
+* :gh:issue:`597` mitogen does not support Ansible 2.8 Python interpreter detection
+* :gh:issue:`655` wait_for_connection gives errors
+* :gh:issue:`672` cannot perform relative import error
+* :gh:issue:`673` mitogen fails on RHEL8 server with bash /usr/bin/python: No such file or directory
+* :gh:issue:`676` mitogen fail to run playbook without “/usr/bin/python” on target host
+* :gh:issue:`716` fetch fails with "AttributeError: 'ShellModule' object has no attribute 'tmpdir'"
 * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would
   timeout, when using recent OpenSSH client versions.
-* :gh:issue:`758` fix initilialisation of callback plugins in test suite, to
-  to address a `KeyError` in
+* :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in
   :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
+* :gh:issue:`775` Add msvcrt to the default module deny list
 
 
 v0.2.9 (2019-11-02)

docs/conf.py

@@ -7,7 +7,7 @@
 VERSION = '%s.%s.%s' % mitogen.__version__
 
 author = u'Network Genomics'
-copyright = u'2019, Network Genomics'
+copyright = u'2021, the Mitogen authors'
 exclude_patterns = ['_build', '.venv']
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput', 'domainrefs']
 

mitogen/__init__.py

@@ -35,7 +35,7 @@
 
 
 #: Library version as a tuple.
-__version__ = (0, 2, 9)
+__version__ = (0, 2, 10, 'rc', 1)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -1269,6 +1269,13 @@ class Importer(object):
         # a negative round-trip.
         'builtins',
         '__builtin__',
+
+        # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries
+        # to import of this Windows-only builtin module.
+        'msvcrt',
+
+        # Python 2.x module that was renamed to _thread in 3.x.
+        # This entry avoids a roundtrip on 2.x -> 3.x.
         'thread',
 
         # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
@@ -3860,7 +3867,7 @@ def _setup_importer(self):
         else:
             core_src_fd = self.config.get('core_src_fd', 101)
             if core_src_fd:
-                fp = os.fdopen(core_src_fd, 'rb', 1)
+                fp = os.fdopen(core_src_fd, 'rb', 0)
                 try:
                     core_src = fp.read()
                     # Strip "ExternalContext.main()" call from last line.