From f5a88406685923158cee238ca9a6217fb6b4b630 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 26 Jul 2024 14:58:21 +0100 Subject: [PATCH 1/3] CI: Use archived RPMs on CentOS 8 CentOS 8 has reached EOL. Packages are no longer mirrored or maintained. A historic snapshot of the packages is kept on vault.centos.org. refs #1088, #1090 --- tests/ansible/hosts/group_vars/all.yml | 1 + tests/ansible/hosts/group_vars/centos8.yml | 26 +++++++++++++++++++ tests/ansible/hosts/group_vars/debian9.yml | 4 +++ .../issue_776__load_plugins_called_twice.yml | 19 +++++--------- 4 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/debian9.yml diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index cea46113d..ad4adabb6 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -1,2 +1,3 @@ --- pkg_mgr_python_interpreter: python +pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index 7b9e34f63..c90dd5f46 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,2 +1,28 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python + +pkg_repos_overrides: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml new file mode 100644 index 000000000..4b180b138 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -0,0 +1,4 @@ +pkg_repos_overrides: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ad5bbd693..44ff68633 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,21 +10,14 @@ ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: - - name: Switch to centos-stream - command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos - when: - - ansible_facts.pkg_mgr in ["dnf"] - - - name: Switch to archive.debian.org - # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html + - name: Switch to archived package repositories copy: - content: | - deb http://archive.debian.org/debian stretch main contrib non-free - dest: /etc/apt/sources.list + dest: "{{ item.dest }}" + content: "{{ item.content }}" mode: u=rw,go=r - when: - - ansible_facts.distribution == "Debian" - - ansible_facts.distribution_major_version == "9" + loop: "{{ pkg_repos_overrides }}" + loop_control: + label: "{{ item.dest }}" - name: Add signing keys copy: From 924dbd6f0c31f1169adc8d969115dbcdfecb3cc9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:25:12 +0100 Subject: [PATCH 2/3] CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs macOS 11 is not longer an available runner on Azure Devops. The minimum is now macOS 12. This runner does not have Python 2.7 installed, so running them would require a custom install - which I'm declaring too much effort for too little gain. refs #1090 --- .ci/azure-pipelines.yml | 17 +++-------------- docs/changelog.rst | 1 + .../integration/become/sudo_nonexistent.yml | 3 +-- .../modules/custom_python_json_args_module.py | 2 +- .../lib/modules/custom_python_os_getcwd.py | 2 +- .../modules/custom_python_want_json_module.py | 2 +- .../issue_591__setuptools_cwd_crash.yml | 3 --- .../issue_655__wait_for_connection_error.yml | 7 +++---- tests/two_three_compat_test.py | 7 +++++++ 9 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 94919e350..51908e928 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -16,32 +16,21 @@ trigger: - docs-master jobs: -- job: mac11 +- job: mac12 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md - vmImage: macOS-11 + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + vmImage: macOS-12 strategy: matrix: - Mito_27: - tox.env: py27-mode_mitogen Mito_312: - python.version: '3.12' tox.env: py312-mode_mitogen - - Loc_27_210: - tox.env: py27-mode_localhost-ansible2.10 Loc_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9 - - Van_27_210: - tox.env: py27-mode_localhost-ansible2.10-strategy_linear Van_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9-strategy_linear - job: Linux diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d77910e8..ffba69fa5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts "ValueError: filedescriptor out of range in select()" * :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` +* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs v0.3.7 (2024-04-08) diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index e7a849c22..912d1ba81 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -21,8 +21,7 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - 'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg) - or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg) + (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast') or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') fail_msg: out={{out}} when: diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ecd..616405790 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index d465ac9e8..c5e264ae4 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index 23eeeb553..f5e338621 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index d5f0815b5..ff102b135 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -19,9 +19,6 @@ # Will crash if process has a nonexistent CWD. - custom_python_os_getcwd: - script: | - import os - self._connection.get_chain().call(os.getcwd) tags: - issue_591 - mitogen_only diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index 9ad42a106..4972d91a2 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -11,11 +11,10 @@ tasks: - meta: end_play when: - # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11 - # is no longer supported by homebrew, so the following install - # task fails. + # TODO CI currently runs on macOS 12 & which isn't supported by Podman + # version available in Homebrew. - ansible_facts.system == 'Darwin' - - ansible_facts.distribution_major_version == '11' + - ansible_facts.distribution_version is version('13.0', '<', strict=True) - name: set up test container and run tests inside it block: diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index 4490e5d26..ab9f4e19e 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,3 +1,6 @@ +import os +import unittest + import mitogen.core import testlib @@ -7,6 +10,10 @@ # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right. # There should be 100 tests in this file. +@unittest.skipIf( + os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, + "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", +) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' From fe435bb7d069764b35a8ca77b05cffe19dbd9c1e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:30:25 +0100 Subject: [PATCH 3/3] CI: Workaround "No module named 'setuptools.command.test'" Pip 72 was released yesterday (2024-07-28), dropping `setup.py test` support. hdrhistogram 0.6.1 requires it to install. For now constrain Pip to earlier releases, so our tests can be run. refs #1090 --- tests/ansible/requirements.txt | 3 +++ tests/constraints.txt | 3 +++ tox.ini | 1 + 3 files changed, 7 insertions(+) create mode 100644 tests/constraints.txt diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 2c3c87c86..8cfb348ab 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,7 @@ paramiko==2.3.2 # Last 2.6-compat version. +# Incompatible with pip >= 72, due to removal of `setup.py test`: +# ModuleNotFoundError: No module named 'setuptools.command.test' +# https://github.com/pypa/setuptools/issues/4519 hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..6adaa30be --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,3 @@ +# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it. +# TODO Bump dependencies and unconstrain Pip. +setuptools<72 diff --git a/tox.ini b/tox.ini index 55fd1842e..a0b0e1ae0 100644 --- a/tox.ini +++ b/tox.ini @@ -109,6 +109,7 @@ setenv = ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 + PIP_CONSTRAINT={toxinidir}/tests/constraints.txt # Only applicable to MODE=mitogen distro_centos5: DISTRO=centos5 distro_centos6: DISTRO=centos6