Merge branch 'main' into bootc-provision

.packit.yaml

@@ -61,9 +61,9 @@ jobs:
     require: &require-full-tests
       label:
         present:
-          - full test
+          - ci | full test
         absent:
-          - discuss
+          - status | discuss
     identifier: full
     tf_extra_params:
       test:

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.11.1"
+    rev: "v1.11.2"
     hooks:
       - id: mypy
         language_version: "3.9"
@@ -44,7 +44,7 @@ repos:
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
 
           # report-junit
-          - "junit_xml>=1.9"
+          - "lxml>=4.6.5"
 
           - "typing-extensions>=4.9.0; python_version < '3.13'"
           - "pytest"
@@ -62,12 +62,13 @@ repos:
           - "types-jinja2"
           - "types-babel"
           - "types-docutils"
+          - "types-lxml"
 
         pass_filenames: false
         args: [--config-file=pyproject.toml]
 
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.371
+    rev: v1.1.379
     hooks:
       - id: pyright
         language_version: "3.9"
@@ -90,7 +91,7 @@ repos:
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
 
           # report-junit
-          - "junit_xml>=1.9"
+          - "lxml>=4.6.5"
 
           - "typing-extensions>=4.9.0; python_version < '3.13'"
           - "pytest"
@@ -108,9 +109,10 @@ repos:
           - "types-jinja2"
           - "types-babel"
           - "types-docutils"
+          - "types-lxml"
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: "0.29.1"
+    rev: "0.29.2"
     hooks:
       - id: check-metaschema
         name: "Check JSON schemas validity"
@@ -139,15 +141,15 @@ repos:
           # in order to be parsed by ansible-lint
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.5.7
+    rev: v0.6.4
     hooks:
       - id: ruff
         args:
           - '--fix'
           - '--show-fixes'
 
   - repo: https://github.com/teemtee/tmt.git
-    rev: 1.35.0
+    rev: 1.36.0
     hooks:
     - id: tmt-lint
       additional_dependencies:
@@ -167,7 +169,7 @@ repos:
           - tomli  # Required for python < 3.11
 
   - repo: https://github.com/djlint/djLint
-    rev: v1.34.1
+    rev: v1.35.2
     hooks:
       - id: djlint
         files: "\\.j2"

Makefile

@@ -86,6 +86,7 @@ images:  ## Build tmt images for podman/docker
 	podman build -t tmt --squash -f ./containers/Containerfile.mini .
 	podman build -t tmt-all --squash -f ./containers/Containerfile.full .
 
+TMT_TEST_IMAGE_BASES = $(shell grep -rh 'FROM ' containers/ | cut -d' ' -f2 | sort | uniq)
 TMT_TEST_IMAGE_TARGET_PREFIX = images-tests
 TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX = tmt/tests/container
 
@@ -117,6 +118,9 @@ TMT_TEST_IMAGES_TARGETS := $(foreach image,$(TMT_TEST_CONTAINER_IMAGES),images-t
 images-tests: $(TMT_TEST_IMAGES_TARGETS)  ## Build customized images for tests
 	podman images | grep 'localhost/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/' | sort
 
+images-tests-bases:  ## Download base images for custom test images
+	podman pull $(TMT_TEST_IMAGE_BASES)
+
 define test-container-image-target-to-name =
 $(subst $(TMT_TEST_IMAGE_TARGET_PREFIX)/,,${1})
 endef

docs/contribute.rst

@@ -662,6 +662,14 @@ make clean
 Release
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+The ``tmt`` project is released monthly. If there are urgent
+changes which need to be released quickly, a hotfix release may be
+created to address the important problem sooner.
+
+
+Regular
+------------------------------------------------------------------
+
 Follow the steps below to create a new major or minor release:
 
 * Update ``overview.rst`` with new contributors since the last release
@@ -700,3 +708,20 @@ __ https://tmt.readthedocs.io/en/stable/releases.html
 __ https://src.fedoraproject.org/rpms/tmt/pull-requests
 __ https://copr.fedorainfracloud.org/coprs/g/teemtee/tmt/builds/
 __ https://pypi.org/project/tmt/
+
+
+Hotfix
+------------------------------------------------------------------
+
+The following steps should be followed when an important urgent
+fix needs to be released before the regular schedule:
+
+* Create a new branch from the ``fedora`` branch
+* Use ``git cherry-pick`` to apply the selected change
+* Mention the hotfix release on the release page
+* Add a ``Release x.y.z`` commit, empty if needed: ``git commit --allow-empty -m "Release x.y.z"``
+* Create a new pull request with the target branch set to ``fedora``
+* Make sure that tests pass and merge the pull request
+* Tag the commit and publish the release in the same way as for
+  regular release
+* Create a pull request with the hotfix release notes changes

docs/releases.rst

@@ -4,8 +4,36 @@
     Releases
 ======================
 
+tmt-1.37.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`/plugins/report/junit` report plugin now validates all the XML
+flavors against their respective XSD schemas and tries to prettify the final
+XML output. These functionalities are always disabled for ``custom`` flavors.
+The prettify functionality can be controlled for non-custom templates by
+``--prettify`` and ``--no-prettify`` arguments.
+
+The :ref:`/plugins/report/junit` report plugin now uses Jinja instead of
+``junit-xml`` library to generate the JUnit XMLs. It also adds support for a
+new ``--flavor`` argument. Using this argument the user can choose between a
+``default`` flavor, which keeps the current behavior untouched, and a
+``custom`` flavor where user must provide a custom template using a
+``--template-path`` argument.
+
+
+tmt-1.36.1
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+tmt will now put SSH master control socket into ``ssh-socket``
+subdirectory of a workdir. Originally, sockets were stored in
+``/run/user/$UID`` directory, but this path led to conflicts when
+multiple tmt instances shared sockets incorrectly. A fix landed in
+1.36 that put sockets into ``provision`` subdirectory of each plan,
+but this solution will break for plans with longer names because of
+unavoidable UNIX socket path limit of 104 (or 108) characters.
+
 
-tmt-1.36
+tmt-1.36.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 tmt will now emit a warning when :ref:`custom test results</spec/tests/result>`
@@ -19,10 +47,11 @@ support for submitting jobs on behalf of other users, through
 ``beaker-job-owner`` key. The current user must be a submission delegate
 for the given job owner.
 
-In preparation for subresults: subresults and their checks have been integrated into HTML report and display plugin, result phase renamed to subresult.
+In preparation for subresults: subresults and their checks have been integrated
+into HTML report and display plugin, result phase renamed to subresult.
 
 
-tmt-1.35
+tmt-1.35.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If during test execution guest freezes in the middle of reboot,
@@ -70,7 +99,7 @@ run or from a run with a given id. Users can also choose to skip
 cleaning ``guests``, ``runs`` or ``images``.
 
 
-tmt-1.34
+tmt-1.34.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :ref:`/spec/tests/duration` now supports multiplication.
@@ -88,7 +117,7 @@ the corresponding ``disk[N].size``
 :ref:`HW requirements</spec/hardware/disk>`.
 
 
-tmt-1.33
+tmt-1.33.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :ref:`/plugins/provision/beaker` provision plugin gains
@@ -127,7 +156,7 @@ This order picks the right package manager in the case when the
 guest is ``ostree-booted`` but has the dnf installed.
 
 
-tmt-1.32
+tmt-1.32.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The hardware specification for :ref:`/spec/hardware/disk` has been
@@ -178,7 +207,7 @@ packages required to be present with ``dist-git-require`` option.
 __ https://pagure.io/testcloud/
 
 
-tmt-1.31
+tmt-1.31.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :ref:`/spec/plans/provision` step is now able to perform
@@ -237,7 +266,7 @@ to be additionally updated. Environment variables were rewritten to
 the uniform form ``TMT_PLUGIN_REPORT_REPORTPORTAL_${option}``.
 
 
-tmt-1.30
+tmt-1.30.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The new :ref:`tmt try</stories/cli/try>` command provides an
@@ -287,7 +316,7 @@ See the `full changelog`__ for more details.
 __ https://github.com/teemtee/tmt/releases/tag/1.30.0
 
 
-tmt-1.29
+tmt-1.29.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test directories can be pruned with the ``prune`` option usable in
@@ -327,7 +356,7 @@ See the `full changelog`__ for more details.
 __ https://github.com/teemtee/tmt/releases/tag/1.29.0
 
 
-tmt-1.28
+tmt-1.28.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The new :ref:`/stories/cli/multiple phases/update-missing` option

pyproject.toml

@@ -61,7 +61,8 @@ provision-virtual = [
     ]
 provision-container = []
 report-junit = [
-    "junit_xml>=1.9",
+    # Required to support XML parsing and checking the XSD schemas.
+    "lxml>=4.6.5",
     ]
 report-polarion = [
     "tmt[report-junit]",
@@ -139,6 +140,7 @@ dependencies = [
     "types-jinja2",
     "types-babel",
     "types-docutils",
+    "types-lxml",
     ]
 features = ["all"]
 
@@ -216,7 +218,7 @@ module = [
     "guestfs.*",
     "html2text.*",
     "fmf.*",
-    "junit_xml.*",
+    "lxml.*",
     "libvirt.*",
     "nitrate.*",
     "pylero.*",
@@ -252,6 +254,7 @@ ignore = [
     "tmt/lint.py",
     "tmt/queue.py",
     "tmt/utils/__init__.py",
+    "tmt/utils/structured_field.py",
     "tmt/hardware.py",  # pyright does not pick up pint's _typing.py or something :/
 ]
 

tests/execute/reboot/out-of-session.sh

@@ -42,12 +42,12 @@ rlJournalStart
         tmt_reboot_command="export TMT_TEST_PIDFILE=/var/tmp/tmt-test.pid; export TMT_TEST_PIDFILE_LOCK=/var/tmp/tmt-test.pid.lock; export TMT_DEBUG=1; tmt-reboot"
 
         if [ "$PROVISION_HOW" = "container" ]; then
-            podman_exec="$(sed -nr 's/\s*Run command: (podman exec .*) \/bin\/bash.*cd.*/\1/p' tmt.output)"
+            podman_exec="$(sed -nr 's/\s*Run command: (podman exec .*) \/bin\/bash.*cd.*/\1/p' tmt.output | head -1)"
 
             $podman_exec bash -c "$tmt_reboot_command"
 
         elif [ "$PROVISION_HOW" = "virtual" ]; then
-            ssh_exec="$(sed -nr "s/\s*Run command: (ssh .* -tt root@.*) 'export .*/\1/p" tmt.output)"
+            ssh_exec="$(sed -nr "s/\s*Run command: (ssh .* -tt root@.*) 'export .*/\1/p" tmt.output | head -1)"
 
             $ssh_exec "$tmt_reboot_command"
         fi

tests/provision/ssh-multiplexing/data/.fmf/version

@@ -0,0 +1 @@
+1

tests/provision/ssh-multiplexing/data/plan.fmf

@@ -0,0 +1,8 @@
+execute:
+    how: tmt
+
+discover:
+    how: shell
+    tests:
+    - name: smoke
+      test: /bin/true

tests/provision/ssh-multiplexing/main.fmf

@@ -0,0 +1,5 @@
+summary: Check that SSH multiplexing works as expected
+tag+:
+  - provision-only
+  - provision-connect
+  - provision-virtual

tests/provision/ssh-multiplexing/test.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "PROVISION_HOW=${PROVISION_HOW:-virtual}"
+        rlRun "run=\$(mktemp -d)" 0 "Create run directory"
+        rlRun "long_run=\$(mktemp -d /tmp/tmp.veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongXXX)" 0 "Create run directory with a very long name"
+        rlRun "pushd data"
+    rlPhaseEnd
+
+    rlPhaseStartTest "SSH multiplexing should be enabled by default ($PROVISION_HOW)"
+        rlRun "tmt -vv run -i $run -a provision -h $PROVISION_HOW"
+        rlAssertGrep "Spawning the SSH master process" "$run/log.txt"
+    rlPhaseEnd
+
+    rlPhaseStartTest "SSH multiplexing should be disabled when SSH socket path gets too long ($PROVISION_HOW)"
+        rlRun "tmt -vv run -i $long_run -a provision -h $PROVISION_HOW"
+        rlAssertGrep "warn: SSH multiplexing will not be used because the SSH socket path '.*' is too long." "$long_run/log.txt"
+        rlAssertGrep "The SSH master process cannot be terminated because it is disabled." "$long_run/log.txt"
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlRun "rm -r $run" 0 "Remove run directory"
+        rlRun "rm -r $long_run" 0 "Remove run directory with the long name"
+    rlPhaseEnd
+rlJournalEnd

tests/report/junit/data/custom.xml.j2

@@ -0,0 +1,6 @@
+<?xml version="1.0" ?>
+<tests>
+    {% for result in RESULTS %}
+        <test name="{{ result.name | e }}" value="{{ result.result.value | e }}"/>
+    {% endfor %}
+</tests>

tests/report/junit/data/main.fmf

@@ -27,3 +27,5 @@
         /timeout:
             test: sleep 10
             duration: 2s
+        /escape"<speci&l>_chars:
+            test: "echo '<speci&l>\"chars'"

tests/report/junit/data/non-xml-custom.j2

@@ -0,0 +1,3 @@
+{% for result in RESULTS %}
+name='{{ result.name | e }}',value='{{ result.result.value | e }}'
+{% endfor %}