Tests: implement system test framework

As discussed at length, this is the implementation of the new system
tests framework for shadow. This is a proof of concept that contains the
key elements to be able to run basic user (i.e. useradd, usermod) and
group (i.e. usermod) tests. If you like the framework the rest of the
functionality will be added in the future.

Some useful facts:
* It is implemented in python
* It is based on pytest and pytest-mh
* It works on all the distributions that are part of our CI
* It can be run in the cloud (VM or container) as well as on-premises
* After the execution of each test the environment is cleaned up
* Logs and other artifacts for failed tests are collected
* It has a rich API that can be extended and extended to cover new
  functionalities

Closes: shadow-maint#835

Signed-off-by: Iker Pedrosa <ipedrosa@redhat.com>

tests/system/framework/config.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import Type
+
+from pytest_mh import MultihostConfig, MultihostDomain, MultihostHost, MultihostRole
+
+__all__ = [
+    "ShadowMultihostConfig",
+    "ShadowMultihostDomain",
+]
+
+
+class ShadowMultihostConfig(MultihostConfig):
+    @property
+    def id_to_domain_class(self) -> dict[str, Type[MultihostDomain]]:
+        """
+        All domains are mapped to :class:`ShadowMultihostDomain`.
+
+        :rtype: Class name.
+        """
+        return {"*": ShadowMultihostDomain}
+
+
+class ShadowMultihostDomain(MultihostDomain[ShadowMultihostConfig]):
+    @property
+    def role_to_host_class(self) -> dict[str, Type[MultihostHost]]:
+        """
+        Map roles to classes:
+
+        * shadow to ShadowHost
+
+        :rtype: Class name.
+        """
+        from .hosts.shadow import ShadowHost
+
+        return {
+            "shadow": ShadowHost,
+        }
+
+    @property
+    def role_to_role_class(self) -> dict[str, Type[MultihostRole]]:
+        """
+        Map roles to classes:
+
+        * shadow to Shadow
+
+        :rtype: Class name.
+        """
+        from .roles.shadow import Shadow
+
+        return {
+            "shadow": Shadow,
+        }

tests/system/framework/fixtures.py

@@ -0,0 +1,45 @@
+"""Pytest fixtures."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def datadir(request: pytest.FixtureRequest) -> str:
+    """
+    Data directory shared for all tests.
+
+    :return: Path to the data directory ``(root-pytest-dir)/data``.
+    :rtype: str
+    """
+    return os.path.join(request.node.path, "data")
+
+
+@pytest.fixture(scope="module")
+def moduledatadir(datadir: str, request: pytest.FixtureRequest) -> str:
+    """
+    Data directory shared for all tests within a single module.
+
+    :return: Path to the data directory ``(root-pytest-dir)/data/$module_name``.
+    :rtype: str
+    """
+    name = request.module.__name__
+    return os.path.join(datadir, name)
+
+
+@pytest.fixture(scope="function")
+def testdatadir(moduledatadir: str, request: pytest.FixtureRequest) -> str:
+    """
+    Data directory for current test.
+
+    :return: Path to the data directory ``(root-pytest-dir)/data/$module_name/$test_name``.
+    :rtype: str
+    """
+    if not isinstance(request.node, pytest.Function):
+        raise TypeError(f"Excepted pytest.Function, got {type(request.node)}")
+
+    name = request.node.originalname
+    return os.path.join(moduledatadir, name)

tests/system/framework/hosts/__init__.py

@@ -0,0 +1,3 @@
+"""shadow multihost hosts."""
+
+from __future__ import annotations

tests/system/framework/hosts/base.py

@@ -0,0 +1,107 @@
+"""Base classes and objects for shadow specific multihost hosts."""
+
+from __future__ import annotations
+
+import csv
+
+from pytest_mh import MultihostBackupHost, MultihostHost
+from pytest_mh.utils.fs import LinuxFileSystem
+
+from ..config import ShadowMultihostDomain
+
+__all__ = [
+    "BaseHost",
+    "BaseLinuxHost",
+]
+
+
+class BaseHost(MultihostBackupHost[ShadowMultihostDomain]):
+    """
+    Base class for all shadow hosts.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        # restore is handled in topology controllers
+        super().__init__(*args, **kwargs)
+
+    @property
+    def features(self) -> dict[str, bool]:
+        """
+        Features supported by the host.
+        """
+        return {}
+
+
+class BaseLinuxHost(MultihostHost[ShadowMultihostDomain]):
+    """
+    Base Linux host.
+
+    Adds linux specific reentrant utilities.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self.fs: LinuxFileSystem = LinuxFileSystem(self)
+        self._os_release: dict = {}
+        self._distro_name: str = "unknown"
+        self._distro_major: int = 0
+        self._distro_minor: int = 0
+
+    def _distro_information(self):
+        """
+        Pulls distro information from a host from /ets/os-release
+        """
+        self.logger.info(f"Detecting distro information on {self.hostname}")
+        os_release = self.fs.read("/etc/os-release")
+        self._os_release = dict(csv.reader([x for x in os_release.splitlines() if x], delimiter="="))
+        if "NAME" in self._os_release:
+            self._distro_name = self._os_release["NAME"]
+        if "VERSION_ID" not in self._os_release:
+            return
+        if "." in self._os_release["VERSION_ID"]:
+            self._distro_major = int(self._os_release["VERSION_ID"].split(".", maxsplit=1)[0])
+            self._distro_minor = int(self._os_release["VERSION_ID"].split(".", maxsplit=1)[1])
+        else:
+            self._distro_major = int(self._os_release["VERSION_ID"])
+
+    @property
+    def distro_name(self) -> str:
+        """
+        Host distribution
+
+        :return: Distribution name or "unknown"
+        :rtype: str
+        """
+        # NAME item from os-release
+        if not self._os_release:
+            self._distro_information()
+        return self._distro_name
+
+    @property
+    def distro_major(self) -> int:
+        """
+        Host distribution major version
+
+        :return: Major version
+        :rtype: int
+        """
+        # First part of VERSION_ID from os-release
+        # Returns zero when could not detect
+        if not self._os_release:
+            self._distro_information()
+        return self._distro_major
+
+    @property
+    def distro_minor(self) -> int:
+        """
+        Host distribution minor version
+
+        :return: Minor version
+        :rtype: int
+        """
+        # Second part of VERSION_ID from os-release
+        # Returns zero when no minor version is present
+        if not self._os_release:
+            self._distro_information()
+        return self._distro_minor

tests/system/framework/hosts/shadow.py

@@ -0,0 +1,175 @@
+"""shadow multihost host."""
+
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+from typing import Any
+
+from pytest_mh.conn import ProcessLogLevel
+
+from .base import BaseHost, BaseLinuxHost
+
+__all__ = [
+    "ShadowHost",
+]
+
+
+class ShadowHost(BaseHost, BaseLinuxHost):
+    """
+    shadow host object.
+
+    This is the host where the tests are run.
+
+    .. note::
+
+        Full backup and restore of shadow state is supported.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self._features: dict[str, bool] | None = None
+        """Features dictionary."""
+
+        self._backup_path: PurePosixPath | None = None
+        """Path to backup files."""
+
+        self._verify_files: [dict[str, str]] = [
+            {"origin": "/etc/passwd", "backup": "passwd"},
+            {"origin": "/etc/shadow", "backup": "shadow"},
+            {"origin": "/etc/group", "backup": "group"},
+            {"origin": "/etc/gshadow", "backup": "gshadow"},
+        ]
+        """Files to verify for mismatch."""
+
+    def pytest_setup(self) -> None:
+        super().pytest_setup()
+
+    def start(self) -> None:
+        """
+        Not supported.
+
+        :raises NotImplementedError: _description_
+        """
+        raise NotImplementedError("Starting shadow service is not implemented.")
+
+    def stop(self) -> None:
+        """
+        Not supported.
+
+        :raises NotImplementedError: _description_
+        """
+        raise NotImplementedError("Stopping shadow service is not implemented.")
+
+    def backup(self) -> Any:
+        """
+        Backup all shadow data.
+
+        :return: Backup data.
+        :rtype: Any
+        """
+        self.logger.info("Creating backup of shadow host")
+
+        result = self.conn.run(
+            """
+            set -ex
+
+            function backup {
+                if [ -d "$1" ] || [ -f "$1" ]; then
+                    cp --force --archive "$1" "$2"
+                fi
+            }
+
+            path=`mktemp -d`
+            backup /etc/login.defs "$path/login.defs"
+            backup /etc/default/useradd "$path/useradd"
+            backup /etc/passwd "$path/passwd"
+            backup /etc/shadow "$path/shadow"
+            backup /etc/group "$path/group"
+            backup /etc/gshadow "$path/gshadow"
+            backup /etc/subuid "$path/subuid"
+            backup /etc/subgid "$path/subgid"
+            backup /home "$path/home"
+            backup /var/log/secure "$path/secure"
+
+            echo $path
+            """,
+            log_level=ProcessLogLevel.Error,
+        )
+
+        self._backup_path = PurePosixPath(result.stdout_lines[-1].strip())
+
+        return PurePosixPath(result.stdout_lines[-1].strip())
+
+    def restore(self, backup_data: Any | None) -> None:
+        """
+        Restore all shadow data.
+
+        :return: Backup data.
+        :rtype: Any
+        """
+        if backup_data is None:
+            return
+
+        if not isinstance(backup_data, PurePosixPath):
+            raise TypeError(f"Expected PurePosixPath, got {type(backup_data)}")
+
+        backup_path = str(backup_data)
+
+        self.logger.info(f"Restoring shadow data from {backup_path}")
+        self.conn.run(
+            f"""
+            set -ex
+
+            function restore {{
+                rm --force --recursive "$2"
+                if [ -d "$1" ] || [ -f "$1" ]; then
+                    cp --force --archive "$1" "$2"
+                fi
+            }}
+
+            rm --force --recursive /var/log/secure
+            restore "{backup_path}/login.defs" /etc/login.defs
+            restore "{backup_path}/useradd" /etc/default/useradd
+            restore "{backup_path}/passwd" /etc/passwd
+            restore "{backup_path}/shadow" /etc/shadow
+            restore "{backup_path}/group" /etc/group
+            restore "{backup_path}/gshadow" /etc/gshadow
+            restore "{backup_path}/subuid" /etc/subuid
+            restore "{backup_path}/subgid" /etc/subgid
+            restore "{backup_path}/home" /home
+            restore "{backup_path}/secure" /var/log/secure
+            """,
+            log_level=ProcessLogLevel.Error,
+        )
+
+    def detect_file_mismatches(self) -> None:
+        """
+        Shadow binaries modify a number of files, but usually do not modify all of them. This is why we add an
+        additional check at the end of the test to verify that the files that should not have been modified are still
+        intact.
+        """
+        self.logger.info(f"Detecting mismatches in shadow files {self._backup_path}")
+
+        for x in self._verify_files:
+            result = self.conn.run(
+                f"""
+                set -ex
+
+                cmp {x['origin']} {self._backup_path}/{x['backup']}
+                """,
+                log_level=ProcessLogLevel.Error,
+                raise_on_error=False,
+            )
+            if result.rc != 0:
+                self.logger.error(f"File mismatch in '{x['origin']}' and '{self._backup_path}/{x['backup']}'")
+                result.throw()
+
+    def discard_file(self, origin: str) -> None:
+        """
+        Discard modified files from the files that should be verified.
+        """
+        for x in self._verify_files:
+            if x["origin"] == origin:
+                self._verify_files.remove(x)
+                break