vdk-plugins: introduced vdk-audit plugin

projects/vdk-plugins/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0

projects/vdk-plugins/vdk-audit/.plugin-ci.yml

@@ -0,0 +1,24 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+.build-vdk-audit:
+  variables:
+    PLUGIN_NAME: vdk-audit
+  extends: .build-plugin
+
+build-py38-vdk-audit:
+  extends: .build-vdk-audit
+  image: "python:3.8"
+
+build-py39-vdk-audit:
+  extends: .build-vdk-audit
+  image: "python:3.9"
+
+build-py310-vdk-audit:
+  extends: .build-vdk-audit
+  image: "python:3.10"
+
+release-vdk-audit:
+  variables:
+    PLUGIN_NAME: vdk-audit
+  extends: .release-plugin

projects/vdk-plugins/vdk-audit/README.md

@@ -0,0 +1,17 @@
+## Versatile Data Kit Audit Plugin
+
+Visibility into the actions provides opportunities for test frameworks, logging
+frameworks, and security tools to monitor and optionally limit actions taken by the
+runtime.
+This plugin provides an ability to audit and potentially limit user operations.
+These operations are typically deep within the Python runtime or standard library, such
+as dynamic code compilation, module imports or OS command invocations. In order to have a
+better understanding of what exactly the job does, we will log all job operations.
+
+### Usage
+
+To use the plugin, just install it using
+
+```bash
+pip install vdk-audit
+```

projects/vdk-plugins/vdk-audit/requirements.txt

@@ -0,0 +1,4 @@
+vdk-core
+
+# testing
+vdk-test-utils

projects/vdk-plugins/vdk-audit/setup.py

@@ -0,0 +1,30 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import pathlib
+
+import setuptools
+
+
+__version__ = "0.1.0"
+
+setuptools.setup(
+    name="vdk-audit",
+    version=__version__,
+    url="https://github.com/vmware/versatile-data-kit",
+    description="Versatile Data Kit SDK Audit plugin restricts forbidden operations.",
+    long_description=pathlib.Path("README.md").read_text(),
+    long_description_content_type="text/markdown",
+    install_requires=["vdk-core"],
+    package_dir={"": "src"},
+    packages=setuptools.find_namespace_packages(where="src"),
+    entry_points={"vdk.plugin.run": ["vdk-audit = vdk.plugin.audit.audit_plugin"]},
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
+)

projects/vdk-plugins/vdk-audit/src/vdk/plugin/audit/audit_config.py

@@ -0,0 +1,67 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+from vdk.internal.core.config import ConfigurationBuilder
+
+AUDIT_HOOK_ENABLED = "AUDIT_HOOK_ENABLED"
+AUDIT_HOOK_FORBIDDEN_EVENTS_LIST = "FORBIDDEN_EVENTS_LIST"
+AUDIT_HOOK_EXIT_ON_FORBIDDEN_EVENT = "EXIT_ON_FORBIDDEN_EVENT"
+AUDIT_HOOK_EXIT_CODE = "EXIT_CODE"
+AUDIT_HOOK_FORBIDDEN_EVENTS_LIST_DEFAULT = (
+    "os.system;os.chdir;os.chflags;os.chmod;os.chown;os.fork;"
+    "os.forkpty;os.getxattr;os.kill;os.killpg;os.link;os.listxattr;"
+    "os.lockf;os.posix_spawn;os.putenv;os.removexattr;os.rmdir;"
+    "os.scandir;os.setxattr;os.spawn;os.startfile;os.symlink;"
+    "os.truncate;os.unsetenv;os.utime;pty.spawn"
+)
+
+
+class AuditConfiguration:
+    def __init__(self, config):
+        self.__config = config
+
+    def enabled(self):
+        return self.__config.get_value(AUDIT_HOOK_ENABLED)
+
+    def forbidden_events_list(self):
+        return self.__config.get_value(AUDIT_HOOK_FORBIDDEN_EVENTS_LIST)
+
+    def exit_code(self):
+        return self.__config.get_value(AUDIT_HOOK_EXIT_CODE)
+
+    def exit_on_forbidden_event(self):
+        return self.__config.get_value(AUDIT_HOOK_EXIT_ON_FORBIDDEN_EVENT)
+
+
+def add_definitions(config_builder: ConfigurationBuilder) -> None:
+    config_builder.add(
+        key=AUDIT_HOOK_ENABLED,
+        default_value=True,
+        description="Set to false if you want to disable audit hook plugin entirely.",
+    )
+    config_builder.add(
+        key=AUDIT_HOOK_FORBIDDEN_EVENTS_LIST,
+        default_value=AUDIT_HOOK_FORBIDDEN_EVENTS_LIST_DEFAULT,
+        description="List of forbidden user operations. These operations are "
+        "typically deep within the Python runtime or standard library, "
+        "such as dynamic code compilation, module imports or OS command "
+        "invocations. "
+        "The field accepts semi-colon separated values. "
+        "Example: 'os.removexattr;os.rename;os.rmdir;os.scandir'",
+    )
+    config_builder.add(
+        key=AUDIT_HOOK_EXIT_ON_FORBIDDEN_EVENT,
+        default_value=True,
+        description="If it is true, the data job will be fully terminated on forbidden "
+        "operation - no cleanup, no per-attempt notifications, etc. "
+        "If it is false, the termination of data job on forbidden "
+        "operation will be disabled.",
+    )
+    config_builder.add(
+        key=AUDIT_HOOK_EXIT_CODE,
+        default_value=0,
+        description="If AUDIT_HOOK_EXIT_ON_FORBIDDEN_EVENT is true, "
+        "the data job will be fully terminated on forbidden operation "
+        "with the exit code defined via this field.",
+    )

projects/vdk-plugins/vdk-audit/src/vdk/plugin/audit/audit_plugin.py

@@ -0,0 +1,58 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+import sys
+from typing import List
+
+from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import IPluginRegistry
+from vdk.internal.builtin_plugins.run.job_context import JobContext
+from vdk.internal.core.config import ConfigurationBuilder
+from vdk.plugin.audit.audit_config import add_definitions
+from vdk.plugin.audit.audit_config import AuditConfiguration
+
+
+logger = logging.getLogger(__name__)
+
+
+class AuditPlugin:
+    @staticmethod
+    @hookimpl
+    def vdk_configure(config_builder: ConfigurationBuilder) -> None:
+        add_definitions(config_builder)
+
+    @hookimpl
+    def initialize_job(self, context: JobContext) -> None:
+        self._config = AuditConfiguration(context.core_context.configuration)
+
+        if not self._config.enabled():
+            return
+
+        forbidden_events_list = self._config.forbidden_events_list().split(";")
+
+        def _audit(event, args):
+            if any(
+                event in not_permitted_event
+                for not_permitted_event in forbidden_events_list
+            ):
+                logger.warning(
+                    f'[Audit] Detected FORBIDDEN operation "{event}" with '
+                    f'arguments "{args}" '
+                )
+
+                if self._config.exit_on_forbidden_event():
+                    logger.error(
+                        f"[Audit] Terminating the data job due to the FORBIDDEN "
+                        f'operation "{event}" with arguments "{args}" '
+                    )
+                    os._exit(self._config.exit_code())
+
+        sys.addaudithook(_audit)
+
+
+@hookimpl
+def vdk_start(plugin_registry: IPluginRegistry, command_line_args: List):
+    plugin_registry.load_plugin_with_hooks_impl(AuditPlugin(), "audit-plugin")

projects/vdk-plugins/vdk-audit/tests/functional/jobs/os-listdir-command-job/10_os_listdir_command_job.py

@@ -0,0 +1,9 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+
+from vdk.api.job_input import IJobInput
+
+
+def run(job_input: IJobInput):
+    os.listdir(".")

projects/vdk-plugins/vdk-audit/tests/functional/jobs/os-system-command-job/10_os_system_command_job.py

@@ -0,0 +1,9 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+
+from vdk.api.job_input import IJobInput
+
+
+def run(job_input: IJobInput):
+    os.system("ls")

projects/vdk-plugins/vdk-audit/tests/functional/test_audit_plugin.py

@@ -0,0 +1,132 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+from unittest import mock
+
+from click.testing import Result
+from vdk.plugin.audit import audit_plugin
+from vdk.plugin.test_utils.util_funcs import cli_assert_equal
+from vdk.plugin.test_utils.util_funcs import CliEntryBasedTestRunner
+from vdk.plugin.test_utils.util_funcs import jobs_path_from_caller_directory
+
+
+def test_audit_multiple_events_disabled_and_forbidden_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "False",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system;os.startfile;os.symlink",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-system-command-job")]
+        )
+
+        print(result.output)
+        cli_assert_equal(0, result)
+        assert not os._exit.called
+
+
+def test_audit_multiple_events_disabled_and_permitted_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "False",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system;os.startfile;os.symlink",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-listdir-command-job")]
+        )
+
+        print(result.output)
+        cli_assert_equal(0, result)
+        assert not os._exit.called
+
+
+def test_audit_single_event_enabled_and_forbidden_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "True",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system",
+            "VDK_AUDIT_HOOK_EXIT_CODE": "0",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-system-command-job")]
+        )
+
+        print(result.output)
+        os._exit.assert_called_with(0)
+
+
+def test_audit_single_event_with_semicolon_enabled_and_forbidden_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "True",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system;",
+            "VDK_AUDIT_HOOK_EXIT_CODE": "0",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-system-command-job")]
+        )
+
+        print(result.output)
+        os._exit.assert_called_with(0)
+
+
+def test_audit_multiple_events_enabled_and_forbidden_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "True",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system;os.startfile;os.symlink",
+            "VDK_AUDIT_HOOK_EXIT_CODE": "0",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-system-command-job")]
+        )
+
+        print(result.output)
+        os._exit.assert_called_with(0)
+
+
+def test_audit_multiple_events_enabled_and_permitted_action():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_AUDIT_HOOK_ENABLED": "True",
+            "VDK_AUDIT_HOOK_FORBIDDEN_EVENTS_LIST": "os.system;os.startfile;os.symlink",
+        },
+    ):
+        os._exit = mock.MagicMock()
+        runner = CliEntryBasedTestRunner(audit_plugin)
+
+        result: Result = runner.invoke(
+            ["run", jobs_path_from_caller_directory("os-listdir-command-job")]
+        )
+
+        print(result.output)
+        cli_assert_equal(0, result)
+        assert not os._exit.called