require test manifest path for running tests (#1229)

* require test manifest path for running tests

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* fix style

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* fix the CI manifest checks

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* address comments

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* update README.md

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* address feedback

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* fix import

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* refactor CI manifests

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* update readme

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* more docs

Signed-off-by: Tianle Huang <tianleh@amazon.com>

* remove type

Signed-off-by: Tianle Huang <tianleh@amazon.com>

ONBOARDING.md

@@ -42,12 +42,12 @@ Add the new plugin to the [opensearch-plugins meta](https://github.com/opensearc
 
 ### Onboard to `test-workflow`
 
-1. Update the test configuration file, [test_manifest.yml](src/test_workflow/config/opensearch/test_manifest.yml), for a particular release, to include your plugin. This test configuration defines full suite of tests - `integ`, `bwc`, that can be run on the plugin.
+1. Update the test configuration file (use 1.3.0 as an example), [opensearch-1.3.0-test.yml](manifests/1.3.0/opensearch-1.3.0-test.yml), for a particular release, to include your plugin. This test configuration defines full suite of tests - `integ`, `bwc`, that can be run on the plugin.
 
-2. For integration testing, the `test-workflow` runs integration tests available in the plugin repository. You will need to add `integ-test` config for your plugin in test_manifest.yml, [example](src/test_workflow/config/opensearch/test_manifest.yml#L3).
+2. For integration testing, the `test-workflow` runs integration tests available in the plugin repository. You will need to add `integ-test` config for your plugin in opensearch-1.3.0-test.yml, [example](manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml).
 
     1. It supports two test configs - `with-security` and `without-security`, which runs test with security plugin enabled and disabled respectively. Choose one or both depending on what your plugin integration tests support.
 
     2. If your plugin is dependent on `job-scheduler` zip, you can define that in `build-dependencies` in the config. Currently, the test workflow only supports `job-scheduler` as build dependency. Please create an issue if your plugin needs more support.
 
-3. For backward compatibility testing, the `test-workflow` runs backward compatibility tests available in the plugin repository, (see [reference]((https://github.com/opensearch-project/anomaly-detection/blob/d9a122d05282f7efc1e24c61d64f18dec0fd47af/build.gradle#L428))). Like integration test, it has a set of configurable options defined in test_manifest.yml, [example](src/test_workflow/config/opensearch/test_manifest.yml#L8).
+3. For backward compatibility testing, the `test-workflow` runs backward compatibility tests available in the plugin repository, (see [reference]((https://github.com/opensearch-project/anomaly-detection/blob/d9a122d05282f7efc1e24c61d64f18dec0fd47af/build.gradle#L428))). Like integration test, it has a set of configurable options defined in opensearch-1.3.0-test.yml, [example](manifests/1.3.0/opensearch-1.3.0-test.yml).

README.md

@@ -82,7 +82,7 @@ See [jenkins](./jenkins) and [docker](./docker) for more information.
 Tests the OpenSearch distribution, including integration, backwards-compatibility and performance tests.
 
 ```bash
-./test.sh <test-type> <path>
+./test.sh <test-type> <test-manifest-path> <path>
 ```
 
 See [src/test_workflow](./src/test_workflow) for more information.

jenkins/test/testsuite/Jenkinsfile

@@ -60,7 +60,7 @@ pipeline {
                 script {
                     basePath = "https://ci.opensearch.org/ci/dbc/bundle-build/${opensearch_version}/${build_id}/${platform}/${architecture}"
                     sh "wget ${basePath}/builds/opensearch/manifest.yml"
-                    sh "./test.sh ${JOB_NAME} ${basePath} --test-run-id ${env.BUILD_NUMBER}"
+                    sh "./test.sh ${JOB_NAME} manifests/${opensearch_version}/opensearch-${opensearch_version}-test.yml ${basePath} --test-run-id ${env.BUILD_NUMBER}"
                 }
             }
             post {

src/ci_workflow/ci_input_manifest.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+
+from ci_workflow.ci_check_lists import CiCheckLists
+from ci_workflow.ci_manifest import CiManifest
+from ci_workflow.ci_target import CiTarget
+from manifests.input_manifest import InputManifest
+from system.temporary_directory import TemporaryDirectory
+
+
+class CiInputManifest(CiManifest):
+    def __init__(self, file, args):
+        super().__init__(InputManifest.from_file(file), args)
+
+    def __check__(self):
+
+        target = CiTarget(version=self.manifest.build.version, snapshot=self.args.snapshot)
+
+        with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
+            logging.info(f"Sanity-testing in {work_dir.name}")
+
+            logging.info(f"Sanity testing {self.manifest.build.name}")
+
+            for component in self.manifest.components.select(focus=self.args.component):
+                logging.info(f"Sanity testing {component.name}")
+
+                ci_check_list = CiCheckLists.from_component(component, target)
+                ci_check_list.checkout(work_dir.name)
+                ci_check_list.check()
+                logging.info("Done.")

src/ci_workflow/ci_manifest.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import abc
+import logging
+
+
+class CiManifest(abc.ABC):
+    def __init__(self, manifest, args):
+        self.manifest = manifest
+        self.args = args
+
+    def check(self):
+        try:
+            self.__check__()
+        except:
+            logging.error("CI Manifest check failed")
+            raise

src/ci_workflow/ci_manifests.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+import re
+
+from ci_workflow.ci_input_manifest import CiInputManifest
+from ci_workflow.ci_test_manifest import CiTestManifest
+
+
+class CiManifests:
+    def __klass(filename):
+        if re.search("-test.yml$", filename):
+            return CiTestManifest
+        else:
+            return CiInputManifest
+
+    @classmethod
+    def from_file(cls, file, args):
+        return cls.__klass(file.name)(file, args)

src/ci_workflow/ci_test_manifest.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+
+import logging
+
+from ci_workflow.ci_manifest import CiManifest
+from manifests.test_manifest import TestManifest
+
+
+class CiTestManifest(CiManifest):
+    def __init__(self, file, args):
+        super().__init__(TestManifest.from_file(file), args)
+
+    def __check__(self):
+        assert self.manifest
+        logging.info("TestManifest schema validation succeeded")
+        logging.info("Done.")

src/run_ci.py

@@ -6,41 +6,19 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-import logging
+
 import sys
 
 from ci_workflow.ci_args import CiArgs
-from ci_workflow.ci_check_lists import CiCheckLists
-from ci_workflow.ci_target import CiTarget
-from manifests.input_manifest import InputManifest
+from ci_workflow.ci_manifests import CiManifests
 from system import console
-from system.temporary_directory import TemporaryDirectory
 
 
 def main():
     args = CiArgs()
     console.configure(level=args.logging_level)
-    manifest = InputManifest.from_file(args.manifest)
-
-    target = CiTarget(version=manifest.build.version, snapshot=args.snapshot)
-
-    with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
-        logging.info(f"Sanity-testing in {work_dir.name}")
-
-        logging.info(f"Sanity testing {manifest.build.name}")
-
-        for component in manifest.components.select(focus=args.component):
-            logging.info(f"Sanity testing {component.name}")
-
-            try:
-                ci_check_list = CiCheckLists.from_component(component, target)
-                ci_check_list.checkout(work_dir.name)
-                ci_check_list.check()
-            except:
-                logging.error(f"Error checking {component.name}, retry with: {args.component_command(component.name)}")
-                raise
 
-    logging.info("Done.")
+    CiManifests.from_file(args.manifest, args).check()
 
 
 if __name__ == "__main__":

src/run_integ_test.py

@@ -24,8 +24,7 @@
 def main():
     args = TestArgs()
     console.configure(level=args.logging_level)
-    test_manifest_path = os.path.join(os.path.dirname(__file__), "test_workflow", "config", "opensearch", "test_manifest.yml")
-    test_manifest = TestManifest.from_path(test_manifest_path)
+    test_manifest = TestManifest.from_path(args.test_manifest_path)
     bundle_manifest = BundleManifest.from_urlpath("/".join([args.path.rstrip("/"), "dist/opensearch/manifest.yml"]))
     build_manifest = BuildManifest.from_urlpath("/".join([args.path.rstrip("/"), "builds/opensearch/manifest.yml"]))
     dependency_installer = DependencyInstallerOpenSearch(args.path, build_manifest, bundle_manifest)

src/test_workflow/README.md

@@ -26,6 +26,7 @@ The following options are available.
 | name                 | description                                                             |
 |----------------------|-------------------------------------------------------------------------|
 | test-type            | Run tests of a test suite. [integ-test, bwc-test, perf-test]            |
+| test-manifest-path   | Specify a test manifest path                                            |
 | path                 | Location of manifest(s).                                                |
 | --test-run-id        | Unique identifier for a test run                                        |
 | --component          | Test a specific component in a manifest                                 |
@@ -41,21 +42,21 @@ To run integration tests locally, use below command. This pulls down the built b
 Usage:
 
 ```bash
-./test.sh integ-test <target>
+./test.sh integ-test <test-manifest-path> <target>
 ```
 
 For example, build locally and run integration tests.
 
 ```bash
-./build.sh manifests/1.2.0/opensearch-1.2.0.yml
+./build.sh manifests/1.3.0/opensearch-1.3.0.yml
 ./assemble.sh builds/opensearch/manifest.yml
-./test.sh integ-test . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml"
+./test.sh integ-test manifests/1.3.0/opensearch-1.3.0-test.yml . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml"
 ```
 
 Or run integration tests against an existing build.
 
 ```bash
-./test.sh integ-test https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml
+./test.sh integ-test manifests/1.3.0/opensearch-1.3.0-test.yml https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml
 ```
 
 ### Backwards Compatibility Tests
@@ -65,7 +66,7 @@ Runs backward compatibility invoking `run_bwc_test.py` in each component from a
 Usage:
 
 ```bash
-./test.sh bwc-test <target>
+./test.sh bwc-test <test-manifest-path> <target>
 ```
 
 ### Performance Tests
@@ -80,7 +81,7 @@ The CI/CD infrastructure is divided into two main workflows - `build` and `test`
 
 Once a new distribution is ready, the `build-job` kicks off the [test-orchestrator-pipeline](../../jenkins_workflow/test/orchestrator/Jenkinsfile) with input parameters `(build_id, architecture, opensearch_version)` that uniquely identify the bundle. The test orchestrator-pipeline generate a unique `test_run_id`, that uniquely identifies the test execution and invokes all three test jobs - `integ-test, bwc-test, perf-test` in parallel.
 
-The [integ-test job](../../jenkins_workflow/test/testsuite/Jenkinsfile) starts by pulling the manifest files and installing the required dependencies for running plugin integration tests. It then kicks off the integration test for each plugin based on the `test-configs` defined in [test-manifest.yml](config/opensearch/test_manifest.yml). It executes each configuration separately from others by spinning up a dedicated local test cluster. It uses `integtest.sh` script to run the integration test. There is a [default](../../scripts/default/integtest.sh) version of this script present in opensearch-build repo and also allows plugins to override the default by having a custom integtest.sh in plugin repo.
+The [integ-test job](../../jenkins_workflow/test/testsuite/Jenkinsfile) starts by pulling the manifest files and installing the required dependencies for running plugin integration tests. It then kicks off the integration test for each plugin based on the `test-configs` defined in [opensearch-1.3.0-test.yml](manifests/1.3.0/opensearch-1.3.0-test.yml). It executes each configuration separately from others by spinning up a dedicated local test cluster. It uses `integtest.sh` script to run the integration test. There is a [default](../../scripts/default/integtest.sh) version of this script present in opensearch-build repo and also allows plugins to override the default by having a custom integtest.sh in plugin repo.
 
 Once all tests complete, the notifications job can send out the notifications to the subscribed channels. Below figure illustrates how different components of the test workflow would interact with each other. 
 
@@ -126,7 +127,7 @@ The development is tracked by [meta issue #126](https://github.com/opensearch-pr
 
 Manifest files are configurations for a particular bundle. `test-workflow` uses three types of manifest files to run test suites. 
 
-1. `test-manifest.yml` provides a list of test configurations to run against a given component in the bundle. An example of a configuration would be, integration test `index-management` plugin `with-security` and `without-security`. This manifest file serves as a support matrix config for the testing and should be updated by plugins if new components or test suites are to be added as part of the release workflow. See [here](config/opensearch/test_manifest.yml)
+1. `test-manifest.yml` provides a list of test configurations to run against a given component in the bundle. An example of a configuration would be, integration test `index-management` plugin `with-security` and `without-security`. This manifest file serves as a support matrix config for the testing and should be updated by plugins if new components or test suites are to be added as part of the release workflow. See [here](manifests/1.3.0/opensearch-1.3.0-test.yml)
 2. `build-manifest.yml` created by the build-workflow and provides a list of artifacts built as part of the build-workflow. It assists `test-workflow` pull the maven and build dependencies to run the test suites.
 3. `bundle-manfest.yml` created by the build-workflow and provides a list of components packaged in a given bundle.  It assists `test-workflow` to identify what components should be tested for a given bundle.
 

src/test_workflow/test_args.py

@@ -23,6 +23,7 @@ class TestArgs:
 
     def __init__(self):
         parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
+        parser.add_argument("test_manifest_path", type=str, help="Specify a test manifest path.")
         parser.add_argument("path", type=str, help="Location of build and bundle manifests.", default=".")
         parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test")
         parser.add_argument("--component", type=str, help="Test a specific component instead of the entire distribution.")
@@ -37,5 +38,7 @@ def __init__(self):
         self.logging_level = args.logging_level
         self.path = args.path if validators.url(args.path) else os.path.realpath(args.path)
 
+        self.test_manifest_path = args.test_manifest_path if validators.url(args.test_manifest_path) else os.path.realpath(args.test_manifest_path)
+
 
 TestArgs.__test__ = False  # type:ignore

tests/test_run_ci.py

@@ -7,7 +7,7 @@
 import os
 import tempfile
 import unittest
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 import pytest
 
@@ -30,11 +30,29 @@ def test_usage(self):
     OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.1.1/opensearch-1.1.1.yml"))
 
     @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_MANIFEST])
-    @patch("run_ci.TemporaryDirectory")
-    @patch("run_ci.CiCheckLists.from_component")
+    @patch("ci_workflow.ci_input_manifest.TemporaryDirectory")
+    @patch("ci_workflow.ci_input_manifest.CiCheckLists.from_component")
     def test_main(self, mock_lists, mock_temp, *mocks):
         mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
         main()
         self.assertNotEqual(mock_lists.call_count, 0)
         self.assertEqual(mock_lists.return_value.checkout.call_count, mock_lists.call_count)
         self.assertEqual(mock_lists.return_value.check.call_count, mock_lists.call_count)
+
+    OPENSEARCH_TEST_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.3.0/opensearch-1.3.0-test.yml"))
+
+    @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_TEST_MANIFEST])
+    @patch("logging.info")
+    def test_main_test_manifest(self, mock_logging, *mocks):
+        main()
+        mock_logging.assert_has_calls([
+            call("TestManifest schema validation succeeded"),
+            call("Done.")
+        ])
+
+    OPENSEARCH_TEST_MANIFEST_NOT_EXIST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.4.0/opensearch-1.3.0-test.yml"))
+
+    @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_TEST_MANIFEST_NOT_EXIST])
+    def test_main_test_manifest_empty(self, *mocks):
+        with self.assertRaises(SystemExit):
+            main()

tests/tests_test_workflow/test_bwc_workflow/bwc_test/test_run_bwc_test.py

@@ -14,8 +14,12 @@
 class TestRunBwcTest(unittest.TestCase):
     @patch(
         "argparse._sys.argv",
-        ["run_bwc_test.py", os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote", "dist", "opensearch", "manifest.yml")])
-    @patch("run_bwc_test.BwcTestSuite")
+        [
+            "run_bwc_test.py",
+            os.path.join(os.path.dirname(__file__), "..", "..", "data", "test_manifest.yml"),
+            os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote", "dist", "opensearch", "manifest.yml")
+        ])
+    @ patch("run_bwc_test.BwcTestSuite")
     def test_run_bwc_test(self, mock_bwc_suite, *mock):
         main()
         self.assertEqual(mock_bwc_suite.return_value.execute.call_count, 1)

tests/tests_test_workflow/test_integ_workflow/integ_test/test_run_integ_test.py

@@ -12,7 +12,13 @@
 
 
 class TestRunIntegTest(unittest.TestCase):
-    @patch("argparse._sys.argv", ["run_integ_test.py", os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote")])
+    @patch(
+        "argparse._sys.argv",
+        [
+            "run_integ_test.py",
+            os.path.join(os.path.dirname(__file__), "..", "..", "data", "test_manifest.yml"),
+            os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote")
+        ])
     @patch("run_integ_test.DependencyInstallerOpenSearch")
     @patch("run_integ_test.TestSuiteResults")
     @patch("run_integ_test.IntegTestSuiteOpenSearch")