From 86f9cbe30fb24729466f4a1d33978da3ae6596a6 Mon Sep 17 00:00:00 2001 From: Tianle Huang <60111637+tianleh@users.noreply.github.com> Date: Fri, 3 Dec 2021 12:30:02 -0800 Subject: [PATCH] require test manifest path for running tests (#1229) * require test manifest path for running tests Signed-off-by: Tianle Huang * fix style Signed-off-by: Tianle Huang * fix the CI manifest checks Signed-off-by: Tianle Huang * address comments Signed-off-by: Tianle Huang * update README.md Signed-off-by: Tianle Huang * address feedback Signed-off-by: Tianle Huang * fix import Signed-off-by: Tianle Huang * refactor CI manifests Signed-off-by: Tianle Huang * update readme Signed-off-by: Tianle Huang * more docs Signed-off-by: Tianle Huang * remove type Signed-off-by: Tianle Huang --- ONBOARDING.md | 6 ++-- README.md | 2 +- jenkins/test/testsuite/Jenkinsfile | 2 +- .../1.3.0/opensearch-1.3.0-test.yml | 0 .../opensearch-dashboards-1.3.0-test.yml | 0 src/ci_workflow/ci_input_manifest.py | 35 +++++++++++++++++++ src/ci_workflow/ci_manifest.py | 21 +++++++++++ src/ci_workflow/ci_manifests.py | 23 ++++++++++++ src/ci_workflow/ci_test_manifest.py | 21 +++++++++++ src/run_ci.py | 28 ++------------- src/run_integ_test.py | 3 +- src/test_workflow/README.md | 15 ++++---- src/test_workflow/test_args.py | 3 ++ tests/test_run_ci.py | 24 +++++++++++-- .../bwc_test/test_run_bwc_test.py | 8 +++-- .../integ_test/test_run_integ_test.py | 8 ++++- tests/tests_test_workflow/test_test_args.py | 23 ++++++++---- 17 files changed, 171 insertions(+), 51 deletions(-) rename src/test_workflow/config/opensearch/test_manifest.yml => manifests/1.3.0/opensearch-1.3.0-test.yml (100%) rename src/test_workflow/config/opensearch-dashboards/test_manifest.yml => manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml (100%) create mode 100644 src/ci_workflow/ci_input_manifest.py create mode 100644 src/ci_workflow/ci_manifest.py create mode 100644 src/ci_workflow/ci_manifests.py create mode 100644 src/ci_workflow/ci_test_manifest.py diff --git a/ONBOARDING.md b/ONBOARDING.md index 7c192eb162..70266d1134 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -42,12 +42,12 @@ Add the new plugin to the [opensearch-plugins meta](https://github.com/opensearc ### Onboard to `test-workflow` -1. Update the test configuration file, [test_manifest.yml](src/test_workflow/config/opensearch/test_manifest.yml), for a particular release, to include your plugin. This test configuration defines full suite of tests - `integ`, `bwc`, that can be run on the plugin. +1. Update the test configuration file (use 1.3.0 as an example), [opensearch-1.3.0-test.yml](manifests/1.3.0/opensearch-1.3.0-test.yml), for a particular release, to include your plugin. This test configuration defines full suite of tests - `integ`, `bwc`, that can be run on the plugin. -2. For integration testing, the `test-workflow` runs integration tests available in the plugin repository. You will need to add `integ-test` config for your plugin in test_manifest.yml, [example](src/test_workflow/config/opensearch/test_manifest.yml#L3). +2. For integration testing, the `test-workflow` runs integration tests available in the plugin repository. You will need to add `integ-test` config for your plugin in opensearch-1.3.0-test.yml, [example](manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml). 1. It supports two test configs - `with-security` and `without-security`, which runs test with security plugin enabled and disabled respectively. Choose one or both depending on what your plugin integration tests support. 2. If your plugin is dependent on `job-scheduler` zip, you can define that in `build-dependencies` in the config. Currently, the test workflow only supports `job-scheduler` as build dependency. Please create an issue if your plugin needs more support. -3. For backward compatibility testing, the `test-workflow` runs backward compatibility tests available in the plugin repository, (see [reference]((https://github.com/opensearch-project/anomaly-detection/blob/d9a122d05282f7efc1e24c61d64f18dec0fd47af/build.gradle#L428))). Like integration test, it has a set of configurable options defined in test_manifest.yml, [example](src/test_workflow/config/opensearch/test_manifest.yml#L8). +3. For backward compatibility testing, the `test-workflow` runs backward compatibility tests available in the plugin repository, (see [reference]((https://github.com/opensearch-project/anomaly-detection/blob/d9a122d05282f7efc1e24c61d64f18dec0fd47af/build.gradle#L428))). Like integration test, it has a set of configurable options defined in opensearch-1.3.0-test.yml, [example](manifests/1.3.0/opensearch-1.3.0-test.yml). diff --git a/README.md b/README.md index b19bb4babe..ef47404389 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ See [jenkins](./jenkins) and [docker](./docker) for more information. Tests the OpenSearch distribution, including integration, backwards-compatibility and performance tests. ```bash -./test.sh +./test.sh ``` See [src/test_workflow](./src/test_workflow) for more information. diff --git a/jenkins/test/testsuite/Jenkinsfile b/jenkins/test/testsuite/Jenkinsfile index 59be3728a6..6d917e35c0 100644 --- a/jenkins/test/testsuite/Jenkinsfile +++ b/jenkins/test/testsuite/Jenkinsfile @@ -60,7 +60,7 @@ pipeline { script { basePath = "https://ci.opensearch.org/ci/dbc/bundle-build/${opensearch_version}/${build_id}/${platform}/${architecture}" sh "wget ${basePath}/builds/opensearch/manifest.yml" - sh "./test.sh ${JOB_NAME} ${basePath} --test-run-id ${env.BUILD_NUMBER}" + sh "./test.sh ${JOB_NAME} manifests/${opensearch_version}/opensearch-${opensearch_version}-test.yml ${basePath} --test-run-id ${env.BUILD_NUMBER}" } } post { diff --git a/src/test_workflow/config/opensearch/test_manifest.yml b/manifests/1.3.0/opensearch-1.3.0-test.yml similarity index 100% rename from src/test_workflow/config/opensearch/test_manifest.yml rename to manifests/1.3.0/opensearch-1.3.0-test.yml diff --git a/src/test_workflow/config/opensearch-dashboards/test_manifest.yml b/manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml similarity index 100% rename from src/test_workflow/config/opensearch-dashboards/test_manifest.yml rename to manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml diff --git a/src/ci_workflow/ci_input_manifest.py b/src/ci_workflow/ci_input_manifest.py new file mode 100644 index 0000000000..ba204033b4 --- /dev/null +++ b/src/ci_workflow/ci_input_manifest.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import logging + +from ci_workflow.ci_check_lists import CiCheckLists +from ci_workflow.ci_manifest import CiManifest +from ci_workflow.ci_target import CiTarget +from manifests.input_manifest import InputManifest +from system.temporary_directory import TemporaryDirectory + + +class CiInputManifest(CiManifest): + def __init__(self, file, args): + super().__init__(InputManifest.from_file(file), args) + + def __check__(self): + + target = CiTarget(version=self.manifest.build.version, snapshot=self.args.snapshot) + + with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir: + logging.info(f"Sanity-testing in {work_dir.name}") + + logging.info(f"Sanity testing {self.manifest.build.name}") + + for component in self.manifest.components.select(focus=self.args.component): + logging.info(f"Sanity testing {component.name}") + + ci_check_list = CiCheckLists.from_component(component, target) + ci_check_list.checkout(work_dir.name) + ci_check_list.check() + logging.info("Done.") diff --git a/src/ci_workflow/ci_manifest.py b/src/ci_workflow/ci_manifest.py new file mode 100644 index 0000000000..66e5e1e9e2 --- /dev/null +++ b/src/ci_workflow/ci_manifest.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import abc +import logging + + +class CiManifest(abc.ABC): + def __init__(self, manifest, args): + self.manifest = manifest + self.args = args + + def check(self): + try: + self.__check__() + except: + logging.error("CI Manifest check failed") + raise diff --git a/src/ci_workflow/ci_manifests.py b/src/ci_workflow/ci_manifests.py new file mode 100644 index 0000000000..e7c59ea04e --- /dev/null +++ b/src/ci_workflow/ci_manifests.py @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import re + +from ci_workflow.ci_input_manifest import CiInputManifest +from ci_workflow.ci_test_manifest import CiTestManifest + + +class CiManifests: + def __klass(filename): + if re.search("-test.yml$", filename): + return CiTestManifest + else: + return CiInputManifest + + @classmethod + def from_file(cls, file, args): + return cls.__klass(file.name)(file, args) diff --git a/src/ci_workflow/ci_test_manifest.py b/src/ci_workflow/ci_test_manifest.py new file mode 100644 index 0000000000..146e8f16e0 --- /dev/null +++ b/src/ci_workflow/ci_test_manifest.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import logging + +from ci_workflow.ci_manifest import CiManifest +from manifests.test_manifest import TestManifest + + +class CiTestManifest(CiManifest): + def __init__(self, file, args): + super().__init__(TestManifest.from_file(file), args) + + def __check__(self): + assert self.manifest + logging.info("TestManifest schema validation succeeded") + logging.info("Done.") diff --git a/src/run_ci.py b/src/run_ci.py index 9bdb1c3a2e..0c32971a28 100755 --- a/src/run_ci.py +++ b/src/run_ci.py @@ -6,41 +6,19 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -import logging + import sys from ci_workflow.ci_args import CiArgs -from ci_workflow.ci_check_lists import CiCheckLists -from ci_workflow.ci_target import CiTarget -from manifests.input_manifest import InputManifest +from ci_workflow.ci_manifests import CiManifests from system import console -from system.temporary_directory import TemporaryDirectory def main(): args = CiArgs() console.configure(level=args.logging_level) - manifest = InputManifest.from_file(args.manifest) - - target = CiTarget(version=manifest.build.version, snapshot=args.snapshot) - - with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: - logging.info(f"Sanity-testing in {work_dir.name}") - - logging.info(f"Sanity testing {manifest.build.name}") - - for component in manifest.components.select(focus=args.component): - logging.info(f"Sanity testing {component.name}") - - try: - ci_check_list = CiCheckLists.from_component(component, target) - ci_check_list.checkout(work_dir.name) - ci_check_list.check() - except: - logging.error(f"Error checking {component.name}, retry with: {args.component_command(component.name)}") - raise - logging.info("Done.") + CiManifests.from_file(args.manifest, args).check() if __name__ == "__main__": diff --git a/src/run_integ_test.py b/src/run_integ_test.py index 6fc8db0b2f..77f2c3ce4a 100755 --- a/src/run_integ_test.py +++ b/src/run_integ_test.py @@ -24,8 +24,7 @@ def main(): args = TestArgs() console.configure(level=args.logging_level) - test_manifest_path = os.path.join(os.path.dirname(__file__), "test_workflow", "config", "opensearch", "test_manifest.yml") - test_manifest = TestManifest.from_path(test_manifest_path) + test_manifest = TestManifest.from_path(args.test_manifest_path) bundle_manifest = BundleManifest.from_urlpath("/".join([args.path.rstrip("/"), "dist/opensearch/manifest.yml"])) build_manifest = BuildManifest.from_urlpath("/".join([args.path.rstrip("/"), "builds/opensearch/manifest.yml"])) dependency_installer = DependencyInstallerOpenSearch(args.path, build_manifest, bundle_manifest) diff --git a/src/test_workflow/README.md b/src/test_workflow/README.md index c4a5154cdd..462bd49843 100644 --- a/src/test_workflow/README.md +++ b/src/test_workflow/README.md @@ -26,6 +26,7 @@ The following options are available. | name | description | |----------------------|-------------------------------------------------------------------------| | test-type | Run tests of a test suite. [integ-test, bwc-test, perf-test] | +| test-manifest-path | Specify a test manifest path | | path | Location of manifest(s). | | --test-run-id | Unique identifier for a test run | | --component | Test a specific component in a manifest | @@ -41,21 +42,21 @@ To run integration tests locally, use below command. This pulls down the built b Usage: ```bash -./test.sh integ-test +./test.sh integ-test ``` For example, build locally and run integration tests. ```bash -./build.sh manifests/1.2.0/opensearch-1.2.0.yml +./build.sh manifests/1.3.0/opensearch-1.3.0.yml ./assemble.sh builds/opensearch/manifest.yml -./test.sh integ-test . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml" +./test.sh integ-test manifests/1.3.0/opensearch-1.3.0-test.yml . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml" ``` Or run integration tests against an existing build. ```bash -./test.sh integ-test https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml +./test.sh integ-test manifests/1.3.0/opensearch-1.3.0-test.yml https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml ``` ### Backwards Compatibility Tests @@ -65,7 +66,7 @@ Runs backward compatibility invoking `run_bwc_test.py` in each component from a Usage: ```bash -./test.sh bwc-test +./test.sh bwc-test ``` ### Performance Tests @@ -80,7 +81,7 @@ The CI/CD infrastructure is divided into two main workflows - `build` and `test` Once a new distribution is ready, the `build-job` kicks off the [test-orchestrator-pipeline](../../jenkins_workflow/test/orchestrator/Jenkinsfile) with input parameters `(build_id, architecture, opensearch_version)` that uniquely identify the bundle. The test orchestrator-pipeline generate a unique `test_run_id`, that uniquely identifies the test execution and invokes all three test jobs - `integ-test, bwc-test, perf-test` in parallel. -The [integ-test job](../../jenkins_workflow/test/testsuite/Jenkinsfile) starts by pulling the manifest files and installing the required dependencies for running plugin integration tests. It then kicks off the integration test for each plugin based on the `test-configs` defined in [test-manifest.yml](config/opensearch/test_manifest.yml). It executes each configuration separately from others by spinning up a dedicated local test cluster. It uses `integtest.sh` script to run the integration test. There is a [default](../../scripts/default/integtest.sh) version of this script present in opensearch-build repo and also allows plugins to override the default by having a custom integtest.sh in plugin repo. +The [integ-test job](../../jenkins_workflow/test/testsuite/Jenkinsfile) starts by pulling the manifest files and installing the required dependencies for running plugin integration tests. It then kicks off the integration test for each plugin based on the `test-configs` defined in [opensearch-1.3.0-test.yml](manifests/1.3.0/opensearch-1.3.0-test.yml). It executes each configuration separately from others by spinning up a dedicated local test cluster. It uses `integtest.sh` script to run the integration test. There is a [default](../../scripts/default/integtest.sh) version of this script present in opensearch-build repo and also allows plugins to override the default by having a custom integtest.sh in plugin repo. Once all tests complete, the notifications job can send out the notifications to the subscribed channels. Below figure illustrates how different components of the test workflow would interact with each other. @@ -126,7 +127,7 @@ The development is tracked by [meta issue #126](https://github.com/opensearch-pr Manifest files are configurations for a particular bundle. `test-workflow` uses three types of manifest files to run test suites. -1. `test-manifest.yml` provides a list of test configurations to run against a given component in the bundle. An example of a configuration would be, integration test `index-management` plugin `with-security` and `without-security`. This manifest file serves as a support matrix config for the testing and should be updated by plugins if new components or test suites are to be added as part of the release workflow. See [here](config/opensearch/test_manifest.yml) +1. `test-manifest.yml` provides a list of test configurations to run against a given component in the bundle. An example of a configuration would be, integration test `index-management` plugin `with-security` and `without-security`. This manifest file serves as a support matrix config for the testing and should be updated by plugins if new components or test suites are to be added as part of the release workflow. See [here](manifests/1.3.0/opensearch-1.3.0-test.yml) 2. `build-manifest.yml` created by the build-workflow and provides a list of artifacts built as part of the build-workflow. It assists `test-workflow` pull the maven and build dependencies to run the test suites. 3. `bundle-manfest.yml` created by the build-workflow and provides a list of components packaged in a given bundle. It assists `test-workflow` to identify what components should be tested for a given bundle. diff --git a/src/test_workflow/test_args.py b/src/test_workflow/test_args.py index d993dd481c..ee15981848 100644 --- a/src/test_workflow/test_args.py +++ b/src/test_workflow/test_args.py @@ -23,6 +23,7 @@ class TestArgs: def __init__(self): parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle") + parser.add_argument("test_manifest_path", type=str, help="Specify a test manifest path.") parser.add_argument("path", type=str, help="Location of build and bundle manifests.", default=".") parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test") parser.add_argument("--component", type=str, help="Test a specific component instead of the entire distribution.") @@ -37,5 +38,7 @@ def __init__(self): self.logging_level = args.logging_level self.path = args.path if validators.url(args.path) else os.path.realpath(args.path) + self.test_manifest_path = args.test_manifest_path if validators.url(args.test_manifest_path) else os.path.realpath(args.test_manifest_path) + TestArgs.__test__ = False # type:ignore diff --git a/tests/test_run_ci.py b/tests/test_run_ci.py index ef3faea604..8342d90cf5 100644 --- a/tests/test_run_ci.py +++ b/tests/test_run_ci.py @@ -7,7 +7,7 @@ import os import tempfile import unittest -from unittest.mock import patch +from unittest.mock import call, patch import pytest @@ -30,11 +30,29 @@ def test_usage(self): OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.1.1/opensearch-1.1.1.yml")) @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_MANIFEST]) - @patch("run_ci.TemporaryDirectory") - @patch("run_ci.CiCheckLists.from_component") + @patch("ci_workflow.ci_input_manifest.TemporaryDirectory") + @patch("ci_workflow.ci_input_manifest.CiCheckLists.from_component") def test_main(self, mock_lists, mock_temp, *mocks): mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() main() self.assertNotEqual(mock_lists.call_count, 0) self.assertEqual(mock_lists.return_value.checkout.call_count, mock_lists.call_count) self.assertEqual(mock_lists.return_value.check.call_count, mock_lists.call_count) + + OPENSEARCH_TEST_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.3.0/opensearch-1.3.0-test.yml")) + + @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_TEST_MANIFEST]) + @patch("logging.info") + def test_main_test_manifest(self, mock_logging, *mocks): + main() + mock_logging.assert_has_calls([ + call("TestManifest schema validation succeeded"), + call("Done.") + ]) + + OPENSEARCH_TEST_MANIFEST_NOT_EXIST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.4.0/opensearch-1.3.0-test.yml")) + + @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_TEST_MANIFEST_NOT_EXIST]) + def test_main_test_manifest_empty(self, *mocks): + with self.assertRaises(SystemExit): + main() diff --git a/tests/tests_test_workflow/test_bwc_workflow/bwc_test/test_run_bwc_test.py b/tests/tests_test_workflow/test_bwc_workflow/bwc_test/test_run_bwc_test.py index 4127e69b2d..55816f48e6 100644 --- a/tests/tests_test_workflow/test_bwc_workflow/bwc_test/test_run_bwc_test.py +++ b/tests/tests_test_workflow/test_bwc_workflow/bwc_test/test_run_bwc_test.py @@ -14,8 +14,12 @@ class TestRunBwcTest(unittest.TestCase): @patch( "argparse._sys.argv", - ["run_bwc_test.py", os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote", "dist", "opensearch", "manifest.yml")]) - @patch("run_bwc_test.BwcTestSuite") + [ + "run_bwc_test.py", + os.path.join(os.path.dirname(__file__), "..", "..", "data", "test_manifest.yml"), + os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote", "dist", "opensearch", "manifest.yml") + ]) + @ patch("run_bwc_test.BwcTestSuite") def test_run_bwc_test(self, mock_bwc_suite, *mock): main() self.assertEqual(mock_bwc_suite.return_value.execute.call_count, 1) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_run_integ_test.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_run_integ_test.py index d03ef6bfcf..37f8b00f56 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_run_integ_test.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_run_integ_test.py @@ -12,7 +12,13 @@ class TestRunIntegTest(unittest.TestCase): - @patch("argparse._sys.argv", ["run_integ_test.py", os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote")]) + @patch( + "argparse._sys.argv", + [ + "run_integ_test.py", + os.path.join(os.path.dirname(__file__), "..", "..", "data", "test_manifest.yml"), + os.path.join(os.path.dirname(__file__), "..", "..", "data", "remote") + ]) @patch("run_integ_test.DependencyInstallerOpenSearch") @patch("run_integ_test.TestSuiteResults") @patch("run_integ_test.IntegTestSuiteOpenSearch") diff --git a/tests/tests_test_workflow/test_test_args.py b/tests/tests_test_workflow/test_test_args.py index c217b76ab3..df01103c4a 100644 --- a/tests/tests_test_workflow/test_test_args.py +++ b/tests/tests_test_workflow/test_test_args.py @@ -18,7 +18,11 @@ class TestTestArgs(unittest.TestCase): os.path.dirname(__file__), "data" ) - @patch("argparse._sys.argv", [ARGS_PY, PATH]) + TEST_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_manifest.yml" + ) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, PATH]) def test_defaults(self): test_args = TestArgs() self.assertEqual(test_args.path, os.path.realpath(self.PATH)) @@ -26,15 +30,22 @@ def test_defaults(self): self.assertIsNone(test_args.component) self.assertFalse(test_args.keep) self.assertEqual(test_args.logging_level, logging.INFO) + self.assertEqual(test_args.test_manifest_path, self.TEST_MANIFEST_PATH) - @patch("argparse._sys.argv", [ARGS_PY, PATH, "--test-run-id", "6"]) + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, PATH, "--test-run-id", "6"]) def test_run_id(self): - self.assertEqual(TestArgs().test_run_id, 6) + test_args = TestArgs() + self.assertEqual(test_args.test_run_id, 6) + self.assertEqual(test_args.test_manifest_path, self.TEST_MANIFEST_PATH) - @patch("argparse._sys.argv", [ARGS_PY, PATH, "--verbose"]) + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, PATH, "--verbose"]) def test_verbose(self): - self.assertEqual(TestArgs().logging_level, logging.DEBUG) + test_args = TestArgs() + self.assertEqual(test_args.logging_level, logging.DEBUG) + self.assertEqual(test_args.test_manifest_path, self.TEST_MANIFEST_PATH) - @patch("argparse._sys.argv", [ARGS_PY, 'https://ci.opensearch.org/x/y', "--verbose"]) + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, 'https://ci.opensearch.org/x/y', "--verbose"]) def test_url(self): + test_args = TestArgs() self.assertEqual(TestArgs().path, 'https://ci.opensearch.org/x/y') + self.assertEqual(test_args.test_manifest_path, self.TEST_MANIFEST_PATH)