-
Notifications
You must be signed in to change notification settings - Fork 272
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
BWC test workflow #1603
BWC test workflow #1603
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
--- | ||
name: OpenSearch Dashboards | ||
components: | ||
- name: OpenSearch-Dashboards | ||
bwc-test: | ||
test-configs: | ||
- without-security | ||
- name: functionalTestDashboards | ||
integ-test: | ||
test-configs: | ||
- with-security | ||
- without-security | ||
schema-version: '1.0' | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -76,6 +76,27 @@ Usage: | |
./test.sh bwc-test <test-manifest-path> <target> | ||
``` | ||
|
||
For example, build locally and run BWC tests. | ||
|
||
```bash | ||
./build.sh manifests/1.3.0/opensearch-1.3.0.yml | ||
./assemble.sh builds/opensearch/manifest.yml | ||
./test.sh bwc-test manifests/1.3.0/opensearch-1.3.0-test.yml . # looks for "./builds/opensearch/manifest.yml" and "./dist/opensearch/manifest.yml" | ||
``` | ||
|
||
Or run BWC tests against an existing build. | ||
|
||
```bash | ||
./test.sh bwc-test manifests/1.3.0/opensearch-1.3.0-test.yml --paths opensearch=https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/869/linux/x64 # looks for https://.../builds/opensearch/manifest.yml and https://.../dist/opensearch/manifest.yml | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the great documentation |
||
``` | ||
|
||
To run OpenSearch Dashboards BWC tests. | ||
|
||
```bash | ||
./test.sh bwc-test manifests/1.3.0/opensearch-dashboards-1.3.0-test.yml --paths | ||
opensearch-dashboards=https://ci.opensearch.org/ci/dbc/bundle-build-dashboards/1.2.0/869/linux/x64 | ||
``` | ||
|
||
### Performance Tests | ||
|
||
TODO | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
import abc | ||
import logging | ||
import os | ||
|
||
from system.temporary_directory import TemporaryDirectory | ||
from test_workflow.test_recorder.test_recorder import TestRecorder | ||
from test_workflow.test_result.test_suite_results import TestSuiteResults | ||
|
||
|
||
class BwcTestRunner(abc.ABC): | ||
def __init__(self, args, test_manifest): | ||
self.args = args | ||
self.test_manifest = test_manifest | ||
|
||
self.tests_dir = os.path.join(os.getcwd(), "test-results") | ||
os.makedirs(self.tests_dir, exist_ok=True) | ||
self.test_recorder = TestRecorder(self.args.test_run_id, "bwc-test", self.tests_dir) | ||
|
||
def run(self): | ||
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir: | ||
all_results = TestSuiteResults() | ||
for component in self.components.select(focus=self.args.component): | ||
if component.name in self.test_manifest.components: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not for this PR, but we do a similar workflow in many places and I don't love the double-select here, first pass One option would be to fail if Another, possibly better, could be to do for component, config in self.components.select(focus=self.args.component, configs: self.test_manifest.components, key: "bwc_test"):
test_suite = ...
... There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Created a cleanup issue here: #1604 and assigned myself. |
||
test_config = self.test_manifest.components[component.name] | ||
if test_config.bwc_test: | ||
test_suite = self.__create_test_suite__(component, test_config, work_dir) | ||
test_results = test_suite.execute_tests() | ||
all_results.append(component.name, test_results) | ||
else: | ||
logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not supported") | ||
else: | ||
logging.info(f"Skipping bwc-tests for {component.name}, as it is currently not declared in the test manifest") | ||
|
||
return all_results |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
import logging | ||
import os | ||
|
||
from manifests.test_manifest import TestManifest | ||
from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner | ||
from test_workflow.bwc_test.bwc_test_start_properties_opensearch import BwcTestStartPropertiesOpenSearch | ||
from test_workflow.bwc_test.bwc_test_suite_opensearch import BwcTestSuiteOpenSearch | ||
from test_workflow.test_args import TestArgs | ||
|
||
|
||
class BwcTestRunnerOpenSearch(BwcTestRunner): | ||
|
||
def __init__(self, args: TestArgs, test_manifest: TestManifest): | ||
super().__init__(args, test_manifest) | ||
self.properties = BwcTestStartPropertiesOpenSearch(args.paths.get("opensearch", os.getcwd())) | ||
|
||
self.components = self.properties.build_manifest.components | ||
|
||
logging.info("Entering BWC test for OpenSearch") | ||
|
||
def __create_test_suite__(self, component, test_config, work_dir): | ||
return BwcTestSuiteOpenSearch( | ||
work_dir.name, | ||
component, | ||
test_config, | ||
self.test_recorder, | ||
self.properties.bundle_manifest, | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
import logging | ||
import os | ||
|
||
from manifests.test_manifest import TestManifest | ||
from test_workflow.bwc_test.bwc_test_runner import BwcTestRunner | ||
from test_workflow.bwc_test.bwc_test_start_properties_opensearch_dashboards import BwcTestStartPropertiesOpenSearchDashboards | ||
from test_workflow.bwc_test.bwc_test_suite_opensearch_dashboards import BwcTestSuiteOpenSearchDashboards | ||
from test_workflow.test_args import TestArgs | ||
|
||
|
||
class BwcTestRunnerOpenSearchDashboards(BwcTestRunner): | ||
|
||
def __init__(self, args: TestArgs, test_manifest: TestManifest): | ||
super().__init__(args, test_manifest) | ||
self.properties = BwcTestStartPropertiesOpenSearchDashboards(args.paths.get("opensearch-dashboards", os.getcwd())) | ||
|
||
self.components = self.properties.build_manifest.components | ||
logging.info("Entering BWC test for OpenSearch Dashboards") | ||
|
||
def __create_test_suite__(self, component, test_config, work_dir): | ||
return BwcTestSuiteOpenSearchDashboards( | ||
work_dir.name, | ||
component, | ||
test_config, | ||
self.test_recorder, | ||
self.properties.bundle_manifest, | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
|
||
from manifests.test_manifest import TestManifest | ||
from test_workflow.bwc_test.bwc_test_runner_opensearch import BwcTestRunnerOpenSearch | ||
from test_workflow.bwc_test.bwc_test_runner_opensearch_dashboards import BwcTestRunnerOpenSearchDashboards | ||
from test_workflow.test_args import TestArgs | ||
|
||
|
||
class BwcTestRunners: | ||
RUNNERS = { | ||
"OpenSearch": BwcTestRunnerOpenSearch, | ||
"OpenSearch Dashboards": BwcTestRunnerOpenSearchDashboards | ||
} | ||
|
||
@classmethod | ||
def from_test_manifest(cls, args: TestArgs, test_manifest: TestManifest): | ||
return cls.RUNNERS[test_manifest.name](args, test_manifest) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
|
||
import abc | ||
|
||
from manifests.build_manifest import BuildManifest | ||
from manifests.bundle_manifest import BundleManifest | ||
|
||
|
||
class BwcTestStartProperties(abc.ABC): | ||
def __init__(self, path, build_dir, bundle_dir): | ||
self.path = path | ||
self.build_dir = build_dir | ||
self.bundle_dir = bundle_dir | ||
|
||
self.bundle_manifest = BundleManifest.from_urlpath("/".join([self.path.rstrip("/"), self.bundle_dir])) | ||
self.build_manifest = BuildManifest.from_urlpath("/".join([self.path.rstrip("/"), self.build_dir])) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties | ||
|
||
|
||
class BwcTestStartPropertiesOpenSearch(BwcTestStartProperties): | ||
def __init__(self, path): | ||
super().__init__(path, "builds/opensearch/manifest.yml", "dist/opensearch/manifest.yml") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
|
||
from test_workflow.bwc_test.bwc_test_start_properties import BwcTestStartProperties | ||
|
||
|
||
class BwcTestStartPropertiesOpenSearchDashboards(BwcTestStartProperties): | ||
def __init__(self, path): | ||
super().__init__(path, "builds/opensearch-dashboards/manifest.yml", "dist/opensearch-dashboards/manifest.yml") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit; Can we move this to the top?