diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 61f6c2828..1328cc863 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -195,11 +195,11 @@ You can also reuse output in payload expectations. See [tests/plugins/index_stat ### Managing Versions -It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. +It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver or a semver range in your test stories or chapters as follows. ```yaml -- synopsis: Search with `phase_took` added in OpenSearch 2.12. - version: '>= 2.12' +- synopsis: Search with `phase_took` added in OpenSearch 2.12 and removed in version 3. + version: '>=2.12 <3' path: /{index}/_search parameters: index: movies diff --git a/tests/default/ingest/pipeline/text_embedding.yaml b/tests/default/ingest/pipeline/text_embedding.yaml index 6ba46e8e9..fe70aee21 100644 --- a/tests/default/ingest/pipeline/text_embedding.yaml +++ b/tests/default/ingest/pipeline/text_embedding.yaml @@ -5,7 +5,7 @@ epilogues: - path: /_ingest/pipeline/books_pipeline method: DELETE status: [200, 404] -version: '>= 2.11, < 3.0' # TODO: re-enable using a 3.0 build with the neural-search plugin +version: '>= 2.11' chapters: - synopsis: Create ingest pipeline for text embedding. path: /_ingest/pipeline/{id} diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml index 5bae694be..3f51b38c3 100644 --- a/tests/default/security/api/securityconfig/config.yaml +++ b/tests/default/security/api/securityconfig/config.yaml @@ -1,7 +1,7 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test securityconfig/config endpoint. -version: '>2.9' +version: '> 2.9' # ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. chapters: diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts new file mode 100644 index 000000000..52de7e899 --- /dev/null +++ b/tools/src/_utils/semver.ts @@ -0,0 +1,20 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from 'semver' + +export function coerce(version: string) : string { + return semver.coerce(version)?.toString() ?? version +} + +export function satisfies(version: string | semver.SemVer | undefined, range: string): boolean { + if (version === undefined || version === '') return true + if (semver.validRange(range) == null) throw `Invalid semver ${range}.` + return semver.satisfies(version, range) +} diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 5d4164b1c..3f2fb396b 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -11,7 +11,7 @@ import _, { extend, isEmpty } from 'lodash' import { delete_matching_keys, find_refs, write_yaml } from '../helpers' import { Logger } from '../Logger' import { type OpenAPIV3 } from 'openapi-types' -import semver from 'semver' +import * as semver from '../_utils/semver' // Extract a versioned API export default class OpenApiVersionExtractor { @@ -22,7 +22,7 @@ export default class OpenApiVersionExtractor { constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { this._source_spec = source_spec - this._target_version = semver.coerce(target_version)?.toString() ?? target_version + this._target_version = semver.coerce(target_version) this._logger = logger this._spec = undefined } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index fede2e397..9020968fa 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -14,7 +14,7 @@ import { overall_result } from './helpers' import { StoryOutputs } from './StoryOutputs' import SupplementalChapterEvaluator from './SupplementalChapterEvaluator' import { ChapterOutput } from './ChapterOutput' -import * as semver from 'semver' +import * as semver from '../_utils/semver' import _ from 'lodash' export default class StoryEvaluator { diff --git a/tools/tests/_utils/semver.test.ts b/tools/tests/_utils/semver.test.ts new file mode 100644 index 000000000..2454549b7 --- /dev/null +++ b/tools/tests/_utils/semver.test.ts @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from "../../src/_utils/semver"; + +describe('coerce', () => { + it ('null', () => { + expect(semver.coerce('')).toEqual('') + expect(semver.coerce('1.2.3')).toEqual('1.2.3') + expect(semver.coerce('1.2')).toEqual('1.2.0') + expect(semver.coerce('1')).toEqual('1.0.0') + }) +}); + +describe('satisfies', () => { + it ('defaults', () => { + expect(semver.satisfies('', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('semver', () => { + expect(semver.satisfies(semver.coerce('2.17.0'), '>= 1.3 < 99.0')).toBe(true) + }) + + it ('~', () => { + expect(semver.satisfies('2.17.0', '~> 2.x')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 1.x')).toBe(false) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.18')).toBe(false) + }) + + it ('> <', () => { + expect(semver.satisfies('2.17.0', '> 2.999.0')).toBe(false) + expect(semver.satisfies('2.17.0', '< 3.0')).toBe(true) + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('ranges', () => { + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('invalid', () => { + expect(() => { semver.satisfies('1.2.3', '>= 1, < 2') }).toThrow('Invalid semver >= 1, < 2.') + }) +}); diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index 12c1b2c32..abe502379 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -176,6 +176,25 @@ chapters: overall: result: SKIPPED message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. + - title: This GET /_cat/health should run (>= 1.3, < 99.0). + overall: + result: PASSED + path: GET /_cat/health + request: + parameters: + format: + result: PASSED + request: + result: PASSED + response: + status: + result: PASSED + payload_body: + result: PASSED + payload_schema: + result: PASSED + output_values: + result: SKIPPED epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml index b4b08b963..a882666b3 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/passed.yaml @@ -89,3 +89,9 @@ chapters: path: /_cat/health parameters: format: json + - synopsis: This GET /_cat/health should run (>= 1.3, < 99.0). + version: '>= 1.3 < 99.0' + method: GET + path: /_cat/health + parameters: + format: json