diff --git a/package.json b/package.json index 974ef1145..0aab88335 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "merge": "ts-node tools/src/merger/merge.ts", "lint:spec": "ts-node tools/src/linter/lint.ts", "lint": "eslint .", + "lint--fix": "eslint . --fix", "test": "jest" }, "dependencies": { diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index baa2f43e0..8bd45447c 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -3529,10 +3529,10 @@ components: in: path description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. - x-data-type: array + items: + type: string required: true create_pit::query.allow_partial_pit_creation: name: allow_partial_pit_creation diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index 8c808e543..2392f08b5 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -1766,13 +1766,13 @@ components: name: local in: query description: Return local information, do not retrieve the state from cluster-manager node. + deprecated: true + x-version-deprecated: '1.0' + x-deprecation-message: This parameter does not cause this API to act locally. schema: type: boolean default: false description: Return local information, do not retrieve the state from cluster-manager node. - x-version-deprecated: '1.0' - x-deprecation-message: This parameter does not cause this API to act locally. - deprecated: true cat.nodes::query.master_timeout: name: master_timeout in: query @@ -2171,10 +2171,10 @@ components: in: path description: Comma-separated list or wildcard expression of index names to limit the returned information. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list or wildcard expression of index names to limit the returned information. - x-data-type: array + items: + type: string required: true cat.segment_replication::query.active_only: name: active_only diff --git a/spec/namespaces/cluster.yaml b/spec/namespaces/cluster.yaml index 128c114d5..182411fc3 100644 --- a/spec/namespaces/cluster.yaml +++ b/spec/namespaces/cluster.yaml @@ -909,7 +909,6 @@ components: description: Awareness attribute name. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: Awareness attribute name. required: true cluster.get_settings::query.cluster_manager_timeout: @@ -962,7 +961,6 @@ components: description: Awareness attribute name. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: Awareness attribute name. required: true cluster.health::path.index: @@ -1180,7 +1178,6 @@ components: description: Awareness attribute name. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: Awareness attribute name. required: true cluster.put_decommission_awareness::path.awareness_attribute_value: @@ -1189,7 +1186,6 @@ components: description: Awareness attribute value. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: Awareness attribute value. required: true cluster.put_settings::query.cluster_manager_timeout: @@ -1230,7 +1226,6 @@ components: description: Awareness attribute name. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: Awareness attribute name. required: true cluster.reroute::query.cluster_manager_timeout: diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index bcdb78eff..0e10a7a87 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -3966,10 +3966,10 @@ components: in: path description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. - x-data-type: array + items: + type: string required: true indices.get_upgrade::query.allow_no_indices: name: allow_no_indices @@ -4980,10 +4980,10 @@ components: in: path description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. - x-data-type: array + items: + type: string required: true indices.upgrade::query.allow_no_indices: name: allow_no_indices diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index f0ab87112..96623c0df 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -296,7 +296,6 @@ components: description: The id of the model. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: The id of the model. required: true knn.get_model::path.model_id: @@ -305,7 +304,6 @@ components: description: The id of the model. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: The id of the model. required: true knn.search_models::query._source: @@ -649,49 +647,49 @@ components: in: path description: Comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes. - x-data-type: array + items: + type: string required: true knn.stats::path.stat: name: stat in: path description: Comma-separated list of stats to retrieve; use `_all` or empty string to retrieve all stats. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of stats to retrieve; use `_all` or empty string to retrieve all stats. - x-enum-options: - - circuit_breaker_triggered - - total_load_time - - eviction_count - - hit_count - - miss_count - - graph_memory_usage - - graph_memory_usage_percentage - - graph_index_requests - - graph_index_errors - - graph_query_requests - - graph_query_errors - - knn_query_requests - - cache_capacity_reached - - load_success_count - - load_exception_count - - indices_in_cache - - script_compilations - - script_compilation_errors - - script_query_requests - - script_query_errors - - nmslib_initialized - - faiss_initialized - - model_index_status - - indexing_from_model_degraded - - training_requests - - training_errors - - training_memory_usage - - training_memory_usage_percentage - x-data-type: array + items: + type: string + enum: + - circuit_breaker_triggered + - total_load_time + - eviction_count + - hit_count + - miss_count + - graph_memory_usage + - graph_memory_usage_percentage + - graph_index_requests + - graph_index_errors + - graph_query_requests + - graph_query_errors + - knn_query_requests + - cache_capacity_reached + - load_success_count + - load_exception_count + - indices_in_cache + - script_compilations + - script_compilation_errors + - script_query_requests + - script_query_errors + - nmslib_initialized + - faiss_initialized + - model_index_status + - indexing_from_model_degraded + - training_requests + - training_errors + - training_memory_usage + - training_memory_usage_percentage required: true knn.stats::query.timeout: name: timeout @@ -705,7 +703,6 @@ components: description: The id of the model. schema: type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ description: The id of the model. required: true knn.train_model::query.preference: @@ -720,8 +717,8 @@ components: in: path description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of indices; use `_all` or empty string to perform the operation on all indices. - x-data-type: array + items: + type: string required: true diff --git a/spec/namespaces/nodes.yaml b/spec/namespaces/nodes.yaml index f9e2c21ba..7e44be0ea 100644 --- a/spec/namespaces/nodes.yaml +++ b/spec/namespaces/nodes.yaml @@ -496,10 +496,10 @@ components: in: path description: Comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes. schema: - type: string - pattern: ^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$ + type: array description: Comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes. - x-data-type: array + items: + type: string required: true nodes.hot_threads::query.ignore_idle_threads: name: ignore_idle_threads diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 97be69ec5..5a1ad3335 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -1040,7 +1040,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/AllowlistConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' required: true security.create_role: content: @@ -1280,7 +1280,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/AllowlistConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.create_role@200: description: '' content: @@ -1448,7 +1448,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/AllowlistConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.get_audit_configuration@200: description: '' content: @@ -1646,7 +1646,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/AllowlistConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.patch_audit_configuration@200: description: '' content: diff --git a/spec/schemas/_common.aggregations.yaml b/spec/schemas/_common.aggregations.yaml index 71e31a5d0..539ea74ee 100644 --- a/spec/schemas/_common.aggregations.yaml +++ b/spec/schemas/_common.aggregations.yaml @@ -2867,6 +2867,7 @@ components: missing: $ref: '#/components/schemas/Missing' MovingAverageAggregation: + type: object discriminator: propertyName: model oneOf: diff --git a/spec/schemas/_common.analysis.yaml b/spec/schemas/_common.analysis.yaml index db4705168..e17a1abf9 100644 --- a/spec/schemas/_common.analysis.yaml +++ b/spec/schemas/_common.analysis.yaml @@ -7,6 +7,7 @@ paths: {} components: schemas: Analyzer: + type: object discriminator: propertyName: type oneOf: @@ -351,6 +352,7 @@ components: - type: string - $ref: '#/components/schemas/CharFilterDefinition' CharFilterDefinition: + type: object discriminator: propertyName: type oneOf: @@ -447,6 +449,7 @@ components: - type: string - $ref: '#/components/schemas/TokenFilterDefinition' TokenFilterDefinition: + type: object discriminator: propertyName: type oneOf: @@ -1473,6 +1476,7 @@ components: required: - type Normalizer: + type: object discriminator: propertyName: type oneOf: @@ -1509,6 +1513,7 @@ components: - type: string - $ref: '#/components/schemas/TokenizerDefinition' TokenizerDefinition: + type: object discriminator: propertyName: type oneOf: diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 8fcf4f4ea..1fb513d24 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -157,6 +157,7 @@ components: unmatch: type: string Property: + type: object discriminator: propertyName: type oneOf: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 952b6326f..375e47802 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -20,7 +20,6 @@ components: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value. - x-data-type: time pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$ type: string Metadata: diff --git a/spec/schemas/_core.rank_eval.yaml b/spec/schemas/_core.rank_eval.yaml index 4cf89360a..b4b3eba9c 100644 --- a/spec/schemas/_core.rank_eval.yaml +++ b/spec/schemas/_core.rank_eval.yaml @@ -100,9 +100,7 @@ components: - type: object properties: normalize: - externalDocs: - url: https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG - description: If set to true, this metric will calculate the Normalized DCG. + description: If set to true, this metric will calculate the Normalized DCG (https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG). type: boolean RankEvalMetricExpectedReciprocalRank: allOf: diff --git a/spec/schemas/cluster.remote_info.yaml b/spec/schemas/cluster.remote_info.yaml index 21b63fe34..71af6ad6d 100644 --- a/spec/schemas/cluster.remote_info.yaml +++ b/spec/schemas/cluster.remote_info.yaml @@ -7,6 +7,7 @@ paths: {} components: schemas: ClusterRemoteInfo: + type: object discriminator: propertyName: mode oneOf: diff --git a/spec/schemas/rollups._common.yaml b/spec/schemas/rollups._common.yaml index e847affe4..e98e3dadd 100644 --- a/spec/schemas/rollups._common.yaml +++ b/spec/schemas/rollups._common.yaml @@ -80,8 +80,8 @@ components: $ref: '#/components/schemas/DateHistogramDimension' terms: $ref: '#/components/schemas/TermsDimension' - minproperties: 1 - maxproperties: 1 + minProperties: 1 + maxProperties: 1 DateHistogramDimension: type: object properties: @@ -135,8 +135,8 @@ components: type: object value_count: type: object - minproperties: 1 - maxproperties: 1 + minProperties: 1 + maxProperties: 1 Cron: type: object properties: @@ -173,12 +173,12 @@ components: type: number next_window_end_time: type: number - status: - type: string - failure_reason: - type: string - stats: - $ref: '#/components/schemas/Stats' + status: + type: string + failure_reason: + type: string + stats: + $ref: '#/components/schemas/Stats' Stats: type: object properties: diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index c7452bb2f..2fcce2ec2 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -57,17 +57,20 @@ components: additionalProperties: $ref: '#/components/schemas/ActionGroup' - AllowlistConfig: + AllowListConfig: type: object properties: config: + $ref: '#/components/schemas/AllowConfig' + + AllowConfig: + type: object + properties: + enabled: + type: boolean + requests: type: object - items: - enabled: - type: boolean - requests: - type: object - description: An object with APIs as key and array of http methods as values. + description: An object with APIs as key and array of http methods as values. AuditConfig: type: object @@ -172,7 +175,8 @@ components: properties: status: type: string - value: 400 + enum: + - 400 message: type: string description: Message returned as part of BAD_REQUEST response. @@ -396,7 +400,8 @@ components: properties: status: type: string - value: 501 + enum: + - 501 message: type: string description: Message returned as part of NOT_IMPLEMENTED response. @@ -422,22 +427,22 @@ components: description: type: string description: Contains the description supplied by the user to describe the token. - required: true service: type: string description: A name of the service if generating a token for that service. - required: false duration: type: string description: Value in seconds. - required: optional + required: + - description Ok: type: object properties: status: type: string - value: 200 + enum: + - 200 message: type: string description: Message returned as part of OK response. @@ -613,7 +618,8 @@ components: properties: status: type: string - value: 403 + enum: + - 403 message: type: string description: Message returned as part of FORBIDDEN response. diff --git a/tools/src/Logger.ts b/tools/src/Logger.ts new file mode 100644 index 000000000..165deda19 --- /dev/null +++ b/tools/src/Logger.ts @@ -0,0 +1,33 @@ +export enum LogLevel { + error = 1, + warn = 2, + info = 3 +} + +export class Logger { + level: LogLevel + + constructor (level: LogLevel) { + this.level = level + } + + info (message: string): void { + this.#log(LogLevel.info, message) + } + + warn (message: string): void { + this.#log(LogLevel.warn, message) + } + + error (message: string): void { + this.#log(LogLevel.error, message) + } + + #log (level: LogLevel, message: string): void { + if (level > this.level) return + const output = `[${level}] ${message}` + if (level === LogLevel.error) console.error(output) + if (level === LogLevel.warn) console.warn(output) + if (level === LogLevel.info) console.info(output) + } +} diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts new file mode 100644 index 000000000..21ca3630a --- /dev/null +++ b/tools/src/linter/SchemasValidator.ts @@ -0,0 +1,113 @@ +import AJV from 'ajv' +import addFormats from 'ajv-formats' +import OpenApiMerger from '../merger/OpenApiMerger' +import { type ValidationError } from '../types' +import { LogLevel } from '../Logger' + +const IGNORED_ERROR_PREFIXES = [ + 'can\'t resolve reference', // errors in referenced schemas will also cause reference errors + 'discriminator: oneOf subschemas' // known bug in ajv: https://github.com/ajv-validator/ajv/issues/2281 +] + +const ADDITIONAL_KEYWORDS = [ + 'x-version-added', + 'x-version-deprecated', + 'x-version-removed', + 'x-deprecation-message' +] + +export default class SchemasValidator { + root_folder: string + spec: Record = {} + ajv: AJV + + constructor (root_folder: string) { + this.root_folder = root_folder + this.ajv = new AJV({ strict: true, discriminator: true }) + addFormats(this.ajv) + for (const keyword of ADDITIONAL_KEYWORDS) this.ajv.addKeyword(keyword) + } + + validate (): ValidationError[] { + this.spec = new OpenApiMerger(this.root_folder, LogLevel.error).merge().components as Record + const named_schemas_errors = this.validate_named_schemas() + if (named_schemas_errors.length > 0) return named_schemas_errors + return [ + ...this.validate_parameter_schemas(), + ...this.validate_request_body_schemas(), + ...this.validate_response_schemas() + ] + } + + validate_named_schemas (): ValidationError[] { + return Object.entries(this.spec.schemas as Record).map(([key, _schema]) => { + const schema = _schema as Record + const error = this.validate_schema(schema, `#/components/schemas/${key}`) + if (error == null) return + + const file = `schemas/${key.split(':')[0]}.yaml` + const location = `#/components/schemas/${key.split(':')[1]}` + return this.error(file, location, error) + }).filter((error) => error != null) as ValidationError[] + } + + validate_parameter_schemas (): ValidationError[] { + return Object.entries(this.spec.parameters as Record).map(([key, param]) => { + const error = this.validate_schema(param.schema as Record) + if (error == null) return + + const namespace = this.group_to_namespace(key.split('::')[0]) + const file = namespace === '_global' ? '_global_parameters.yaml' : `namespaces/${namespace}.yaml` + const location = namespace === '_global' ? param.name as string : `#/components/parameters/${key}` + return this.error(file, location, error) + }).filter((error) => error != null) as ValidationError[] + } + + validate_request_body_schemas (): ValidationError[] { + return Object.entries(this.spec.requestBodies as Record).flatMap(([namespace, body]) => { + const file = `namespaces/${namespace}.yaml` + const location = `#/components/requestBodies/${namespace}` + return this.validate_content_schemas(file, location, body.content as Record) + }) + } + + validate_response_schemas (): ValidationError[] { + return Object.entries(this.spec.responses as Record).flatMap(([key, response]) => { + const namespace = this.group_to_namespace(key.split('@')[0]) + const file = `namespaces/${namespace}.yaml` + const location = `#/components/responses/${key}` + const content = response.content as Record + return this.validate_content_schemas(file, location, content) + }) + } + + validate_content_schemas (file: string, location: string, content: Record | undefined): ValidationError[] { + return Object.entries(content ?? {}).map(([media_type, value]) => { + const schema = value.schema as Record + const error = this.validate_schema(schema) + if (error != null) return this.error(file, `${location}/content/${media_type}`, error) + }).filter(e => e != null) as ValidationError[] + } + + validate_schema (schema: Record, key: string | undefined = undefined): Error | undefined { + if (schema == null || schema.$ref != null) return + try { + if (key != null) this.ajv.addSchema(schema, key) + this.ajv.compile(schema) + } catch (_e: any) { + const error = _e as Error + for (const prefix of IGNORED_ERROR_PREFIXES) if (error.message.startsWith(prefix)) return + return error + } + } + + group_to_namespace (group: string): string { + if (group === '_global') return '_global' + const [, namespace] = group.split('.').reverse() + return namespace ?? '_core' + } + + error (file: string, location: string, error: Error): ValidationError { + return { file, location, message: error.message } + } +} diff --git a/tools/src/linter/SpecValidator.ts b/tools/src/linter/SpecValidator.ts index 12da4175c..51ccf2a27 100644 --- a/tools/src/linter/SpecValidator.ts +++ b/tools/src/linter/SpecValidator.ts @@ -5,12 +5,14 @@ import SchemaRefsValidator from './SchemaRefsValidator' import SupersededOperationsFile from './components/SupersededOperationsFile' import InfoFile from './components/InfoFile' import InlineObjectSchemaValidator from './InlineObjectSchemaValidator' +import SchemasValidator from './SchemasValidator' export default class SpecValidator { superseded_ops_file: SupersededOperationsFile info_file: InfoFile namespaces_folder: NamespacesFolder schemas_folder: SchemasFolder + schemas_validator: SchemasValidator schema_refs_validator: SchemaRefsValidator inline_object_schema_validator: InlineObjectSchemaValidator @@ -19,6 +21,7 @@ export default class SpecValidator { this.info_file = new InfoFile(`${root_folder}/_info.yaml`) this.namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`) this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`) + this.schemas_validator = new SchemasValidator(root_folder) this.schema_refs_validator = new SchemaRefsValidator(this.namespaces_folder, this.schemas_folder) this.inline_object_schema_validator = new InlineObjectSchemaValidator(this.namespaces_folder, this.schemas_folder) } @@ -34,7 +37,8 @@ export default class SpecValidator { ...this.schema_refs_validator.validate(), ...this.superseded_ops_file.validate(), ...this.info_file.validate(), - ...this.inline_object_schema_validator.validate() + ...this.inline_object_schema_validator.validate(), + ...this.schemas_validator.validate() ] } } diff --git a/tools/src/merger/OpenApiMerger.ts b/tools/src/merger/OpenApiMerger.ts index c2d8bec6f..a541293d7 100644 --- a/tools/src/merger/OpenApiMerger.ts +++ b/tools/src/merger/OpenApiMerger.ts @@ -4,16 +4,19 @@ import _ from 'lodash' import { read_yaml, write_yaml } from '../../helpers' import SupersededOpsGenerator from './SupersededOpsGenerator' import GlobalParamsGenerator from './GlobalParamsGenerator' +import { Logger, LogLevel } from '../Logger' // Create a single-file OpenAPI spec from multiple files for OpenAPI validation and programmatic consumption export default class OpenApiMerger { root_folder: string spec: Record + logger: Logger paths: Record> = {} // namespace -> path -> path_item_object schemas: Record> = {} // category -> schema -> schema_object - constructor (root_folder: string) { + constructor (root_folder: string, log_level: LogLevel = LogLevel.warn) { + this.logger = new Logger(log_level) this.root_folder = fs.realpathSync(root_folder) this.spec = { openapi: '3.1.0', @@ -116,7 +119,7 @@ export default class OpenApiMerger { // Generate superseded operations from _superseded_operations.yaml file. #generate_superseded_ops (): void { - const gen = new SupersededOpsGenerator(this.root_folder) + const gen = new SupersededOpsGenerator(this.root_folder, this.logger) gen.generate(this.spec) } } diff --git a/tools/src/merger/SupersededOpsGenerator.ts b/tools/src/merger/SupersededOpsGenerator.ts index b673aa3d1..c8bffd3c1 100644 --- a/tools/src/merger/SupersededOpsGenerator.ts +++ b/tools/src/merger/SupersededOpsGenerator.ts @@ -1,11 +1,14 @@ import { type OperationSpec, type SupersededOperationMap } from 'types' import _ from 'lodash' import { read_yaml } from '../../helpers' +import { Logger, LogLevel } from '../Logger' export default class SupersededOpsGenerator { + logger: Logger superseded_ops: SupersededOperationMap - constructor (root_path: string) { + constructor (root_path: string, logger: Logger | undefined) { + this.logger = logger ?? new Logger(LogLevel.warn) const file_path = root_path + '/_superseded_operations.yaml' this.superseded_ops = read_yaml(file_path) as SupersededOperationMap delete this.superseded_ops.$schema @@ -17,8 +20,8 @@ export default class SupersededOpsGenerator { const operation_keys = operations.map(op => op.toLowerCase()) const superseded_path = this.copy_params(superseded_by, path) const path_entry = _.entries(spec.paths as Document).find(([path, _]) => regex.test(path)) - if (!path_entry) console.log(`Path not found: ${superseded_by}`) - else spec.paths[superseded_path] = this.path_object(path_entry[1], operation_keys) + if (path_entry != null) spec.paths[superseded_path] = this.path_object(path_entry[1], operation_keys) + else this.logger.warn(`Path not found: ${superseded_by}`) } } diff --git a/tools/src/merger/merge.ts b/tools/src/merger/merge.ts index c38f29c97..a8b33526a 100644 --- a/tools/src/merger/merge.ts +++ b/tools/src/merger/merge.ts @@ -5,7 +5,7 @@ import { resolve } from 'path' const command = new Command() .description('Merges the multi-file OpenSearch spec into a single file for programmatic use.') .addOption(new Option('-s, --source ', 'path to the root folder of the multi-file spec').default(resolve(__dirname, '../../../spec'))) - .addOption(new Option('-o, --output ', 'output file name').default(resolve(__dirname, '../../../build/opensearch-openapi.yaml'))) + .addOption(new Option('-o, --output ', 'output file name').default(resolve(__dirname, '../../opensearch-openapi.yaml'))) .allowExcessArguments(false) .parse() diff --git a/tools/tests/linter/SchemasValidator.test.ts b/tools/tests/linter/SchemasValidator.test.ts new file mode 100644 index 000000000..cbd16a998 --- /dev/null +++ b/tools/tests/linter/SchemasValidator.test.ts @@ -0,0 +1,43 @@ +import SchemasValidator from '../../src/linter/SchemasValidator' + +test('validate() - named_schemas', () => { + const validator = new SchemasValidator('./tools/tests/linter/fixtures/schemas_validator/named_schemas') + expect(validator.validate()).toEqual([ + { + file: 'schemas/actions.yaml', + location: '#/components/schemas/Bark', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'schemas/animals.yaml', + location: '#/components/schemas/Dog', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + } + ]) +}) + +test('validate() - anonymous_schemas', () => { + const validator = new SchemasValidator('./tools/tests/linter/fixtures/schemas_validator/anonymous_schemas') + expect(validator.validate()).toEqual([ + { + file: '_global_parameters.yaml', + location: 'human', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/_core.yaml', + location: '#/components/parameters/adopt::path.docket', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/adopt.yaml', + location: '#/components/requestBodies/adopt/content/application/json', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/_core.yaml', + location: '#/components/responses/adopt@200/content/application/json', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + } + ]) +}) diff --git a/tools/tests/linter/SpecValidator.test.ts b/tools/tests/linter/SpecValidator.test.ts index 0ba8cb576..7658ee860 100644 --- a/tools/tests/linter/SpecValidator.test.ts +++ b/tools/tests/linter/SpecValidator.test.ts @@ -2,11 +2,11 @@ import SpecValidator from 'linter/SpecValidator' test('validate()', () => { const validator = new SpecValidator('./tools/tests/linter/fixtures/empty') - expect(validator.validate()).toEqual([]) - validator.namespaces_folder.validate = jest.fn().mockReturnValue([{ file: 'namespaces/', message: 'namespace error' }]) validator.schemas_folder.validate = jest.fn().mockReturnValue([{ file: 'schemas/', message: 'schema error' }]) validator.schema_refs_validator.validate = jest.fn().mockReturnValue([{ file: 'schema_refs', message: 'schema refs error' }]) + validator.schemas_validator.validate = jest.fn().mockReturnValue([{ file: 'schemas/', message: 'schema error' }]) + validator.inline_object_schema_validator.validate = jest.fn().mockReturnValue([{ file: 'inline_file', message: 'inline_object_schema_validator error' }]) expect(validator.validate()).toEqual([ { file: 'namespaces/', message: 'namespace error' }, @@ -17,6 +17,8 @@ test('validate()', () => { validator.schemas_folder.validate = jest.fn().mockReturnValue([]) expect(validator.validate()).toEqual([ - { file: 'schema_refs', message: 'schema refs error' } + { file: 'schema_refs', message: 'schema refs error' }, + { file: 'inline_file', message: 'inline_object_schema_validator error' }, + { file: 'schemas/', message: 'schema error' } ]) }) diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml new file mode 100644 index 000000000..479cfaaa4 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: + title: '' + version: '' +components: + parameters: + human: + name: human + in: query + description: Whether to return human readable values for statistics. + schema: + type: bogus + default: true \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml new file mode 100644 index 000000000..9eb26cb84 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml @@ -0,0 +1,54 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + '/adopt/{animal}/dockets/{docket}': + get: + operationId: adopt.0 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + responses: + '200': + $ref: '#/components/responses/adopt@200' + post: + operationId: adopt.1 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + requestBody: + $ref: '#/components/requestBodies/adopt' + responses: + '200': + $ref: '#/components/responses/adopt@200' +components: + requestBodies: + adopt: { + content: { + application/json: { + schema: { + type: object2 + } + } + } + } + parameters: + adopt::path.animal: + name: animal + in: path + schema: + $ref: '../schemas/animals.yaml#/components/schemas/Animal' + adopt::path.docket: + name: docket + in: path + schema: + type: number2 + responses: + adopt@200: + description: '' + content: + application/json: + schema: + type: object2 \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml new file mode 100644 index 000000000..89bec0877 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml @@ -0,0 +1,11 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Bark: + type: string + Meow: + type: string \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml new file mode 100644 index 000000000..bdb82a16f --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml @@ -0,0 +1,21 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Animal: + oneOf: + - $ref: '#/components/schemas/Dog' + - $ref: '#/components/schemas/Cat' + Dog: + type: object + properties: + bark: + $ref: 'actions.yaml#/components/schemas/Bark' + Cat: + type: object + properties: + meow: + $ref: 'actions.yaml#/components/schemas/Meow' \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml new file mode 100644 index 000000000..479cfaaa4 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: + title: '' + version: '' +components: + parameters: + human: + name: human + in: query + description: Whether to return human readable values for statistics. + schema: + type: bogus + default: true \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml new file mode 100644 index 000000000..0f1a2970b --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml @@ -0,0 +1,45 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + '/adopt/{animal}/dockets/{docket}': + get: + operationId: adopt.0 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + responses: + '200': + $ref: '#/components/responses/adopt@200' + post: + operationId: adopt.1 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + requestBody: + $ref: '#/components/requestBodies/adopt' + responses: + '200': + $ref: '#/components/responses/adopt@200' +components: + requestBodies: + adopt: {} + parameters: + adopt::path.animal: + name: animal + in: path + schema: + $ref: '../schemas/animals.yaml#/components/schemas/Animal' + adopt::path.docket: + name: docket + in: path + schema: + type: number + responses: + adopt@200: + description: '' + application/json: + schema: + type: object \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml new file mode 100644 index 000000000..24dbbb8d6 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml @@ -0,0 +1,11 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Bark: + type: bogus + Meow: + type: string \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml new file mode 100644 index 000000000..6bf0f10f5 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml @@ -0,0 +1,21 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Animal: + oneOf: + - $ref: '#/components/schemas/Dog' + - $ref: '#/components/schemas/Cat' + Dog: + type: bogus + properties: + bark: + $ref: 'actions.yaml#/components/schemas/Bark' + Cat: + type: object + properties: + meow: + $ref: 'actions.yaml#/components/schemas/Meow' \ No newline at end of file diff --git a/tools/tests/merger/OpenApiMerger.test.ts b/tools/tests/merger/OpenApiMerger.test.ts index f261aef0b..33146327f 100644 --- a/tools/tests/merger/OpenApiMerger.test.ts +++ b/tools/tests/merger/OpenApiMerger.test.ts @@ -1,8 +1,9 @@ import OpenApiMerger from 'merger/OpenApiMerger' import fs from 'fs' +import { LogLevel } from '../../src/Logger' test('merge()', async () => { - const merger = new OpenApiMerger('./tools/tests/merger/fixtures/spec/') + const merger = new OpenApiMerger('./tools/tests/merger/fixtures/spec/', LogLevel.error) merger.merge('./tools/tests/merger/opensearch-openapi.yaml') expect(fs.readFileSync('./tools/tests/merger/fixtures/expected.yaml', 'utf8')) .toEqual(fs.readFileSync('./tools/tests/merger/opensearch-openapi.yaml', 'utf8'))