Merge pull request #229 from postmanlabs/feature/validate-metadata

Validate metadata with option `validateMetadata`.

Author: VShingala

assets/json-schema-faker.js

@@ -10,6 +10,8 @@
  * Date: 2018-04-09 17:23:23.954Z
  */
 
+var validateSchema = require('../lib/ajvValidation').validateSchema;
+
 (function (global, factory) {
   typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :
   typeof define === 'function' && define.amd ? define(factory) :
@@ -24547,7 +24549,12 @@ function extend() {
           return;
       }
       if (optionAPI('useExamplesValue') && 'example' in schema) {
-        return schema.example;
+        var result = validateSchema(schema, schema.example);
+
+        // Use example only if valid 
+        if (result && result.length === 0) {
+          return schema.example;
+        }
       }
       if (optionAPI('useDefaultValue') && 'default' in schema) {
           return schema.default;

lib/ajvValidation.js

@@ -0,0 +1,89 @@
+var _ = require('lodash'),
+  Ajv = require('ajv');
+
+// Following keyword are supoorted for Ajv but not by OAS
+const IGNORED_KEYWORDS = ['propertyNames', 'const', 'additionalItems', 'dependencies'];
+
+/**
+ * Checks if value is postman variable or not
+ *
+ * @param {*} value - Value to check for
+ * @returns {Boolean} postman variable or not
+ */
+function isPmVariable (value) {
+  // collection/environment variables are in format - {{var}}
+  return _.isString(value) && _.startsWith(value, '{{') && _.endsWith(value, '}}');
+}
+
+/**
+ * Used to validate schema against a value.
+ * NOTE: Used in assets/json-schema-faker.js to validate schema example
+ *
+ * @param {*} schema - schema to validate
+ * @param {*} valueToUse - value to validate schema against
+ * @param {*} options - a standard list of options that's globally passed around. Check options.js for more.
+ * @returns {*} - Found Validation Errors
+ */
+function validateSchema (schema, valueToUse, options = {}) {
+  var ajv,
+    validate,
+    filteredValidationError;
+
+  try {
+    // add Ajv options to support validation of OpenAPI schema.
+    // For more details see https://ajv.js.org/#options
+    ajv = new Ajv({
+      // the unknown formats are ones that are allowed in OAS, but not JSON schema.
+      unknownFormats: ['int32', 'int64'],
+
+      // check all rules collecting all errors. instead returning after the first error.
+      allErrors: true,
+
+      // supports keyword "nullable" from Open API 3 specification.
+      nullable: true
+    });
+    validate = ajv.compile(schema);
+    validate(valueToUse);
+  }
+  catch (e) {
+    // something went wrong validating the schema
+    // input was invalid. Don't throw mismatch
+    return filteredValidationError;
+  }
+
+  // Filter validation errors for following cases
+  filteredValidationError = _.filter(_.get(validate, 'errors', []), (validationError) => {
+    let dataPath = _.get(validationError, 'dataPath', '');
+
+    // discard the leading '.' if it exists
+    if (dataPath[0] === '.') {
+      dataPath = dataPath.slice(1);
+    }
+
+    // for invalid `propertyNames` two error are thrown by Ajv, which include error with `pattern` keyword
+    if (validationError.keyword === 'pattern') {
+      return !_.has(validationError, 'propertyName');
+    }
+
+    // As OAS only supports some of Json Schema keywords, and Ajv is supporting all keywords from Draft 7
+    // Remove keyword currently not supported in OAS to make both compatible with each other
+    else if (_.includes(IGNORED_KEYWORDS, validationError.keyword)) {
+      return false;
+    }
+
+    // Ignore unresolved variables from mismatch if option is set
+    else if (options.ignoreUnresolvedVariables && isPmVariable(_.get(valueToUse, dataPath))) {
+      return false;
+    }
+    return true;
+  });
+
+  // sort errors based on dataPath, as this will ensure no overriding later
+  filteredValidationError = _.sortBy(filteredValidationError, ['dataPath']);
+
+  return filteredValidationError;
+}
+
+module.exports = {
+  validateSchema
+};

lib/deref.js

@@ -55,12 +55,14 @@ module.exports = {
    * @param {*} components components in openapi spec.
    * @param {object} schemaResolutionCache stores already resolved references
    * @param {*} resolveFor - resolve refs for validation/conversion (value to be one of VALIDATION/CONVERSION)
+   * @param {string} resolveTo The desired JSON-generation mechanism (schema: prefer using the JSONschema to
+    generate a fake object, example: use specified examples as-is). Default: schema
    * @param {*} stack counter which keeps a tab on nested schemas
    * @param {*} seenRef References that are repeated. Used to identify circular references.
    * @returns {*} schema - schema that adheres to all individual schemas in schemaArr
    */
   resolveAllOf: function (schemaArr, parameterSourceOption, components, schemaResolutionCache,
-    resolveFor = 'CONVERSION', stack = 0, seenRef) {
+    resolveFor = 'CONVERSION', resolveTo = 'schema', stack = 0, seenRef) {
 
     if (!(schemaArr instanceof Array)) {
       return null;
@@ -69,13 +71,13 @@ module.exports = {
     if (schemaArr.length === 1) {
       // for just one entry in allOf, don't need to enforce type: object restriction
       return this.resolveRefs(schemaArr[0], parameterSourceOption, components, schemaResolutionCache, resolveFor,
-        stack, seenRef);
+        resolveTo, stack, seenRef);
     }
 
     // generate one object for each schema
     let indivObjects = schemaArr.map((schema) => {
         return this.resolveRefs(schema, parameterSourceOption, components, schemaResolutionCache, resolveFor,
-          stack, seenRef);
+          resolveTo, stack, seenRef);
       }).filter((schema) => {
         return schema.type === 'object';
       }),
@@ -113,13 +115,15 @@ module.exports = {
    * @param {*} components components in openapi spec.
    * @param {object} schemaResolutionCache stores already resolved references
    * @param {*} resolveFor - resolve refs for validation/conversion (value to be one of VALIDATION/CONVERSION)
+   * @param {string} resolveTo The desired JSON-generation mechanism (schema: prefer using the JSONschema to
+    generate a fake object, example: use specified examples as-is). Default: schema
    * @param {*} stack counter which keeps a tab on nested schemas
    * @param {*} seenRef - References that are repeated. Used to identify circular references.
    * @returns {*} schema satisfying JSON-schema-faker.
    */
 
   resolveRefs: function (schema, parameterSourceOption, components, schemaResolutionCache,
-    resolveFor = 'CONVERSION', stack = 0, seenRef = {}) {
+    resolveFor = 'CONVERSION', resolveTo = 'schema', stack = 0, seenRef = {}) {
     var resolvedSchema, prop, splitRef,
       ERR_TOO_MANY_LEVELS = '<Error: Too many levels of nesting to fake this schema>';
 
@@ -136,15 +140,15 @@ module.exports = {
 
     if (schema.anyOf) {
       return this.resolveRefs(schema.anyOf[0], parameterSourceOption, components, schemaResolutionCache, resolveFor,
-        stack, _.cloneDeep(seenRef));
+        resolveTo, stack, _.cloneDeep(seenRef));
     }
     if (schema.oneOf) {
       return this.resolveRefs(schema.oneOf[0], parameterSourceOption, components, schemaResolutionCache, resolveFor,
-        stack, _.cloneDeep(seenRef));
+        resolveTo, stack, _.cloneDeep(seenRef));
     }
     if (schema.allOf) {
       return this.resolveAllOf(schema.allOf, parameterSourceOption, components, schemaResolutionCache, resolveFor,
-        stack, _.cloneDeep(seenRef));
+        resolveTo, stack, _.cloneDeep(seenRef));
     }
     if (schema.$ref && _.isFunction(schema.$ref.split)) {
       let refKey = schema.$ref;
@@ -186,7 +190,7 @@ module.exports = {
 
       if (resolvedSchema) {
         let refResolvedSchema = this.resolveRefs(resolvedSchema, parameterSourceOption,
-          components, schemaResolutionCache, resolveFor, stack, _.cloneDeep(seenRef));
+          components, schemaResolutionCache, resolveFor, resolveTo, stack, _.cloneDeep(seenRef));
 
         if (refResolvedSchema && refResolvedSchema.value !== ERR_TOO_MANY_LEVELS) {
           schemaResolutionCache[refKey] = refResolvedSchema;
@@ -221,8 +225,8 @@ module.exports = {
               continue;
             }
             /* eslint-enable */
-            tempSchema.properties[prop] = this.resolveRefs(property,
-              parameterSourceOption, components, schemaResolutionCache, resolveFor, stack, _.cloneDeep(seenRef));
+            tempSchema.properties[prop] = this.resolveRefs(property, parameterSourceOption, components,
+              schemaResolutionCache, resolveFor, resolveTo, stack, _.cloneDeep(seenRef));
           }
         }
         return tempSchema;
@@ -260,13 +264,13 @@ module.exports = {
       // without this, schemas with circular references aren't faked correctly
       let tempSchema = _.omit(schema, 'items');
       tempSchema.items = this.resolveRefs(schema.items, parameterSourceOption,
-        components, schemaResolutionCache, resolveFor, stack, _.cloneDeep(seenRef));
+        components, schemaResolutionCache, resolveFor, resolveTo, stack, _.cloneDeep(seenRef));
       return tempSchema;
     }
     else if (!schema.hasOwnProperty('default')) {
       if (schema.hasOwnProperty('type')) {
-        // Don't override default value to schema for validation
-        if (resolveFor === 'CONVERSION') {
+        // Override default value to schema for CONVERSION only for parmeter resolution set to schema
+        if (resolveFor === 'CONVERSION' && resolveTo === 'schema') {
           if (!schema.hasOwnProperty('format')) {
             schema.default = '<' + schema.type + '>';
           }

lib/options.js

@@ -39,7 +39,7 @@ module.exports = {
         ' If “Fallback” is selected, the request will be named after one of the following schema' +
         ' values: `description`, `operationid`, `url`.',
         external: true,
-        usage: ['CONVERSION']
+        usage: ['CONVERSION', 'VALIDATION']
       },
       {
         name: 'Set indent character',
@@ -154,6 +154,24 @@ module.exports = {
         description: 'Whether to provide fixes for patching corresponding mismatches.',
         external: true,
         usage: ['VALIDATION']
+      },
+      {
+        name: 'Show Metadata validation messages',
+        id: 'validateMetadata',
+        type: 'boolean',
+        default: false,
+        description: 'Whether to show mismatches for incorrect name and description of request',
+        external: true,
+        usage: ['VALIDATION']
+      },
+      {
+        name: 'Ignore mismatch for unresolved postman variables',
+        id: 'ignoreUnresolvedVariables',
+        type: 'boolean',
+        default: false,
+        description: 'Whether to ignore mismatches resulting from unresolved variables in the Postman request',
+        external: true,
+        usage: ['VALIDATION']
       }
     ];