Merge branch 'master' into hyperupcall-various-improvements

Author: hyperupcall

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.0.0"
+    rev: "v3.0.1"
     hooks:
       - id: prettier
         types_or: [yaml, json, javascript, css, markdown]

src/Gruntfile.cjs

@@ -10,21 +10,20 @@ const Ajv2019 = require('ajv/dist/2019')
 const Ajv2020 = require('ajv/dist/2020')
 const AjvDraft06SchemaJson = require('ajv/dist/refs/json-schema-draft-06.json')
 const AjvStandalone = require('ajv/dist/standalone').default
-const tv4 = require('tv4')
 const TOML = require('@ltd/j-toml')
 const YAML = require('yaml')
 const schemasafe = require('@exodus/schemasafe')
 const prettier = require('prettier')
 const axios = require('axios').default
 const findDuplicatedPropertyKeys = require('find-duplicated-property-keys')
 
+const temporaryPreviousTv4OnlySchemas = ['tslint.json', 'cloudify.json']
 const temporaryCoverageDir = 'temp'
 const schemaDir = 'schemas/json'
 const testPositiveDir = 'test'
 const testNegativeDir = 'negative_test'
 const urlSchemaStore = 'https://json.schemastore.org/'
 const catalog = require('./api/json/catalog.json')
-const schemaV4JSON = require(path.resolve(schemaDir, 'schema-draft-v4.json'))
 const schemaValidation = require('./schema-validation.json')
 const schemasToBeTested = fs.readdirSync(schemaDir)
 const foldersPositiveTest = fs.readdirSync(testPositiveDir)
@@ -146,7 +145,6 @@ module.exports = function (grunt) {
   /**
    * @typedef {Object} localSchemaFileAndTestFileParameter2
    * @prop {boolean} fullScanAllFiles
-   * @prop {boolean} calledByTV4Validator
    * @prop {boolean} skipReadFile
    * @prop {boolean} ignoreSkiptest
    * @prop {string} processOnlyThisOneSchemaFile
@@ -169,7 +167,6 @@ module.exports = function (grunt) {
     },
     {
       fullScanAllFiles = false,
-      calledByTV4Validator = false,
       skipReadFile = true,
       ignoreSkiptest = false,
       processOnlyThisOneSchemaFile = undefined,
@@ -198,13 +195,7 @@ module.exports = function (grunt) {
       if (fullScanAllFiles) {
         return true // All tests are always performed.
       } else {
-        // Schema must be run for tv4 or AJV validator
-        // calledByTV4Validator is only set true when it is called by tv4 validator
-        // If schema is present in "tv4test" list then it can only be run if calledByTV4Validator = true
-        // If schema is NOT present in "tv4test" list then it can only be run if calledByTV4Validator = false
-        return schemaValidation.tv4test.includes(jsonFilename)
-          ? calledByTV4Validator
-          : !calledByTV4Validator
+        return true
       }
     }
 
@@ -411,111 +402,6 @@ module.exports = function (grunt) {
     }
   }
 
-  function tv4Validator() {
-    // tv4 validator can only process draft-04 schema
-    // All unknown keyword used in draft-06 and newer are just ignored.
-    // This is the correct implementation of the json schema specification.
-    const schemaVersion = showSchemaVersions()
-    const textValidate = 'validate              | '
-    const textPassSchema = 'pass schema          | '
-    const textPositivePassTest = 'pass positive test   | '
-    const textPositiveFailedTest = 'failed positive test | '
-
-    let schemaToBeValidated
-    let countSchema = 0
-
-    const processSchemaFile = (/** @type {Schema} */ schema) => {
-      // Start validate the JSON schema
-      let validated
-      let versionObj
-      let schemaVersionStr = 'unknown'
-      try {
-        // select the correct AJV object for this schema
-        schemaToBeValidated = schema.jsonObj
-        versionObj = schemaVersion.getObj(schemaToBeValidated)
-
-        // What schema draft version is it?
-        schemaVersionStr = versionObj ? versionObj.schemaName : 'unknown'
-
-        // validate the schema with draft-04. This is the only draft version it understands.
-        validated = tv4.validate(schemaToBeValidated, schemaV4JSON)
-      } catch (err) {
-        throwWithErrorText([
-          `${textValidate}${schema.urlOrFilePath} (${schemaVersionStr})`,
-          err,
-        ])
-      }
-      if (!validated) {
-        throwWithErrorText([
-          `${textValidate}${schema.urlOrFilePath} (${schemaVersionStr}`,
-          `(Schema file) keywordLocation: ${tv4.error.schemaPath}`,
-          `(Test file) instanceLocation:  ${tv4.error.dataPath}`,
-          `(Message)  ${tv4.error.message}`,
-          'Error in validation',
-        ])
-      }
-      countSchema++
-      grunt.log.writeln()
-      grunt.log.ok(
-        `${textPassSchema}${schema.urlOrFilePath} (${schemaVersionStr})`,
-      )
-    }
-
-    const processSchemaFileDone = () => {
-      grunt.log.writeln()
-      grunt.log.writeln(`Total schemas validated with tv4: ${countSchema}`)
-      countSchema = 0
-    }
-
-    const processPositiveTestFile = (/** @type {Schema} */ schema) => {
-      const testFile = schema.jsonObj
-      const validated = tv4.validate(testFile, schemaToBeValidated)
-      if (tv4.missing.length > 0) {
-        throwWithErrorText([
-          `${textPositiveFailedTest}${schema.urlOrFilePath}`,
-          `Missing URL: ${tv4.missing[0]}`,
-          'Must add URL and schema file in schema-validation.json (tv4ExternalRef list)',
-        ])
-      }
-      if (validated) {
-        grunt.log.ok(`${textPositivePassTest}${schema.urlOrFilePath}`)
-      } else {
-        throwWithErrorText([
-          `${textPositiveFailedTest}${schema.urlOrFilePath}`,
-          `(Schema file) keywordLocation: ${tv4.error.schemaPath}`,
-          `(Test file) instanceLocation:  ${tv4.error.dataPath}`,
-          `(Message)  ${tv4.error.message}`,
-          'Error in positive test.',
-        ])
-      }
-    }
-
-    const loadExternalSchema = () => {
-      for (const property in schemaValidation.tv4ExternalRef) {
-        try {
-          const schema = require(path.resolve(
-            '.',
-            schemaDir,
-            schemaValidation.tv4ExternalRef[property],
-          ))
-          tv4.addSchema(property, schema)
-        } catch (err) {
-          throwWithErrorText([
-            `Error in processing tv4ExternalRef ${property} : ${schemaValidation.tv4ExternalRef[property]}`,
-            err,
-          ])
-        }
-      }
-    }
-
-    loadExternalSchema()
-    return {
-      testSchemaFile: processSchemaFile,
-      testSchemaFileDone: processSchemaFileDone,
-      positiveTestFile: processPositiveTestFile,
-    }
-  }
-
   /**
    * @typedef {Object} FactoryAJVParameter
    * @prop {string} schemaName
@@ -888,23 +774,6 @@ module.exports = function (grunt) {
     },
   )
 
-  grunt.registerTask(
-    'local_test_tv4_only_for_non_compliance_schema',
-    'Use tv4 to validate local schemas in ./test/',
-    function () {
-      const x = tv4Validator()
-      // tv4 is an outdated/unreliable validator. Do not add a negative test scan here.
-      localSchemaFileAndTestFile(
-        {
-          schemaForTestScan: x.testSchemaFile,
-          positiveTestScan: x.positiveTestFile,
-          schemaForTestScanDone: x.testSchemaFileDone,
-        },
-        { calledByTV4Validator: true, skipReadFile: false },
-      )
-    },
-  )
-
   grunt.registerTask(
     'local_test_ajv',
     'Use AJV to validate local schemas in ./test/',
@@ -1136,7 +1005,7 @@ module.exports = function (grunt) {
           }
         }
       }
-      // Get all the json file for AJV and tv4
+      // Get all the json file for AJV
       localSchemaFileAndTestFile(
         { schemaOnlyScan: schemaFileCompare },
         { fullScanAllFiles: true },
@@ -1578,7 +1447,6 @@ module.exports = function (grunt) {
           }
         }
       }
-      checkForDuplicateInList(schemaValidation.tv4test, 'tv4test[]')
       checkForDuplicateInList(
         schemaValidation.ajvNotStrictMode,
         'ajvNotStrictMode[]',
@@ -1724,25 +1592,6 @@ module.exports = function (grunt) {
     },
   )
 
-  grunt.registerTask(
-    'local_assert_tv4_validator_has_no_negative_tests',
-    'Check for forbidden negative test folder',
-    function () {
-      const found = foldersNegativeTest.find((x) =>
-        schemaValidation.tv4test.includes(x + '.json'),
-      )
-      if (found) {
-        throwWithErrorText([
-          `Negative folder found for TV4 validator => ${path.join(
-            testNegativeDir,
-            found,
-          )}`,
-        ])
-      }
-      grunt.log.ok('OK')
-    },
-  )
-
   grunt.registerTask(
     'local_assert_schema-validation.json_no_missing_schema_files',
     'Check if all schema JSON files are present',
@@ -1760,7 +1609,6 @@ module.exports = function (grunt) {
           }
         })
       }
-      x(schemaValidation.tv4test)
       x(schemaValidation.ajvNotStrictMode)
       x(schemaValidation.skiptest)
       x(schemaValidation.missingcatalogurl)
@@ -1783,61 +1631,6 @@ module.exports = function (grunt) {
     },
   )
 
-  grunt.registerTask(
-    'local_assert_schema-validation.json_valid_skiptest',
-    'schemas in skiptest[] list must not be present anywhere else',
-    function () {
-      let countSchemaValidationItems = 0
-      const x = (list, listName) => {
-        list.forEach((schemaName) => {
-          if (schemaName.endsWith('.json')) {
-            countSchemaValidationItems++
-            if (schemaValidation.skiptest.includes(schemaName)) {
-              throwWithErrorText([
-                `Disabled/skiptest[] schema: ${schemaName} found in => ${listName}[]`,
-              ])
-            }
-          }
-        })
-      }
-      x(schemaValidation.tv4test, 'tv4test')
-      x(schemaValidation.ajvNotStrictMode, 'ajvNotStrictMode')
-      x(schemaValidation.missingcatalogurl, 'missingcatalogurl')
-      x(schemaValidation.highSchemaVersion, 'highSchemaVersion')
-
-      for (const item of schemaValidation.options) {
-        const schemaName = Object.keys(item).pop()
-        if (schemaName !== 'readme_example.json') {
-          countSchemaValidationItems++
-          if (schemaValidation.skiptest.includes(schemaName)) {
-            throwWithErrorText([
-              `Disabled/skiptest[] schema: ${schemaName} found in => options[]`,
-            ])
-          }
-        }
-      }
-
-      // Test folder must not exist if defined in skiptest[]
-      schemaValidation.skiptest.forEach((schemaName) => {
-        countSchemaValidationItems++
-        const folderName = schemaName.replace('.json', '')
-        if (foldersPositiveTest.includes(folderName)) {
-          throwWithErrorText([
-            `Disabled/skiptest[] schema: ${schemaName} cannot have positive test folder`,
-          ])
-        }
-        if (foldersNegativeTest.includes(folderName)) {
-          throwWithErrorText([
-            `Disabled/skiptest[] schema: ${schemaName} cannot have  negative test folder`,
-          ])
-        }
-      })
-      grunt.log.ok(
-        `Total schema-validation.json items check: ${countSchemaValidationItems}`,
-      )
-    },
-  )
-
   grunt.registerTask(
     'local_coverage',
     'Run one selected schema in coverage mode',
@@ -1968,12 +1761,6 @@ module.exports = function (grunt) {
           'Must start "make" file with schema name parameter.',
         ])
       }
-      // Not for tv4 schema files
-      if (schemaValidation.tv4test.includes(schemaNameToBeCoverage)) {
-        throwWithErrorText([
-          `Coverage is not possible for tv4-validator schema file :${schemaNameToBeCoverage}`,
-        ])
-      }
       generateCoverage(schemaNameToBeCoverage)
       grunt.log.ok('OK')
     },
@@ -2065,11 +1852,11 @@ module.exports = function (grunt) {
     'lint_catalog_entry_no_schema_word',
     'lint_top_level_$ref_is_standalone',
   ])
+
   grunt.registerTask('local_test_filesystem', [
     'local_assert_directory_structure_is_valid',
     'local_assert_filenames_have_correct_extensions',
     'local_assert_test_folders_have_at_least_one_test_schema',
-    'local_assert_tv4_validator_has_no_negative_tests',
   ])
   grunt.registerTask('local_test_schema_validation_json', [
     'local_assert_schema-validation.json_no_duplicate_list',
@@ -2099,7 +1886,6 @@ module.exports = function (grunt) {
     'local_print_schemas_tested_in_full_strict_mode',
     'local_print_schemas_without_positive_test_files',
     'local_test_ajv',
-    'local_test_tv4_only_for_non_compliance_schema',
     'local_print_url_counts_in_catalog',
     'local_print_count_schema_versions',
   ])