diff --git a/.vscode/settings.json b/.vscode/settings.json index e65a2f0e79e1..2c785c6085d6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,5 +20,35 @@ "[yaml]": {"editor.formatOnSave": true}, "[json]": {"editor.formatOnSave": true}, "[json5]": {"editor.formatOnSave": true}, - "[markdown]": {"editor.formatOnSave": true} + "[markdown]": {"editor.formatOnSave": true}, + + // Validate JSON schemas on the fly. Also used by the `amp check-json-schemas` + // task. Do not use URLs from the web, they are unsupported by this task. + "json.schemas": [ + { + "fileMatch": [ + "build-system/global-configs/canary-config.json", + "build-system/global-configs/prod-config.json" + ], + "url": "./build-system/json-schemas/amp-config.json" + }, + { + "fileMatch": ["build-system/tasks/bundle-size/APPROVERS.json"], + "url": "./build-system/json-schemas/APPROVERS.json" + }, + { + "fileMatch": ["build-system/global-configs/caches.json"], + "url": "./build-system/json-schemas/caches.json" + }, + { + "fileMatch": [ + "build-system/global-configs/client-side-experiments-config.json" + ], + "url": "./build-system/json-schemas/client-side-experiments-config.json" + }, + { + "fileMatch": ["build-system/tasks/bundle-size/filesize.json"], + "url": "./build-system/json-schemas/filesize.json" + } + ] } diff --git a/amp.js b/amp.js index 9e684728af4f..76d16c21900c 100755 --- a/amp.js +++ b/amp.js @@ -23,13 +23,13 @@ createTask('ava'); createTask('babel-plugin-tests', 'babelPluginTests'); createTask('build'); createTask('bundle-size', 'bundleSize'); -createTask('caches-json', 'cachesJson'); createTask('check-analytics-vendors-list', 'checkAnalyticsVendorsList'); createTask('check-asserts', 'checkAsserts'); createTask('check-build-system', 'checkBuildSystem'); createTask('check-exact-versions', 'checkExactVersions'); createTask('check-ignore-lists', 'checkIgnoreLists'); createTask('check-invalid-whitespaces', 'checkInvalidWhitespaces'); +createTask('check-json-schemas', 'checkJsonSchemas'); createTask('check-links', 'checkLinks'); createTask('check-owners', 'checkOwners'); createTask('check-renovate-config', 'checkRenovateConfig'); diff --git a/build-system/json-schemas/APPROVERS.json b/build-system/json-schemas/APPROVERS.json new file mode 100644 index 000000000000..f80996c5119c --- /dev/null +++ b/build-system/json-schemas/APPROVERS.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "@ampproject/filesize JSON config", + "type": "object", + "patternProperties": { + "^dist.+$": { + "type": "object", + "required": [ + "approvers", + "threshold" + ], + "properties": { + "approvers": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "pattern": "^ampproject/wg-[\\w_-]+$" + } + }, + "threshold": { + "type": "number", + "exclusiveMinimum": 0 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/build-system/json-schemas/amp-config.json b/build-system/json-schemas/amp-config.json new file mode 100644 index 000000000000..ba81a590a20d --- /dev/null +++ b/build-system/json-schemas/amp-config.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "AMP_CONFIG for canary/prod configurations", + "type": "object", + "required": [ + "allow-doc-opt-in", + "allow-url-opt-in", + "canary" + ], + "properties": { + "allow-doc-opt-in": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w-]+$" + } + }, + "allow-url-opt-in": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w-]+$" + } + }, + "canary": { + "type": "number", + "enum": [ + 0, + 1 + ] + } + }, + "patternProperties": { + "^(?!allow-doc-opt-in|allow-url-opt-in|canary)[\\w-]+$": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } +} diff --git a/build-system/json-schemas/caches.json b/build-system/json-schemas/caches.json new file mode 100644 index 000000000000..77998a181e85 --- /dev/null +++ b/build-system/json-schemas/caches.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "List of known AMP caches", + "type": "object", + "required": [ + "caches" + ], + "properties": { + "caches": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name", + "docs", + "cacheDomain", + "updateCacheApiDomainSuffix", + "thirdPartyFrameDomainSuffix" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9]+$" + }, + "name": { + "type": "string" + }, + "docs": { + "type": "string", + "format": "uri" + }, + "cacheDomain": { + "type": "string" + }, + "updateCacheApiDomainSuffix": { + "type": "string" + }, + "thirdPartyFrameDomainSuffix": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false +} diff --git a/build-system/json-schemas/client-side-experiments-config.json b/build-system/json-schemas/client-side-experiments-config.json new file mode 100644 index 000000000000..6be488e656f8 --- /dev/null +++ b/build-system/json-schemas/client-side-experiments-config.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Client-side (AMP_EXP) experiment definitions", + "type": "object", + "required": [ + "experiments" + ], + "properties": { + "experiments": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "percentage" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z][\\w-]*$" + }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "rtvPrefixes": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\d\\.]+$" + } + } + } + } + } + }, + "additionalProperties": false +} diff --git a/build-system/json-schemas/filesize.json b/build-system/json-schemas/filesize.json new file mode 100644 index 000000000000..5bb721f22768 --- /dev/null +++ b/build-system/json-schemas/filesize.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "@ampproject/filesize JSON config", + "type": "object", + "required": [ + "filesize" + ], + "properties": { + "filesize": { + "type": "object", + "required": [ + "track" + ], + "properties": { + "track": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "trackFormat": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "brotli", + "gzip", + "none" + ] + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/build-system/pr-check/build-targets.js b/build-system/pr-check/build-targets.js index 650b23e96c25..53fc47c1e4c2 100644 --- a/build-system/pr-check/build-targets.js +++ b/build-system/pr-check/build-targets.js @@ -7,6 +7,8 @@ */ const config = require('../test-configs/config'); const fastGlob = require('fast-glob'); +const fs = require('fs'); +const json5 = require('json5'); const minimatch = require('minimatch'); const path = require('path'); const {cyan} = require('kleur/colors'); @@ -25,6 +27,7 @@ let buildTargets; * Used to prevent the repeated expansion of globs during PR jobs. */ const fileLists = {}; +const jsonFilesWithSchemas = []; /*** * All of AMP's build targets that can be tested during CI. @@ -35,7 +38,6 @@ const Targets = { AVA: 'AVA', BABEL_PLUGIN: 'BABEL_PLUGIN', BUILD_SYSTEM: 'BUILD_SYSTEM', - CACHES_JSON: 'CACHES_JSON', DEV_DASHBOARD: 'DEV_DASHBOARD', DOCS: 'DOCS', E2E_TEST: 'E2E_TEST', @@ -43,6 +45,7 @@ const Targets = { IGNORE_LIST: 'IGNORE_LIST', INTEGRATION_TEST: 'INTEGRATION_TEST', INVALID_WHITESPACES: 'INVALID_WHITESPACES', + JSON_FILES: 'JSON_FILES', LINT: 'LINT', LINT_RULES: 'LINT_RULES', OWNERS: 'OWNERS', @@ -65,7 +68,6 @@ const Targets = { */ const nonRuntimeTargets = [ Targets.AVA, - Targets.CACHES_JSON, Targets.DEV_DASHBOARD, Targets.DOCS, Targets.E2E_TEST, @@ -146,15 +148,6 @@ const targetMatchers = { file.endsWith('.json'))) ); }, - [Targets.CACHES_JSON]: (file) => { - if (isOwnersFile(file)) { - return false; - } - return ( - file == 'build-system/tasks/caches-json.js' || - file == 'build-system/global-configs/caches.json' - ); - }, [Targets.DOCS]: (file) => { if (isOwnersFile(file)) { return false; @@ -210,6 +203,12 @@ const targetMatchers = { file.startsWith('build-system/test-configs') ); }, + [Targets.JSON_FILES]: (file) => { + return ( + jsonFilesWithSchemas.includes(file) || + file == 'build-system/tasks/check-json-schemas.js' + ); + }, [Targets.LINT]: (file) => { if (isOwnersFile(file)) { return false; @@ -398,6 +397,22 @@ function expandFileLists() { const fileListName = globName.replace('Globs', 'Files'); fileLists[fileListName] = fastGlob.sync(config[globName], {dot: true}); } + + const vscodeSettings = json5.parse( + fs.readFileSync('.vscode/settings.json', 'utf8') + ); + /** @type {Array<{fileMatch: string[], url: string}>} */ + const schemas = vscodeSettings['json.schemas']; + const jsonGlobs = schemas.flatMap(({fileMatch, url}) => [ + ...fileMatch, + path.normalize(url), + ]); + jsonFilesWithSchemas.push( + fastGlob.sync(jsonGlobs, { + dot: true, + ignore: ['**/node_modules'], + }) + ); } module.exports = { diff --git a/build-system/pr-check/checks.js b/build-system/pr-check/checks.js index 3d2c43712948..d8c168927c29 100644 --- a/build-system/pr-check/checks.js +++ b/build-system/pr-check/checks.js @@ -19,11 +19,11 @@ function pushBuildWorkflow() { timedExecOrDie('amp validate-html-fixtures'); timedExecOrDie('amp lint'); timedExecOrDie('amp prettify'); + timedExecOrDie('amp check-json-schemas'); timedExecOrDie('amp ava'); timedExecOrDie('amp check-build-system'); timedExecOrDie('amp check-ignore-lists'); timedExecOrDie('amp babel-plugin-tests'); - timedExecOrDie('amp caches-json'); timedExecOrDie('amp check-exact-versions'); timedExecOrDie('amp check-renovate-config'); timedExecOrDie('amp server-tests'); @@ -69,6 +69,10 @@ function prBuildWorkflow() { timedExecOrDie('amp prettify'); } + if (buildTargetsInclude(Targets.JSON_FILES)) { + timedExecOrDie('amp check-json-schemas'); + } + if (buildTargetsInclude(Targets.AVA)) { timedExecOrDie('amp ava'); } @@ -81,10 +85,6 @@ function prBuildWorkflow() { timedExecOrDie('amp babel-plugin-tests'); } - if (buildTargetsInclude(Targets.CACHES_JSON)) { - timedExecOrDie('amp caches-json'); - } - if (buildTargetsInclude(Targets.DOCS)) { timedExecOrDie('amp check-links --local_changes'); // only for PR builds timedExecOrDie('amp markdown-toc'); diff --git a/build-system/tasks/caches-json.js b/build-system/tasks/caches-json.js deleted file mode 100644 index e4a714590864..000000000000 --- a/build-system/tasks/caches-json.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict'; - -const path = require('path'); -const {cyan, green, red} = require('kleur/colors'); -const {log, logLocalDev} = require('../common/logging'); - -const expectedCaches = ['google', 'bing']; -const cachesJsonPath = '../global-configs/caches.json'; - -/** - * Entry point for amp caches-jason. - * @return {Promise} - */ -async function cachesJson() { - const filename = path.basename(cachesJsonPath); - let jsonContent; - try { - jsonContent = require(cachesJsonPath); - } catch (e) { - log(red('ERROR:'), 'Could not parse', cyan(filename)); - process.exitCode = 1; - return; - } - const foundCaches = []; - for (const foundCache of jsonContent.caches) { - foundCaches.push(foundCache.id); - } - for (const cache of expectedCaches) { - if (foundCaches.includes(cache)) { - logLocalDev(green('✔'), 'Found', cyan(cache), 'in', cyan(filename)); - } else { - log(red('✖'), 'Missing', cyan(cache), 'in', cyan(filename)); - process.exitCode = 1; - } - } -} - -module.exports = { - cachesJson, -}; - -cachesJson.description = 'Check that caches.json contains all expected caches'; diff --git a/build-system/tasks/check-json-schemas.js b/build-system/tasks/check-json-schemas.js new file mode 100644 index 000000000000..79a0a2449632 --- /dev/null +++ b/build-system/tasks/check-json-schemas.js @@ -0,0 +1,71 @@ +'use strict'; + +const fastGlob = require('fast-glob'); +const fs = require('fs'); +const json5 = require('json5'); +const {cyan, green, red} = require('kleur/colors'); +const {default: addFormats} = require('ajv-formats'); +const {default: Ajv} = require('ajv'); +const {log} = require('../common/logging'); + +/** + * Fetches the content of a JSON/JSON5 file. + * + * @param {string} file repo root relative. + * @return {any} + */ +function getJsonContents(file) { + return json5.parse(fs.readFileSync(file, 'utf8')); +} + +/** + * Checks JSON files against their JSON Schemas. + */ +function checkJsonSchemas() { + log('Validating JSON files'); + const ajv = new Ajv({allErrors: true}); + addFormats(ajv); + + const vscodeSettings = getJsonContents('.vscode/settings.json'); + const schemas = vscodeSettings['json.schemas']; + + for (const {fileMatch, url: schemaFile} of schemas.values()) { + log('Using schema', `${cyan(schemaFile)}:`); + const schemaJson = getJsonContents(schemaFile); + const validate = ajv.compile(schemaJson); + + const jsonFiles = fastGlob.sync(fileMatch, { + dot: true, + ignore: ['**/node_modules'], + }); + + for (const jsonFile of jsonFiles) { + try { + const jsonData = getJsonContents(jsonFile); + if (validate(jsonData)) { + log('⤷', cyan(jsonFile), '-', green('valid')); + } else { + process.exitCode = 1; + log('⤷', cyan(jsonFile), '-', red('invalid')); + for (const error of validate.errors || []) { + log(' ⤷', cyan(error.instancePath), red(error.message || '')); + } + } + } catch (error) { + process.exitCode = 1; + if (error instanceof SyntaxError) { + log('⤷', cyan(jsonFile), '-', red('unable to parse as a JSON file')); + } else { + throw error; + } + } + } + } +} + +module.exports = { + checkJsonSchemas, +}; + +checkJsonSchemas.description = + 'Checks JSON files against their required schemas'; diff --git a/build-system/tasks/pr-check.js b/build-system/tasks/pr-check.js index da8042d96f63..2c7a7a02eebe 100644 --- a/build-system/tasks/pr-check.js +++ b/build-system/tasks/pr-check.js @@ -71,10 +71,6 @@ async function prCheck() { runCheck('amp babel-plugin-tests'); } - if (buildTargetsInclude(Targets.CACHES_JSON)) { - runCheck('amp caches-json'); - } - if (buildTargetsInclude(Targets.DOCS)) { runCheck('amp check-links --local_changes'); } diff --git a/package-lock.json b/package-lock.json index 7a1ded10f2ef..03e1a67d5dbc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4669,6 +4669,18 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "globals": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.11.0.tgz", @@ -4684,6 +4696,12 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -6540,17 +6558,26 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.6.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.3.tgz", + "integrity": "sha512-SMJOdDP6LqTkD0Uq8qLi+gMwSt0imXLSV080qFVwJCpH9U6Mb+SUGHAXM0KNbcBPguytWyvFxcHgMLe2D2XSpw==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } }, + "ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dev": true, + "requires": { + "ajv": "^8.0.0" + } + }, "alphanum-sort": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz", @@ -10447,6 +10474,18 @@ "@babel/highlight": "^7.10.4" } }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -10508,6 +10547,12 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -12829,6 +12874,26 @@ "requires": { "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + } } }, "has": { @@ -16811,9 +16876,9 @@ "dev": true }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, "json-stable-stringify": { diff --git a/package.json b/package.json index c7962d9c86d7..edddf889584b 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,8 @@ "@types/mocha": "8.2.3", "@types/node": "14.17.21", "acorn-globals": "6.0.0", + "ajv": "8.6.3", + "ajv-formats": "2.1.1", "amphtml-validator": "1.0.35", "ast-replace": "1.1.3", "atob": "2.1.2",