Translation tools to make our lives easier (github#17712)

* create script to fix easy-to-fix frontmatter errors in translation

* improve reset-translated-file.js: allow reverting to the same file from `main` branch

* fix release-notes as well

* also lints liquid in frontmatter

jest.config.js

@@ -4,11 +4,11 @@ const isBrowser = process.env.BROWSER
 const isActions = Boolean(process.env.GITHUB_ACTIONS)
 const testTranslation = Boolean(process.env.TEST_TRANSLATION)
 
-let reporters = ['default']
+const reporters = ['default']
 
 if (testTranslation) {
   // only use custom reporter if we are linting translations
-  reporters = ['<rootDir>/tests/helpers/lint-translation-reporter.js']
+  reporters.push('<rootDir>/tests/helpers/lint-translation-reporter.js')
 } else if (isActions) {
   reporters.push('jest-github-actions-reporter')
 }

script/fix-translation-errors.js

@@ -0,0 +1,91 @@
+const { execSync } = require('child_process')
+const { get, set } = require('lodash')
+const fs = require('fs')
+const path = require('path')
+const fm = require('../lib/frontmatter')
+const matter = require('gray-matter')
+const chalk = require('chalk')
+const yaml = require('js-yaml')
+const ghesReleaseNotesSchema = require('../lib/release-notes-schema')
+const revalidator = require('revalidator')
+
+const fixableFmProps = ['type', 'changelog', 'mapTopic', 'hidden', 'layout', 'defaultPlatform', 'showMiniToc', 'allowTitleToDifferFromFilename', 'interactive', 'beta_product']
+const fixableYmlProps = ['date']
+
+// [start-readme]
+//
+// Run this script to fix known frontmatter errors by copying values from english file
+// Currently only fixing errors in: 'type', 'changelog'
+// Please double check the changes created by this script before committing.
+//
+// [end-readme]
+
+const loadAndValidateContent = async (path, schema) => {
+  let fileContents
+  try {
+    fileContents = await fs.promises.readFile(path, 'utf8')
+  } catch (e) {
+    console.error(e.message)
+    return null
+  }
+
+  if (path.endsWith('yml')) {
+    let data; let errors = []
+    try {
+      data = yaml.safeLoad(fileContents)
+    } catch {}
+    if (data && schema) {
+      ({ errors } = revalidator.validate(data, schema))
+    }
+    return { data, errors, content: null }
+  } else {
+    return fm(fileContents)
+  }
+}
+
+const cmd = 'git diff --name-only origin/main | egrep "^translations/.*/(content/.+.md|data/release-notes/.*.yml)$"'
+const changedFilesRelPaths = execSync(cmd).toString().split('\n')
+
+changedFilesRelPaths.forEach(async (relPath) => {
+  if (!relPath || relPath.endsWith('README.md')) return
+
+  const localisedAbsPath = path.join(__dirname, '..', relPath)
+  // find the corresponding english file by removing the first 2 path segments: /translation/<language code>
+  const engAbsPath = path.join(__dirname, '..', relPath.split(path.sep).slice(2).join(path.sep))
+
+  const localisedResult = await loadAndValidateContent(localisedAbsPath, ghesReleaseNotesSchema)
+  if (!localisedResult) return
+  const { data, errors, content } = localisedResult
+
+  const fixableProps = relPath.endsWith('yml') ? fixableYmlProps : fixableFmProps
+
+  const fixableErrors = errors.filter(({ property }) => {
+    const prop = property.split('.')
+    return fixableProps.includes(prop[0])
+  })
+
+  if (!data || fixableErrors.length === 0) return
+
+  const engResult = await loadAndValidateContent(engAbsPath)
+  if (!engResult) return
+  const { data: engData } = engResult
+
+  console.log(chalk.red('fixing errors in ') + chalk.bold(relPath))
+
+  const newData = data
+
+  fixableErrors.forEach(({ property }) => {
+    const correctValue = get(engData, property)
+    console.log(`  [${property}]: ${get(data, property)} -> ${correctValue}`)
+    set(newData, property, correctValue)
+  })
+
+  let toWrite
+  if (content) {
+    toWrite = matter.stringify(content, newData, { lineWidth: 10000, forceQuotes: true })
+  } else {
+    toWrite = yaml.safeDump(newData, { lineWidth: 10000, forceQuotes: true })
+  }
+
+  fs.writeFileSync(localisedAbsPath, toWrite)
+})

script/reset-translated-file.js

@@ -29,12 +29,19 @@
 //
 // [end-readme]
 
+const program = require('commander')
+const { execSync } = require('child_process')
 const assert = require('assert')
 const fs = require('fs')
 const path = require('path')
 const languages = require('../lib/languages')
 
-const [pathArg] = process.argv.slice(2)
+program
+  .description('reset translated files')
+  .option('-m, --use-main', 'Reset file to the translated file from `main` branch instead of from English source.')
+  .parse(process.argv)
+
+const [pathArg] = program.args
 assert(pathArg, 'first arg must be a target filename')
 let languageCode
 
@@ -43,21 +50,26 @@ let relativePath = fs.existsSync(pathArg)
   ? path.relative(process.cwd(), pathArg)
   : pathArg
 
-// extract relative path and language code if pathArg is in the format `translations/<lang>/path/to/file`
-if (relativePath.startsWith('translations/')) {
-  languageCode = Object.values(languages).find(language => relativePath.startsWith(language.dir) && language.code !== 'en').code
-  relativePath = relativePath.split(path.sep).slice(2).join(path.sep)
-}
+if (program.useMain) {
+  execSync(`git checkout main -- ${relativePath}`)
+  console.log('reverted to file from main branch: %s', relativePath)
+} else {
+  // extract relative path and language code if pathArg is in the format `translations/<lang>/path/to/file`
+  if (relativePath.startsWith('translations/')) {
+    languageCode = Object.values(languages).find(language => relativePath.startsWith(language.dir) && language.code !== 'en').code
+    relativePath = relativePath.split(path.sep).slice(2).join(path.sep)
+  }
 
-const englishFile = path.join(process.cwd(), relativePath)
-assert(fs.existsSync(englishFile), `file does not exist: ${englishFile}`)
-const englishContent = fs.readFileSync(englishFile, 'utf8')
+  const englishFile = path.join(process.cwd(), relativePath)
+  assert(fs.existsSync(englishFile), `file does not exist: ${englishFile}`)
+  const englishContent = fs.readFileSync(englishFile, 'utf8')
 
-Object.values(languages).forEach(({ code }) => {
-  if (code === 'en') return
-  if (languageCode && languageCode !== code) return
+  Object.values(languages).forEach(({ code }) => {
+    if (code === 'en') return
+    if (languageCode && languageCode !== code) return
 
-  const translatedFile = path.join(process.cwd(), languages[code].dir, relativePath)
-  fs.writeFileSync(translatedFile, englishContent)
-  console.log('reverted to English: %s', path.relative(process.cwd(), translatedFile))
-})
+    const translatedFile = path.join(process.cwd(), languages[code].dir, relativePath)
+    fs.writeFileSync(translatedFile, englishContent)
+    console.log('reverted to English: %s', path.relative(process.cwd(), translatedFile))
+  })
+}

tests/content/lint-files.js

@@ -205,7 +205,7 @@ if (!process.env.TEST_TRANSLATION) {
   const changedFilesRelPaths = execSync('git diff --name-only origin/main | egrep "^translations/.*/.+.(yml|md)$"').toString().split('\n')
   console.log(`Found ${changedFilesRelPaths.length} translated files.`)
 
-  const { mdRelPaths, ymlRelPaths, releaseNotesRelPaths } = groupBy(changedFilesRelPaths, (path) => {
+  const { mdRelPaths = [], ymlRelPaths = [], releaseNotesRelPaths = [] } = groupBy(changedFilesRelPaths, (path) => {
     // separate the changed files to different groups
     if (path.endsWith('README.md')) {
       return 'throwAway'
@@ -247,14 +247,15 @@ describe('lint markdown content', () => {
   describe.each(mdToLint)(
     '%s',
     (markdownRelPath, markdownAbsPath) => {
-      let content, ast, links, isHidden, isEarlyAccess, isSitePolicy, frontmatterErrors
+      let content, ast, links, isHidden, isEarlyAccess, isSitePolicy, frontmatterErrors, frontmatterData
 
       beforeAll(async () => {
         const fileContents = await fs.promises.readFile(markdownAbsPath, 'utf8')
         const { data, content: bodyContent, errors } = frontmatter(fileContents)
 
         content = bodyContent
         frontmatterErrors = errors
+        frontmatterData = data
         ast = generateMarkdownAST(content)
         isHidden = data.hidden === true
         isEarlyAccess = markdownRelPath.split('/').includes('early-access')
@@ -393,6 +394,17 @@ describe('lint markdown content', () => {
           const errorMessage = frontmatterErrors.map(error => `- [${error.property}]: ${error.actual}, ${error.message}`).join('\n')
           expect(frontmatterErrors.length, errorMessage).toBe(0)
         })
+
+        test('frontmatter contains valid liquid', async () => {
+          const fmKeysWithLiquid = ['title', 'shortTitle', 'intro', 'product', 'permission']
+            .filter(key => Boolean(frontmatterData[key]))
+
+          for (const key of fmKeysWithLiquid) {
+            expect(() => renderContent.liquid.parse(frontmatterData[key]))
+              .not
+              .toThrow()
+          }
+        })
       }
     }
   )