Improvements in tools to help merge crowdin PRs (github#18409)

- add `script/test-render-translation.js` to render all translated content to catch malformed liquid that would cause render errors
- improve test output for `script/fix-translation-errors.js` and `tests/content/lint-files.js`
- make it so `script/reset-translated-file.js` can handle files that have been renamed

Author: vanessayuenn

jest.config.js

@@ -4,11 +4,11 @@ const isBrowser = process.env.BROWSER
 const isActions = Boolean(process.env.GITHUB_ACTIONS)
 const testTranslation = Boolean(process.env.TEST_TRANSLATION)
 
-const reporters = ['default']
+let reporters = ['default']
 
 if (testTranslation) {
   // only use custom reporter if we are linting translations
-  reporters.push('<rootDir>/tests/helpers/lint-translation-reporter.js')
+  reporters = ['<rootDir>/tests/helpers/lint-translation-reporter.js']
 } else if (isActions) {
   reporters.push('jest-github-actions-reporter')
 }

package-lock.json

@@ -158,6 +158,7 @@
     "replace": "^1.2.0",
     "robots-parser": "^2.1.1",
     "start-server-and-test": "^1.12.0",
+    "strip-ansi": "^6.0.0",
     "supertest": "^4.0.2",
     "url-template": "^2.0.8",
     "webpack-dev-middleware": "^3.7.2",

package.json

@@ -71,13 +71,14 @@ changedFilesRelPaths.forEach(async (relPath) => {
   if (!engResult) return
   const { data: engData } = engResult
 
-  console.log(chalk.red('fixing errors in ') + chalk.bold(relPath))
+  console.log(chalk.bold(relPath))
 
   const newData = data
 
-  fixableErrors.forEach(({ property }) => {
+  fixableErrors.forEach(({ property, message }) => {
     const correctValue = get(engData, property)
-    console.log(`  [${property}]: ${get(data, property)} -> ${correctValue}`)
+    console.log(chalk.red(`  error message: [${property}] ${message}`))
+    console.log(`  fix property [${property}]: ${get(data, property)} -> ${correctValue}`)
     set(newData, property, correctValue)
   })
 

script/fix-translation-errors.js

@@ -13,63 +13,56 @@
 //
 // Examples:
 //
-// reset a single translated file using a relative path:
 // $ script/reset-translated-file.js translations/es-XL/content/actions/index.md
 //
-// reset a single translated file using a full path:
-// $ script/reset-translated-file.js /Users/z/git/github/docs-internal/translations/es-XL/content/actions/index.md
-//
-// reset all language variants of a single English file (using a relative path):
-// $ script/reset-translated-file.js content/actions/index.md
-// $ script/reset-translated-file.js data/ui.yml
-//
-// reset all language variants of a single English file (using a full path):
-// $ script/reset-translated-file.js /Users/z/git/github/docs-internal/content/desktop/index.md
-// $ script/reset-translated-file.js /Users/z/git/github/docs-internal/data/ui.yml
-//
 // [end-readme]
 
 const program = require('commander')
 const { execSync } = require('child_process')
 const assert = require('assert')
 const fs = require('fs')
 const path = require('path')
-const languages = require('../lib/languages')
+const chalk = require('chalk')
 
 program
   .description('reset translated files')
-  .option('-m, --use-main', 'Reset file to the translated file from `main` branch instead of from English source.')
+  .option('-m, --prefer-main', 'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.')
   .parse(process.argv)
 
+const resetToEnglishSource = (translationFilePath) => {
+  assert(translationFilePath.startsWith('translations/'), 'path argument must be in the format `translations/<lang>/path/to/file`')
+  assert(fs.existsSync(translationFilePath), `file does not exist: ${translationFilePath}`)
+
+  const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep)
+  const englishFile = path.join(process.cwd(), relativePath)
+  assert(fs.existsSync(englishFile), `file does not exist: ${englishFile}`)
+
+  // replace file with English source
+  const englishContent = fs.readFileSync(englishFile, 'utf8')
+  fs.writeFileSync(translationFilePath, englishContent)
+  console.log('-> reverted to English: %s', path.relative(process.cwd(), translationFilePath))
+}
+
 const [pathArg] = program.args
 assert(pathArg, 'first arg must be a target filename')
-let languageCode
 
 // Is the arg a fully-qualified path?
-let relativePath = fs.existsSync(pathArg)
+const relativePath = fs.existsSync(pathArg)
   ? path.relative(process.cwd(), pathArg)
   : pathArg
 
-if (program.useMain) {
-  execSync(`git checkout main -- ${relativePath}`)
-  console.log('reverted to file from main branch: %s', relativePath)
-} else {
-  // extract relative path and language code if pathArg is in the format `translations/<lang>/path/to/file`
-  if (relativePath.startsWith('translations/')) {
-    languageCode = Object.values(languages).find(language => relativePath.startsWith(language.dir) && language.code !== 'en').code
-    relativePath = relativePath.split(path.sep).slice(2).join(path.sep)
+if (program.preferMain) {
+  try {
+    execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' })
+    console.log('-> reverted to file from main branch: %s', relativePath)
+  } catch (e) {
+    if (e.message.includes('pathspec')) {
+      console.warn(chalk.red(`cannot find ${relativePath} in main branch (likely because it was renamed); falling back to English source file.`))
+      resetToEnglishSource(relativePath)
+    } else {
+      console.warn(e.message)
+    }
   }
-
-  const englishFile = path.join(process.cwd(), relativePath)
-  assert(fs.existsSync(englishFile), `file does not exist: ${englishFile}`)
-  const englishContent = fs.readFileSync(englishFile, 'utf8')
-
-  Object.values(languages).forEach(({ code }) => {
-    if (code === 'en') return
-    if (languageCode && languageCode !== code) return
-
-    const translatedFile = path.join(process.cwd(), languages[code].dir, relativePath)
-    fs.writeFileSync(translatedFile, englishContent)
-    console.log('reverted to English: %s', path.relative(process.cwd(), translatedFile))
-  })
+} else {
+  resetToEnglishSource(relativePath)
 }

script/reset-translated-file.js

@@ -0,0 +1,57 @@
+const renderContent = require('../lib/render-content')
+const loadSiteData = require('../lib/site-data')
+const { loadPages } = require('../lib/pages')
+const languages = require('../lib/languages')
+const path = require('path')
+const { execSync } = require('child_process')
+const fs = require('fs')
+const frontmatter = require('../lib/frontmatter')
+const chalk = require('chalk')
+
+const main = async () => {
+  const siteData = loadSiteData()
+  const pages = await loadPages()
+  const contextByLanguage = {}
+  for (const lang in languages) {
+    const langObj = languages[lang]
+    const [crowdinLangCode] = langObj.dir === '' ? 'en' : langObj.dir.split('/').slice(1)
+    if (!crowdinLangCode) continue
+    contextByLanguage[crowdinLangCode] = {
+      site: siteData[langObj.code].site,
+      currentLanguage: langObj.code,
+      currentVersion: 'free-pro-team@latest'
+    }
+  }
+
+  const rootDir = path.join(__dirname, '..')
+
+  const changedFilesRelPaths = execSync('git diff --name-only origin/main | egrep "^translations/.*/.+.md$"', { maxBuffer: 1024 * 1024 * 100 })
+    .toString()
+    .split('\n')
+    .filter(path => path !== '' && !path.endsWith('README.md'))
+    .sort()
+
+  console.log(`Found ${changedFilesRelPaths.length} translated files.`)
+
+  changedFilesRelPaths.forEach(async (relPath) => {
+    const fullPath = path.join(rootDir, relPath)
+    const lang = relPath.split('/')[1]
+    const context = {
+      ...contextByLanguage[lang],
+      pages,
+      page: pages.find(page => page.fullPath === fullPath),
+      redirects: {}
+    }
+    if (!context.page && !relPath.includes('data/reusables')) return
+    const fileContents = await fs.promises.readFile(fullPath, 'utf8')
+    const { content } = frontmatter(fileContents)
+    try {
+      await renderContent.liquid.parseAndRender(content, context)
+    } catch (err) {
+      console.log(chalk.bold(relPath))
+      console.log(chalk.red(`  error message: ${err.message}`))
+    }
+  })
+}
+
+main()