refactor(scripts/optim): support model optimisation in rulesToJSON + …

…some refactoring
incubateur-ademe · Feb 15, 2023 · cc60df5 · cc60df5
1 parent 1faf80a
commit cc60df5
Show file tree

Hide file tree

Showing 3 changed files with 146 additions and 132 deletions.
diff --git a/scripts/i18n/addRegionToBaseRules.js b/scripts/i18n/addRegionToBaseRules.js
@@ -15,6 +15,8 @@ const addRegionToBaseRules = (baseRules, newRegionalRules) => {
 	}
 	Object.entries(newRegionalRules).forEach(([rule, attrs]) => {
 		if (!Object.keys(baseRules).includes(rule)) {
+			// TODO: to discuss about the wanted behavior.
+			// Should we throw an error or just ignore the rule?
 			return baseRules
 		}
 		Object.entries(attrs).forEach(([attr, transVal]) => {

diff --git a/scripts/modelOptim.mjs b/scripts/modelOptim.mjs
@@ -1,8 +1,54 @@
+// Description: contains wrappers around the constant folding optimization pass from
+// 				[publiopti] to be used in the build scripts: rulesToJSON.
+//
+// [publiopti]: https:github.com/EmileRolley/publiopti
+
 import Engine from 'publicodes'
 import path from 'path'
 import { readFileSync, writeFileSync } from 'fs'
 import { constantFolding, disabledLogger, getRawNodes } from 'publiopti'
 
+// Rule names which should be kept in the optimized model.
+const rulesToKeep = [
+	'bilan',
+	'actions',
+	'transport',
+	'pétrole . pleins',
+	'transport . voiture . thermique',
+	'logement . gaz',
+	'pétrole . volume plein',
+]
+
+export function compressRules(jsonPathWithoutExtension, destLang, markdown) {
+	const destPath = `${jsonPathWithoutExtension}-opti.json`
+	const err = constantFoldingFromJSONFile(
+		jsonPathWithoutExtension + '.json',
+		destPath,
+		['**/translated-*.yaml'],
+		([ruleName, ruleNode]) => {
+			return rulesToKeep.includes(ruleName) || 'icônes' in ruleNode.rawNode
+		}
+	)
+
+	if (err) {
+		if (markdown) {
+			console.log(
+				`| Rules compression for _${destLang}_ | ❌ | <details><summary>See error:</summary><br /><br /><code>${err}</code></details> |`
+			)
+		} else {
+			console.log(' ❌ An error occured while compressing rules in:', destPath)
+			console.log(err)
+		}
+		exit(-1)
+	} else {
+		console.log(
+			markdown
+				? `| Rules compression for _${destLang}_ | :heavy_check_mark: | Ø |`
+				: ` ✅ The rules have been correctly compressed in: ${destPath}`
+		)
+	}
+}
+
 /**
  * Applies a constant folding optimization pass to the parsed rules from the [model] path.
  *

diff --git a/scripts/rulesToJSON.mjs b/scripts/rulesToJSON.mjs
@@ -16,10 +16,12 @@ import utils from './i18n/utils.js'
 import { addRegionToBaseRules } from './i18n/addRegionToBaseRules.js'
 import { addTranslationToBaseRules } from './i18n/addTranslationToBaseRules.js'
 
-import { constantFoldingFromJSONFile } from './modelOptim.mjs'
+import { compressRules } from './modelOptim.mjs'
 
 const outputJSONPath = './public'
 
+const t9nDir = path.resolve('data/i18n/t9n')
+
 const { srcLang, srcFile, destLangs, destRegions, markdown } = cli.getArgs(
 	`Aggregates the model to an unique JSON file.`,
 
@@ -33,70 +35,85 @@ const { srcLang, srcFile, destLangs, destRegions, markdown } = cli.getArgs(
 	}
 )
 
-// The objective of supportedRegions function is to read regions models defined (only XX.yaml files) in 'data/i18n/models' and create a json file containing params of each region.
+/// ---------------------- Retrieving the regions models ----------------------
+
+const regionsModelsPath = path.resolve('data/i18n/models')
+const defaultModelCode = 'FR'
+const defaultRegionModelParam = {
+	[defaultModelCode]: {
+		nom: 'France métropolitaine',
+		gentilé: 'française',
+		code: defaultModelCode,
+	},
+}
+const supportedRegionPath = path.join(outputJSONPath, `supportedRegions.json`)
+
+//
+// Reads all regions models and create a json file containing params of each region.
+//
+// Only XX.yaml files are read in 'data/i18n/models' directory, their are the base models.
+// (XX-YY.yaml files are not read, they are the translation of the base models.)
+//
+// The default region and hardcoded one is FR.
+//
 const supportedRegions = fs
-	.readdirSync(path.resolve('data/i18n/models'))
-	.reduce(
-		(acc, filename) => {
-			if (!filename.match(/([A-Z]{2}).yaml/)) return acc
-			try {
-				const regionPath = path.resolve(`data/i18n/models/${filename}`)
-				const rules = utils.readYAML(regionPath)
-				const params = rules['params']
-				if (!params) {
-					console.log(
-						'Make sure a attribute "params" is defined in your region file'
-					)
-					exit(-1)
-				}
-				return { ...acc, [rules.params.code]: params }
-			} catch (err) {
+	.readdirSync(regionsModelsPath)
+	.reduce((acc, filename) => {
+		if (!filename.match(/([A-Z]{2})-fr.yaml/)) return acc
+		try {
+			const regionPath = path.join(regionsModelsPath, filename)
+			const rules = utils.readYAML(regionPath)
+			const params = rules['params']
+			if (params === undefined) {
 				console.log(
-					' ❌ Une erreur est survenue lors de la lecture du fichier',
-					filename,
-					':\n\n',
-					err.message
+					` ❌ The file ${filename} doesn't contain a 'params' key, aborting...`
 				)
 				exit(-1)
 			}
-		},
-		{
-			FR: {
-				nom: 'France métropolitaine',
-				gentilé: 'française',
-				code: 'FR',
-			},
+			return { ...acc, [rules.params.code]: params }
+		} catch (err) {
+			console.log(
+				' ❌ An error occured while reading the file:',
+				filename,
+				':\n\n',
+				err.message
+			)
+			exit(-1)
 		}
-	)
+	}, defaultRegionModelParam)
 
 const supportedRegionCodes = Object.keys(supportedRegions)
-const defaultModel = 'FR'
 
-const regions = (destRegions ?? supportedRegionCodes).filter((r) => {
-	if (!supportedRegionCodes.includes(r)) {
-		cli.printWarn(`SKIP: the region '${r}' is not supported.`)
-		return false
-	}
-	if (r === defaultModel) return false
-	return r
-})
+const regions =
+	destRegions?.filter((r) => {
+		if (!supportedRegionCodes.includes(r)) {
+			cli.printWarn(`[WARN] - the region '${r}' is not supported, skipping it.`)
+			return false
+		}
+		console.log(r, '!==', defaultModelCode)
+		return r !== defaultModelCode
+	}) ?? supportedRegionCodes
+
+/// ---------------------- Writting helpers ----------------------
 
 function writeSupportedRegions() {
-	const destPath = path.join(outputJSONPath, `supportedRegions.json`)
 	try {
-		fs.writeFileSync(destPath, JSON.stringify(supportedRegions))
+		fs.writeFileSync(supportedRegionPath, JSON.stringify(supportedRegions))
 		console.log(
 			markdown
-				? `| Supported Regions file | :heavy_check_mark: | Ø |`
-				: ` ✅ The rules have been correctly written in: ${destPath}`
+				? `| Supported regions | :heavy_check_mark: | Ø |`
+				: ` ✅ The rules have been correctly written in: ${supportedRegionPath}`
 		)
 	} catch (err) {
 		if (markdown) {
 			console.log(
-				`| Supported Regions file | ❌ | <details><summary>See error:</summary><br /><br /><code>${err}</code></details> |`
+				`| Supported regions | ❌ | <details><summary>See error:</summary><br /><br /><code>${err}</code></details> |`
 			)
 		} else {
-			console.log(' ❌ An error occured while writting rules in:', destPath)
+			console.log(
+				' ❌ An error occured while writting rules in:',
+				supportedRegionPath
+			)
 			console.log(err.message)
 		}
 		exit(-1)
@@ -124,52 +141,16 @@ function writeRules(rules, path, destLang) {
 	}
 }
 
-const rulesToKeep = [
-	'bilan',
-	'actions',
-	'transport',
-	'pétrole . pleins',
-	'transport . voiture . thermique',
-	'logement . gaz',
-	'pétrole . volume plein',
-]
+/// ---------------------- Main ----------------------
 
-function compressRules(jsonPathWithoutExtension, destLang) {
-	const destPath = `${jsonPathWithoutExtension}-opti.json`
-	const err = constantFoldingFromJSONFile(
-		jsonPathWithoutExtension + '.json',
-		destPath,
-		['**/translated-*.yaml'],
-		([ruleName, ruleNode]) => {
-			return rulesToKeep.includes(ruleName) || 'icônes' in ruleNode.rawNode
-		}
-	)
-
-	if (err) {
-		if (markdown) {
-			console.log(
-				`| Rules compression for _${destLang}_ | ❌ | <details><summary>See error:</summary><br /><br /><code>${err}</code></details> |`
-			)
-		} else {
-			console.log(' ❌ An error occured while compressing rules in:', destPath)
-			console.log(err)
-		}
-		exit(-1)
-	} else {
-		console.log(
-			markdown
-				? `| Rules compression for _${destLang}_ | :heavy_check_mark: | Ø |`
-				: ` ✅ The rules have been correctly compressed in: ${destPath}`
-		)
-	}
+if (markdown) {
+	console.log('| Task | Status | Message |')
+	console.log('|:-----|:------:|:-------:|')
 }
 
 writeSupportedRegions()
+
 glob(srcFile, { ignore: ['data/i18n/**'] }, (_, files) => {
-	const defaultDestPathWithoutExtension = path.join(
-		outputJSONPath,
-		`co2-model.FR-lang.${srcLang}`
-	)
 	const baseRules = files.reduce((acc, filename) => {
 		try {
 			const rules = utils.readYAML(path.resolve(filename))
@@ -187,63 +168,48 @@ glob(srcFile, { ignore: ['data/i18n/**'] }, (_, files) => {
 
 	try {
 		new Engine(baseRules).evaluate('bilan')
-
-		if (markdown) {
-			console.log('| Task | Status | Message |')
-			console.log('|:-----|:------:|:-------:|')
-		}
 		console.log(
 			markdown
 				? `| Rules evaluation | :heavy_check_mark: | Ø |`
 				: ' ✅ Les règles ont été évaluées sans erreur !'
 		)
 
-		writeRules(baseRules, defaultDestPathWithoutExtension + '.json', srcLang)
-		compressRules(defaultDestPathWithoutExtension, srcLang)
-
-		regions.forEach((region) => {
-			const destPath = path.join(
-				outputJSONPath,
-				`co2-model.${region.toUpperCase()}-lang.fr.json`
-			)
-			const regionRuleAttrs =
-				utils.readYAML(path.resolve(`data/i18n/models/${region}.yaml`)) ?? {}
-			const rehydratedRules = addRegionToBaseRules(baseRules, regionRuleAttrs)
-			writeRules(rehydratedRules, destPath, region)
-		})
-
+		destLangs.push(srcLang)
 		destLangs.forEach((destLang) => {
-			const destPathWithoutExtension = path.join(
-				outputJSONPath,
-				`co2-model.FR-lang.${destLang}`
-			)
-			const destPath = destPathWithoutExtension + '.json'
-			const translatedRuleAttrs =
-				utils.readYAML(
-					path.resolve(`data/i18n/t9n/translated-rules-${destLang}.yaml`)
-				) ?? {}
-			const translatedRules = addTranslationToBaseRules(
-				baseRules,
-				translatedRuleAttrs
-			)
-			writeRules(translatedRules, destPath, destLang)
-			regions.forEach((region) => {
-				const destPath = path.join(
+			const translatedBaseRules =
+				destLang === srcLang
+					? baseRules
+					: addTranslationToBaseRules(
+							baseRules,
+							utils.readYAML(
+								path.join(t9nDir, `translated-rules-${destLang}.yaml`)
+							) ?? {}
+					  )
+			regions.forEach((regionCode) => {
+				const localizedTranslatedBaseRules =
+					regionCode === defaultModelCode
+						? translatedBaseRules
+						: addRegionToBaseRules(
+								translatedBaseRules,
+								utils.readYAML(
+									path.join(
+										regionsModelsPath,
+										`${regionCode}-${destLang}.yaml`
+									) ?? {}
+								)
+						  )
+
+				const destPathWithoutExtension = path.join(
 					outputJSONPath,
-					`co2-model.${region}-lang.${destLang}.json`
+					`co2-model.${regionCode}-lang.${destLang}`
 				)
-				const regionRuleAttrs =
-					utils.readYAML(
-						path.resolve(`data/i18n/models/${region}-${destLang}.yaml`)
-					) ?? {}
-				const rehydratedRules = addRegionToBaseRules(
-					translatedRules,
-					regionRuleAttrs
+				writeRules(
+					localizedTranslatedBaseRules,
+					destPathWithoutExtension + '.json',
+					destLang
 				)
-				writeRules(rehydratedRules, destPath, region)
+				compressRules(destPathWithoutExtension, destLang)
 			})
-
-			compressRules(destPathWithoutExtension, destLang)
 		})
 	} catch (err) {
 		if (markdown) {