extracted generated “themed” tokens validation to its own action/code

Author: didoo

packages/tokens/scripts/build-parts/generateThemingCssFiles.ts

@@ -10,6 +10,8 @@ import chalk from 'chalk';
 import type { Dictionary, PlatformConfig }  from 'style-dictionary';
 import { fileHeader } from 'style-dictionary/utils';
 
+import { getSourceFromFileWithRootSelector } from './getSourceFromFileWithRootSelector.ts';
+
 export async function generateThemingCssFiles(_dictionary: Dictionary, config: PlatformConfig): Promise<void> {
 
   const commonSource = await getSourceFromFileWithRootSelector(config, 'default', 'common-tokens.css');
@@ -91,31 +93,6 @@ export async function generateThemingCssFiles(_dictionary: Dictionary, config: P
     await fs.ensureDir(outputFolder);
     await fs.writeFile(`${outputFolder}tokens.css`, outputTokensCss);
   }
-
-  // extra validation check to make sure that all the common files have actually the same content
-  const commonSourceCdsG0 = await getSourceFromFileWithRootSelector(config, 'cds-g0', 'common-tokens.css');
-  const commonSourceCdsG10 = await getSourceFromFileWithRootSelector(config, 'cds-g10', 'common-tokens.css');
-  const commonSourceCdsG90 = await getSourceFromFileWithRootSelector(config, 'cds-g90', 'common-tokens.css');
-  const commonSourceCdsG100 = await getSourceFromFileWithRootSelector(config, 'cds-g100', 'common-tokens.css');
-
-  Object.entries({
-    'cds-g0': commonSourceCdsG0,
-    'cds-g10': commonSourceCdsG10,
-    'cds-g90': commonSourceCdsG90,
-    'cds-g100': commonSourceCdsG100
-  }).forEach(([mode, source]: [string, string]) => {
-      if (source !== commonSource) {
-        // we want to interrupt the execution of the script if one of the generated "common" files is different from the others
-        // note: comment this out if you need to debug why they differ, so the files are saved with the different content
-        throw new Error(`❌ ${chalk.red.bold('ERROR')} - Generated "common" tokens for mode '${mode}' differ from the ones generated for the 'default' mode (expected to be identical)`);
-      }
-  });
-}
-
-async function getSourceFromFileWithRootSelector(config: PlatformConfig, theme: string, path: string): Promise<string> {
-  const rawSource = await fs.readFile(`${config.buildPath}themed-tokens/with-root-selector/${theme}/${path}`, 'utf8');
-  const header = await fileHeader({});
-  return rawSource.replace(header, '');
 }
 
 function getCssVariablesStalenessComment(): string {

packages/tokens/scripts/build-parts/getSourceFromFileWithRootSelector.ts

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+import fs from 'fs-extra';
+
+import type { PlatformConfig }  from 'style-dictionary';
+import { fileHeader } from 'style-dictionary/utils';
+
+export async function getSourceFromFileWithRootSelector(config: PlatformConfig, theme: string, path: string): Promise<string> {
+  const rawSource = await fs.readFile(`${config.buildPath}themed-tokens/with-root-selector/${theme}/${path}`, 'utf8');
+  const header = await fileHeader({});
+  return rawSource.replace(header, '');
+}

packages/tokens/scripts/build-parts/getStyleDictionaryConfig.ts

@@ -114,7 +114,7 @@ export function getStyleDictionaryConfig({ target, mode }: { target: Target, mod
                 filter: excludePrivateTokens,
               }
             ],
-            actions: ['generate-css-helpers', 'generate-theming-css-files'],
+            actions: ['generate-css-helpers', 'validate-theming-css-files', 'generate-theming-css-files'],
           },
           'docs/json': {
             buildPath: 'dist/docs/products/',

packages/tokens/scripts/build-parts/validateThemingCssFiles.ts

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+import chalk from 'chalk';
+
+import type { Dictionary, PlatformConfig }  from 'style-dictionary';
+
+import { getSourceFromFileWithRootSelector } from './getSourceFromFileWithRootSelector.ts';
+import type { Mode } from './getStyleDictionaryConfig.ts';
+import { modes } from './getStyleDictionaryConfig.ts';
+
+export async function validateThemingCssFiles(_dictionary: Dictionary, config: PlatformConfig): Promise<void> {
+
+  // store all the sources in memory
+  const allSources = {} as Record<Mode, Record<string, string>>;
+  for (const mode of modes) {
+    const commonSource = await getSourceFromFileWithRootSelector(config, mode, 'common-tokens.css');
+    const themedSource = await getSourceFromFileWithRootSelector(config, mode, 'themed-tokens.css');
+    allSources[mode] = { commonSource, themedSource }
+  }
+
+  // first validation: make sure that all the common files have actually the same content
+  const comparisonModes = modes.filter(mode => mode !== 'default');
+  comparisonModes.forEach((comparisonMode: Mode) => {
+    if (allSources[comparisonMode].commonSource !== allSources.default.commonSource) {
+      // we want to interrupt the execution of the script if one of the generated "common" files is different from the others
+      // note: comment this out if you need to debug why they differ, so the files are saved with the different content
+      throw new Error(`❌ ${chalk.red.bold('ERROR')} - Generated "common" tokens for mode '${comparisonMode}' differ from the ones generated for the 'default' mode (expected to be identical)`);
+    }
+  });
+
+  // second validation: make sure there are no orphans CSS variables when "common" and "themed" files are used together
+  for (const mode of modes) {
+    const { partialDefinitions: commonDefinitions, partialUsages: commonUsages } = extractAllCssVariables(allSources[mode].commonSource);
+    const { partialDefinitions: themedDefinitions, partialUsages: themedUsages } = extractAllCssVariables(allSources[mode].themedSource);
+    const allDefinitions = new Set([...commonDefinitions, ...themedDefinitions]);
+    const allUsages = new Set([...commonUsages, ...themedUsages]);
+    const undefinedVariables = [...allUsages].filter(usage => !allDefinitions.has(usage));
+    if (undefinedVariables.length > 0) {
+      throw new Error(`❌ ${chalk.red.bold('ERROR')} - Generated "common/themed" token files for mode '${mode} contain CSS variables that not defined in any of the generated files: ${undefinedVariables.map((variable: string) => `\`--token-${variable}\``).join(', ')}`);
+    }
+  }
+}
+
+// regex for variable definition (`--token-***: ***`) and usage: (`var(--token-***)`)
+const varDefRegex = /--token-([a-zA-Z0-9-_]+)\s*:/g;
+const varUsageRegex = /var\(\s*--token-([a-zA-Z0-9-_]+)\s*\)/g;
+
+function extractAllCssVariables(source: string) {
+  const cleanSource = stripCssComments(source);
+  const partialDefinitions = [];
+  const partialUsages = [];
+
+  // find all definitions and usages in this file
+  let match;
+  while ((match = varDefRegex.exec(cleanSource))) {
+    partialDefinitions.push(match[1]);
+  }
+  while ((match = varUsageRegex.exec(cleanSource))) {
+    partialUsages.push(match[1]);
+  }
+
+  return { partialDefinitions, partialUsages };
+}
+
+function stripCssComments(source: string) {
+  return source.replace(/\/\*[\s\S]*?\*\//g, '');
+}

packages/tokens/scripts/build.ts

@@ -18,6 +18,7 @@ import { targets, modes, getStyleDictionaryConfig } from './build-parts/getStyle
 import { customFormatCssThemedTokensFunctionForTarget } from './build-parts/customFormatCssThemedTokens.ts';
 import { customFormatDocsJsonFunction } from './build-parts/customFormatDocsJson.ts';
 import { generateCssHelpers } from './build-parts/generateCssHelpers.ts';
+import { validateThemingCssFiles } from './build-parts/validateThemingCssFiles.ts';
 import { generateThemingCssFiles } from './build-parts/generateThemingCssFiles.ts';
 import { generateThemingDocsFiles } from './build-parts/generateThemingDocsFiles.ts';
 
@@ -281,6 +282,12 @@ StyleDictionary.registerAction({
     undo: () => {}
 });
 
+StyleDictionary.registerAction({
+    name: 'validate-theming-css-files',
+    do: validateThemingCssFiles,
+    undo: () => {}
+});
+
 StyleDictionary.registerAction({
     name: 'generate-theming-docs-files',
     do: generateThemingDocsFiles,