feat: add dedupe for tokens

packages/core/src/compiler/__tests__/tokens-compiler.test.ts

@@ -77,4 +77,13 @@ describe('tokens-compiler', () => {
       expect(error instanceof NotFoundRef).toBeTruthy()
     }
   })
+
+  test('should merge tokens in different formats', () => {
+    const tokens: RawToken[] = [
+      { token1: { token2: { value: 'value-1' } } },
+      { 'token1-token2': { value: 'value-2' } },
+    ]
+    const result = compileTokens(tokens)
+    expect(result[0].value).toEqual('value-2')
+  })
 })

packages/core/src/compiler/dedupe.ts

@@ -0,0 +1,31 @@
+import dm from 'deepmerge'
+import { Token } from '../types'
+
+/**
+ * Remove duplicates from tokens array.
+ *
+ * @param tokens - Input tokens.
+ * @returns Compiled tokens without duplicates.
+ */
+export function dedupe(tokens: Token[]) {
+  const result: Token[] = []
+  const visited = new Map<string, number>()
+
+  const arrayMerge = (source: any[], target: any[]) => target
+
+  for (let token of tokens) {
+    const key = token.path.join('.')
+
+    if (visited.has(key)) {
+      const index = visited.get(key) as number
+      const mergedToken = dm.all([result[index], token], { arrayMerge }) as Token
+
+      result[index] = mergedToken
+    } else {
+      result.push(token)
+      visited.set(key, result.length - 1)
+    }
+  }
+
+  return result
+}

packages/core/src/compiler/tokenizer.ts

@@ -1,3 +1,5 @@
+import { dotCase } from 'change-case'
+
 import type { Token } from '../types'
 import { isObject } from '../utils/is-object'
 
@@ -17,11 +19,13 @@ export function tokenize(
   const result = prev
 
   for (const key in tokens) {
-    if (isObject(tokens[key]) && tokens[key].value) {
+    if (isObject(tokens[key]) && tokens[key].value !== undefined) {
       result.push({
         comment: tokens[key].comment,
         name: '',
-        path: [...context, key],
+        // `path` can consist out of complex words (['color', 'viewAction'])
+        // `normalizePath` converts into uniform form (['color', 'view', 'action'])
+        path: normalizePath([...context, key]),
         refs: [],
         value: tokens[key].value,
         original: {
@@ -35,3 +39,12 @@ export function tokenize(
 
   return result
 }
+
+function normalizePath(path: string[]) {
+  return (
+    path
+      .map((chunk) => dotCase(chunk))
+      .join('.')
+      .split('.')
+  )
+}

packages/core/src/compiler/tokens-compiler.ts

@@ -1,6 +1,7 @@
 import type { RawToken, Token } from '../types'
 import { deepMerge } from '../utils/deep-merge'
 import { tokenize } from './tokenizer'
+import { dedupe } from './dedupe'
 import { resolveTokensAliases } from './tokens-resolver'
 
 /**
@@ -13,6 +14,7 @@ export function compileTokens(tokens: RawToken[]): Token[] {
   let compiledTokens: Token[] = tokens as any[]
   compiledTokens = deepMerge(compiledTokens)
   compiledTokens = tokenize(compiledTokens)
+  compiledTokens = dedupe(compiledTokens)
   compiledTokens = resolveTokensAliases(compiledTokens)
 
   return compiledTokens

packages/core/src/internal/observer.ts

@@ -1,19 +1,17 @@
 import { compile } from '../index'
 import { CompileResult, CompileOptions } from '../compiler'
-import { RawToken, TokenValue } from '../types'
+import { TokenValue } from '../types'
 
 export type Watcher = (payload: CompileResult) => void
 type Compile = typeof compile
 
 export class ThemekitObserver {
-  private tokens: RawToken[]
   private options: CompileOptions
   private compile: Compile
   private watchers: Set<Watcher> = new Set()
 
   constructor(options: CompileOptions, _compile: Compile = compile) {
     this.compile = _compile
-    this.tokens = options.tokens
     this.options = options
     this.run(options)
   }
@@ -27,10 +25,8 @@ export class ThemekitObserver {
   }
 
   update(token: string, value: TokenValue) {
-    const tokens = [...this.tokens, { [token]: { value } }]
-    const options = { ...this.options, tokens }
-
-    this.run(options)
+    this.options.tokens.push({ [token]: { value } })
+    this.run(this.options)
   }
 
   private run(options: CompileOptions) {