feat(core): support object style htmlStyle in themed token, support new htmlAttrs

Author: antfu

packages/core/src/highlight/code-to-hast.ts

@@ -171,11 +171,14 @@ export function tokensToHast(
       let tokenNode: Element = {
         type: 'element',
         tagName: 'span',
-        properties: {},
+        properties: {
+          ...token.htmlAttrs,
+        },
         children: [{ type: 'text', value: token.content }],
       }
 
-      const style = token.htmlStyle || stringifyTokenStyle(getTokenStyleObject(token))
+      // TODO: Shiki2: Deprecate `string` type of `htmlStyle`
+      const style = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token))
       if (style)
         tokenNode.properties.style = style
 

packages/core/src/highlight/code-to-tokens.ts

@@ -1,6 +1,6 @@
 import type { CodeToTokensOptions, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from '@shikijs/types'
 import { ShikiError } from '../../../types/src/error'
-import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements, stringifyTokenStyle } from '../utils'
+import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements } from '../utils'
 import { codeToTokensBase } from './code-to-tokens-base'
 import { codeToTokensWithThemes } from './code-to-tokens-themes'
 
@@ -103,27 +103,23 @@ function mergeToken(
 
   // Get all style keys, for themes that missing some style, we put `inherit` to override as needed
   const styleKeys = new Set(styles.flatMap(t => Object.keys(t)))
-  const mergedStyles = styles.reduce((acc, cur, idx) => {
+  const mergedStyles: Record<string, string> = {}
+
+  styles.forEach((cur, idx) => {
     for (const key of styleKeys) {
       const value = cur[key] || 'inherit'
 
       if (idx === 0 && defaultColor) {
-        acc[key] = value
+        mergedStyles[key] = value
       }
       else {
         const keyName = key === 'color' ? '' : key === 'background-color' ? '-bg' : `-${key}`
         const varKey = cssVariablePrefix + variantsOrder[idx] + (key === 'color' ? '' : keyName)
-        if (acc[key])
-          acc[key] += `;${varKey}:${value}`
-        else
-          acc[key] = `${varKey}:${value}`
+        mergedStyles[varKey] = value
       }
     }
-    return acc
-  }, {} as Record<string, string>)
+  })
 
-  token.htmlStyle = defaultColor
-    ? stringifyTokenStyle(mergedStyles)
-    : Object.values(mergedStyles).join(';')
+  token.htmlStyle = mergedStyles
   return token
 }

packages/core/src/utils.ts

@@ -210,7 +210,9 @@ export function getTokenStyleObject(token: TokenStyles): Record<string, string>
   return styles
 }
 
-export function stringifyTokenStyle(token: Record<string, string>): string {
+export function stringifyTokenStyle(token: string | Record<string, string>): string {
+  if (typeof token === 'string')
+    return token
   return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(';')
 }
 

packages/shiki/test/out/multiple-themes.html

@@ -183,7 +183,9 @@ function toggleTheme() {
     })
 
     expect(code1)
-      .toContain('font-style:italic;--shiki-dark-font-style:inherit')
+      .toContain('font-style:italic')
+    expect(code1)
+      .toContain('--shiki-dark-font-style:inherit')
 
     const code2 = await codeToHtml(input, {
       lang: 'js',
@@ -195,7 +197,9 @@ function toggleTheme() {
     })
 
     expect(code2)
-      .toContain('font-style:inherit;--shiki-light-font-style:italic')
+      .toContain('font-style:inherit')
+    expect(code2)
+      .toContain('--shiki-light-font-style:italic')
   })
 
   it('should not have empty style', async () => {

packages/shiki/test/themes.test.ts

@@ -0,0 +1,32 @@
+import { expect, it } from 'vitest'
+import { createHighlighter } from '../src'
+
+it('transformers tokens', async () => {
+  const shiki = await createHighlighter({
+    themes: ['vitesse-light'],
+    langs: ['javascript'],
+  })
+
+  expect(shiki.codeToHtml('console.log', {
+    lang: 'js',
+    theme: 'vitesse-light',
+    transformers: [
+      {
+        name: 'test',
+        tokens(tokens) {
+          for (const line of tokens) {
+            for (const token of line) {
+              token.htmlAttrs = { class: 'test' }
+              if (typeof token.htmlStyle !== 'string') {
+                token.htmlStyle ||= {}
+                token.htmlStyle.display = 'block'
+              }
+            }
+          }
+          return tokens
+        },
+      },
+    ],
+  }))
+    .toMatchInlineSnapshot(`"<pre class="shiki vitesse-light" style="background-color:#ffffff;color:#393a34" tabindex="0"><code><span class="line"><span class="test" style="display:block">console</span><span class="test" style="display:block">.</span><span class="test" style="display:block">log</span></span></code></pre>"`)
+})

packages/shiki/test/transformers.test.ts

@@ -143,8 +143,13 @@ export interface TokenStyles {
   /**
    * Override with custom inline style for HTML renderer.
    * When specified, `color` and `fontStyle` will be ignored.
+   * Prefer use object style for merging with other styles.
    */
-  htmlStyle?: string
+  htmlStyle?: string | Record<string, string>
+  /**
+   * Extra HTML attributes for the token.
+   */
+  htmlAttrs?: Record<string, string>
 }
 
 export interface ThemedTokenWithVariants extends TokenBase {