refactor: delay creation of operatorsTrie and hide this implementation

BREAKING CHANGE: use `operators` instead of `operatorsTrie` as Tokenizer constructor argument, #500

rollup.config.mjs

@@ -16,6 +16,7 @@ const treeshake = {
   propertyReadSideEffects: false
 }
 const tsconfig = (target) => ({
+  check: true,
   tsconfigOverride: {
     include: [ 'src' ],
     exclude: [ 'test', 'benchmark' ],

src/liquid-options.ts

@@ -4,7 +4,6 @@ import { LRU } from './cache/lru'
 import { FS } from './fs/fs'
 import * as fs from './fs/node'
 import { defaultOperators, Operators } from './render/operator'
-import { createTrie, Trie } from './util/operator-trie'
 import { filters } from './filters'
 import { assert } from './types'
 
@@ -99,7 +98,6 @@ interface NormalizedOptions extends LiquidOptions {
   layouts?: string[];
   cache?: LiquidCache;
   outputEscape?: OutputEscape;
-  operatorsTrie?: Trie;
 }
 
 export interface NormalizedFullOptions extends NormalizedOptions {
@@ -130,7 +128,6 @@ export interface NormalizedFullOptions extends NormalizedOptions {
   globals: object;
   keepOutputType: boolean;
   operators: Operators;
-  operatorsTrie: Trie;
 }
 
 export const defaultOptions: NormalizedFullOptions = {
@@ -160,14 +157,10 @@ export const defaultOptions: NormalizedFullOptions = {
   lenientIf: false,
   globals: {},
   keepOutputType: false,
-  operators: defaultOperators,
-  operatorsTrie: createTrie(defaultOperators)
+  operators: defaultOperators
 }
 
 export function normalize (options: LiquidOptions): NormalizedFullOptions {
-  if (options.hasOwnProperty('operators')) {
-    (options as NormalizedOptions).operatorsTrie = createTrie(options.operators!)
-  }
   if (options.hasOwnProperty('root')) {
     if (!options.hasOwnProperty('partials')) options.partials = options.root
     if (!options.hasOwnProperty('layouts')) options.layouts = options.root

src/parser/parser.ts

@@ -29,7 +29,7 @@ export default class Parser {
     this.loader = new Loader(this.liquid.options)
   }
   public parse (html: string, filepath?: string): Template[] {
-    const tokenizer = new Tokenizer(html, this.liquid.options.operatorsTrie, filepath)
+    const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath)
     const tokens = tokenizer.readTopLevelTokens(this.liquid.options)
     return this.parseTokens(tokens)
   }

src/parser/tokenizer.ts

@@ -22,21 +22,24 @@ import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
 import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character'
 import { matchOperator } from './match-operator'
-import { Trie } from '../util/operator-trie'
+import { Trie, createTrie } from '../util/operator-trie'
 import { Expression } from '../render/expression'
+import { Operators } from '../render/operator'
 import { LiquidTagToken } from '../tokens/liquid-tag-token'
 
 export class Tokenizer {
   p = 0
   N: number
   private rawBeginAt = -1
+  private opTrie: Trie
 
   constructor (
     public input: string,
-    private trie: Trie = defaultOptions.operatorsTrie,
+    operators: Operators = defaultOptions.operators,
     public file: string = ''
   ) {
     this.N = input.length
+    this.opTrie = createTrie(operators)
   }
 
   readExpression () {
@@ -62,7 +65,7 @@ export class Tokenizer {
   }
   readOperator (): OperatorToken | undefined {
     this.skipBlank()
-    const end = matchOperator(this.input, this.p, this.trie)
+    const end = matchOperator(this.input, this.p, this.opTrie)
     if (end === -1) return
     return new OperatorToken(this.input, this.p, (this.p = end), this.file)
   }

src/tags/assign.ts

@@ -2,7 +2,7 @@ import { Value, Tokenizer, assert, TagImplOptions, TagToken, Context } from '../
 
 export default {
   parse: function (token: TagToken) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
     this.key = tokenizer.readIdentifier().content
     tokenizer.skipBlank()
     assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)

src/tags/capture.ts

@@ -3,7 +3,7 @@ import { evalQuotedToken } from '../render/expression'
 
 export default {
   parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
-    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)
     this.variable = readVariableName(tokenizer)
     assert(this.variable, () => `${tagToken.args} not valid identifier`)
 

src/tags/case.ts

@@ -12,7 +12,7 @@ export default {
       .on('tag:when', (token: TagToken) => {
         p = []
 
-        const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+        const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
 
         while (!tokenizer.end()) {
           const value = tokenizer.readValue()

src/tags/cycle.ts

@@ -4,7 +4,7 @@ import { Tokenizer } from '../parser/tokenizer'
 
 export default {
   parse: function (tagToken: TagToken) {
-    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)
     const group = tokenizer.readValue()
     tokenizer.skipBlank()
 

src/tags/decrement.ts

@@ -3,7 +3,7 @@ import { isNumber, stringify } from '../util/underscore'
 
 export default {
   parse: function (token: TagToken) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
     this.variable = tokenizer.readIdentifier().content
   },
   render: function (context: Context, emitter: Emitter) {

src/tags/for.ts

@@ -10,7 +10,7 @@ type valueof<T> = T[keyof T]
 export default {
   type: 'block',
   parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
 
     const variable = tokenizer.readIdentifier()
     const inStr = tokenizer.readIdentifier()

src/tags/include.ts

@@ -7,7 +7,7 @@ export default {
   renderFilePath,
   parse: function (token: TagToken) {
     const args = token.args
-    const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(args, this.liquid.options.operators)
     this['file'] = this.parseFilePath(tokenizer, this.liquid)
     this['currentFile'] = token.file
 

src/tags/increment.ts

@@ -3,7 +3,7 @@ import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../types'
 
 export default {
   parse: function (token: TagToken) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
     this.variable = tokenizer.readIdentifier().content
   },
   render: function (context: Context, emitter: Emitter) {

src/tags/layout.ts

@@ -7,7 +7,7 @@ export default {
   parseFilePath,
   renderFilePath,
   parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
     this['file'] = this.parseFilePath(tokenizer, this.liquid)
     this['currentFile'] = token.file
     this.hash = new Hash(tokenizer.remaining())

src/tags/liquid.ts

@@ -4,7 +4,7 @@ import { Tokenizer } from '../parser/tokenizer'
 
 export default {
   parse: function (token: TagToken) {
-    const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
     const tokens = tokenizer.readLiquidTagTokens(this.liquid.options)
     this.tpls = this.liquid.parser.parseTokens(tokens)
   },

src/tags/render.ts

@@ -10,7 +10,7 @@ export default {
   renderFilePath,
   parse: function (token: TagToken) {
     const args = token.args
-    const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(args, this.liquid.options.operators)
     this['file'] = this.parseFilePath(tokenizer, this.liquid)
     this['currentFile'] = token.file
     while (!tokenizer.end()) {

src/tags/tablerow.ts

@@ -5,7 +5,7 @@ import { Tokenizer } from '../parser/tokenizer'
 
 export default {
   parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
-    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)
 
     const variable = tokenizer.readIdentifier()
     tokenizer.skipBlank()

src/template/value.ts

@@ -12,7 +12,7 @@ export class Value {
    * @param str the value to be valuated, eg.: "foobar" | truncate: 3
    */
   public constructor (str: string, liquid: Liquid) {
-    const tokenizer = new Tokenizer(str, liquid.options.operatorsTrie)
+    const tokenizer = new Tokenizer(str, liquid.options.operators)
     this.initial = tokenizer.readExpression()
     this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, liquid.filters.get(name), args, liquid))
   }

src/tokens/liquid-tag-token.ts

@@ -22,7 +22,7 @@ export class LiquidTagToken extends DelimitedToken {
       this.name = ''
       this.args = ''
     } else {
-      const tokenizer = new Tokenizer(this.content, options.operatorsTrie)
+      const tokenizer = new Tokenizer(this.content, options.operators)
       this.name = tokenizer.readTagName()
       if (!this.name) throw new TokenizationError(`illegal liquid tag syntax`, this)
 

src/tokens/tag-token.ts

@@ -18,7 +18,7 @@ export class TagToken extends DelimitedToken {
     const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
     super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
 
-    const tokenizer = new Tokenizer(this.content, options.operatorsTrie)
+    const tokenizer = new Tokenizer(this.content, options.operators)
     this.name = tokenizer.readTagName()
     if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)