Support parsing set forks

Author: ajyoon

new-version-ideas.org

@@ -24,7 +24,7 @@
    #+end_src
 ** Ideally leave room for some way to do smart grammar things
    Things like pluralization matching can be pretty tedious and often push me to skipping some chancelation variety. If there was some kind of way to support this in the language with a plugin that would be very helpful. It seems pretty complicated though..
-** TODO Support "re-using" choice blocks
+** DONE Support "re-using" choice blocks
    Sometimes I find myself wanting to reuse a choice block without copying the picked result from the original. Maybe this could be done with syntax like:
 
    #+begin_src bml

src/analysis.ts

@@ -95,6 +95,7 @@ function deriveForkNode(choiceFork: ChoiceFork, forkIdMap: ForkIdMap): ForkNode
 }
 
 function deriveRefNode(ref: Reference, forkIdMap: ForkIdMap): RefNode {
+  // TODO handle re-executing refs
   let forkMapLookupResult = forkIdMap.get(ref.id);
   if (!forkMapLookupResult) {
     // Handle unmapped refs gracefully - this is expected

src/choiceFork.ts

@@ -13,11 +13,13 @@ export class ChoiceFork {
   weights: WeightedChoice[];
   identifier: string | null;
   isSilent: boolean;
+  isSet: boolean;
 
-  constructor(weights: WeightedChoice[], identifier: string | null, isSilent: boolean) {
+  constructor(weights: WeightedChoice[], identifier: string | null, isSilent: boolean, isSet: boolean) {
     this.weights = normalizeWeights(weights);
     this.identifier = identifier;
     this.isSilent = isSilent;
+    this.isSet = isSet;
   }
 
   /**

src/lexer.ts

@@ -93,9 +93,15 @@ export class Lexer {
     } else if (this.str[this.index] === '@') {
       tokenType = TokenType.AT;
       tokenString = '@';
+    } else if (this.str[this.index] === '#') {
+      tokenType = TokenType.HASH;
+      tokenString = '#';
     } else if (this.str[this.index] === '!') {
       tokenType = TokenType.BANG;
       tokenString = '!';
+    } else if (this.str[this.index] === '$') {
+      tokenType = TokenType.DOLLAR;
+      tokenString = '$';
     } else if (this.str[this.index] === '[') {
       tokenType = TokenType.OPEN_BRACKET;
       tokenString = '[';
@@ -152,6 +158,7 @@ export class Lexer {
         if (token.tokenType === TokenType.CLOSE_BLOCK_COMMENT) {
           // Block comments output a single whitespace positioned at
           // the closing slash of the `*/`
+          // TODO why???? isn't it more intuitive that they should emit nothing???
           let virtualSpaceIdx = token.index + 1;
           return new Token(TokenType.WHITESPACE, virtualSpaceIdx, virtualSpaceIdx + 1, ' ');
         }

src/parsers.ts

@@ -128,12 +128,13 @@ export function parseFork(lexer: Lexer): ChoiceFork | Reference {
 
   // Big blob in 2nd capture is for identifiers inclusive of non-ascii chars
   // It's an approximation of JS identifiers.
-  let idRe = /(@|#|@!|)([_a-zA-Z\xA0-\uFFFF][_a-zA-Z0-9\xA0-\uFFFF]*)(:?)/y;
+  let idRe = /(@|#|@!|\$|#\$|)([_a-zA-Z\xA0-\uFFFF][_a-zA-Z0-9\xA0-\uFFFF]*)(:?)/y;
 
   let id = null;
   let isReference = false;
   let isSilent = false;
   let isReExecuting = false;
+  let isSet = false;
 
   let acceptId = true;
   let acceptWeight = false;
@@ -240,7 +241,7 @@ export function parseFork(lexer: Lexer): ChoiceFork | Reference {
           if (isReference) {
             return new Reference(id!, mappedChoices, unmappedChoices, isReExecuting);
           } else {
-            return new ChoiceFork(unmappedChoices, id, isSilent)
+            return new ChoiceFork(unmappedChoices, id, isSilent, isSet)
           }
         } else {
           if (!mappedChoices.size && !unmappedChoices.length) {
@@ -256,8 +257,10 @@ export function parseFork(lexer: Lexer): ChoiceFork | Reference {
           throw new BMLSyntaxError('Unexpected close brace in fork',
             lexer.str, token.index);
         }
-      case TokenType.TEXT:
       case TokenType.AT:
+      case TokenType.HASH:
+      case TokenType.DOLLAR:
+      case TokenType.TEXT:
         if (acceptId) {
           idRe.lastIndex = lexer.index;
           let idMatch = idRe.exec(lexer.str);
@@ -281,13 +284,17 @@ export function parseFork(lexer: Lexer): ChoiceFork | Reference {
             isReference = true;
             isReExecuting = true;
             if (includesColon) {
-              // error
               throw new BMLSyntaxError(`Re-executing reference '${id}' should not have a colon, `
                 + 'since re-executing references cannot have mappings.',
                 lexer.str, token.index, `Did you mean '{@!${id}}'?`)
             } else {
               acceptBlockEnd = true;
             }
+          } else if (typeSlug == '$') {
+            isSet = true;
+          } else if (typeSlug == '#$') {
+            isSet = true;
+            isSilent = true;
           }
           lexer.overrideIndex(lexer.index + idMatch[0].length);
           acceptId = false;

src/reference.ts

@@ -1,6 +1,5 @@
 import { Choice, WeightedChoice } from './weightedChoice';
 import { ChoiceFork } from './choiceFork';
-import { threadId } from 'worker_threads';
 
 export type ReferenceMap = Map<number, Choice>;
 
@@ -15,7 +14,7 @@ export class Reference {
     this.id = id;
     this.referenceMap = choiceMap;
     if (fallbackChocies.length) {
-      this.fallbackChoiceFork = new ChoiceFork(fallbackChocies, null, false);
+      this.fallbackChoiceFork = new ChoiceFork(fallbackChocies, null, false, false);
     } else {
       this.fallbackChoiceFork = null;
     }

src/renderer.ts

@@ -14,6 +14,8 @@ import { EvalContext } from './evalBlock';
 import * as fileUtils from './fileUtils';
 
 
+// If the referred fork is a silent set fork that has not yet been executed,
+// choiceIndex will be -1 and renderedOutput will be ''.
 export type ExecutedFork = { choiceFork: ChoiceFork, choiceIndex: number, renderedOutput: string }
 export type ExecutedForkMap = Map<string, ExecutedFork>
 

src/tokenType.ts

@@ -18,7 +18,9 @@ export enum TokenType {
   COMMA = 'COMMA',
   COLON = 'COLON',
   AT = 'AT',
+  HASH = 'HASH',
   BANG = 'BANG',
+  DOLLAR = 'DOLLAR',
   ARROW = 'ARROW',
   NUMBER = 'NUMBER',
   TEXT = 'TEXT',

test/testLexer.ts

@@ -115,12 +115,24 @@ describe('Lexer', function() {
     expect(lexer.next()).toBeNull();
   });
 
+  it('tokenizes hash', function() {
+    let lexer = new Lexer('#');
+    expect(lexer.next()).toEqual(new Token(TokenType.HASH, 0, 1, '#'));
+    expect(lexer.next()).toBeNull();
+  });
+
   it('tokenizes bang', function() {
     let lexer = new Lexer('!');
     expect(lexer.next()).toEqual(new Token(TokenType.BANG, 0, 1, '!'));
     expect(lexer.next()).toBeNull();
   });
 
+  it('tokenizes dollar', function() {
+    let lexer = new Lexer('$');
+    expect(lexer.next()).toEqual(new Token(TokenType.DOLLAR, 0, 1, '$'));
+    expect(lexer.next()).toBeNull();
+  });
+
   it('tokenizes arrow', function() {
     let lexer = new Lexer('->');
     expect(lexer.next()).toEqual(new Token(TokenType.ARROW, 0, 2, '->'));

test/testParsers.ts

@@ -180,8 +180,27 @@ describe('parseFork', function() {
     expect(result).toBeInstanceOf(ChoiceFork);
     expect((result as ChoiceFork).identifier).toBe('TestChoice');
     expect((result as ChoiceFork).isSilent).toBe(true);
+    expect((result as ChoiceFork).isSet).toBe(false);
   });
 
+  it('allows non-silent sets', function() {
+    let lexer = new Lexer('$TestChoice: (test)}');
+    let result = parseFork(lexer);
+    expect(result).toBeInstanceOf(ChoiceFork);
+    expect((result as ChoiceFork).identifier).toBe('TestChoice');
+    expect((result as ChoiceFork).isSilent).toBe(false);
+    expect((result as ChoiceFork).isSet).toBe(true);
+  })
+
+  it('allows silent sets', function() {
+    let lexer = new Lexer('#$TestChoice: (test)}');
+    let result = parseFork(lexer);
+    expect(result).toBeInstanceOf(ChoiceFork);
+    expect((result as ChoiceFork).identifier).toBe('TestChoice');
+    expect((result as ChoiceFork).isSilent).toBe(true);
+    expect((result as ChoiceFork).isSet).toBe(true);
+  })
+
   it('allows references', function() {
     let lexer = new Lexer('@TestChoice: 0 -> (foo)}');
     let result = parseFork(lexer);
@@ -226,11 +245,11 @@ describe('parseFork', function() {
         // Nested fork
         new ChoiceFork([
           new WeightedChoice(['foo'], 100)
-        ], null, false)
+        ], null, false, false)
       ], 60),
       // Alternate branch in outer fork
       new WeightedChoice(['bar'], 40)
-    ], null, false);
+    ], null, false, false);
     expect(result).toEqual(expectedResult);
   });
 });