Place ending heregex tokens one index earlier

This is an upstream port of decaffeinate#17

The lexer generates fake tokens for interpolated heregexes, and the ending
tokens were being placed where the start (inclusive) and end (inclusive) index
were one past the end of the heregex. This meant that in a case like
`[a ///#{b}///]`, the end tokens of the heregex and also the implicit function
call end were all being placed at the `]`, so the AST location data would say
that the function call ends at the end of the `]`.

To fix, I can just subtract 1 from the position of those ending heregex tokens
so that their end lines up with the end of the heregex itself. This is similar
to previous fixes that changed `OUTDENT` and `CALL_END` tokens so that the end
of the token lines up with the end of the AST node.

Author: alangpierce

lib/coffee-script/lexer.js

@@ -332,10 +332,10 @@ exports.Lexer = class Lexer
         @token 'CALL_START', '(', 0, 0
         @mergeInterpolationTokens tokens, {delimiter: '"', double: yes}, @formatHeregex
         if flags
-          @token ',', ',', index, 0
-          @token 'STRING', '"' + flags + '"', index, flags.length
-        @token ')', ')', end, 0
-        @token 'REGEX_END', ')', end, 0
+          @token ',', ',', index - 1, 0
+          @token 'STRING', '"' + flags + '"', index - 1, flags.length
+        @token ')', ')', end - 1, 0
+        @token 'REGEX_END', ')', end - 1, 0
 
     end
 

src/lexer.coffee

@@ -564,6 +564,29 @@ test "Verify indented heredocs have the right position", ->
   eq stringToken[2].last_line, 3
   eq stringToken[2].last_column, 4
 
+test "Verify heregexes with interpolations have the right ending position", ->
+  source = '''
+    [a ///#{b}///g]
+  '''
+  [..., stringEnd, comma, flagsString, regexCallEnd, regexEnd, fnCallEnd,
+    arrayEnd, terminator] = CoffeeScript.tokens source
+
+  eq comma[0], ','
+  eq arrayEnd[0], ']'
+
+  assertColumn = (token, column) ->
+    eq token[2].first_line, 0
+    eq token[2].first_column, column
+    eq token[2].last_line, 0
+    eq token[2].last_column, column
+
+  arrayEndColumn = arrayEnd[2].first_column
+  for token in [comma, flagsString]
+    assertColumn token, arrayEndColumn - 2
+  for token in [regexCallEnd, regexEnd, fnCallEnd]
+    assertColumn token, arrayEndColumn - 1
+  assertColumn arrayEnd, arrayEndColumn
+
 test "Verify all tokens get a location", ->
   doesNotThrow ->
     tokens = CoffeeScript.tokens testScript