Char Tokens

src/org/elixir_lang/Elixir.bnf

@@ -80,7 +80,7 @@ private quote ::= (charList | string)
 stringHeredoc ::= STRING_HEREDOC_PROMOTER EOL
                   interpolatedStringBody
                   STRING_HEREDOC_TERMINATOR
-private expression ::= (atom | NUMBER | charListHeredoc | quote | sigil | stringHeredoc)
+private expression ::= (atom | CHAR_TOKEN | NUMBER | charListHeredoc | quote | sigil | stringHeredoc)
 // allow multiple EOLs to separate expressions to handle lines comments
 private expressionList ::= expression (EOL+ expression)*
 interpolation ::= INTERPOLATION_START expressionList? INTERPOLATION_END

src/org/elixir_lang/Elixir.flex

@@ -182,6 +182,25 @@ COLON = :
 
 HEXADECIMAL_DIGIT = [A-Fa-f0-9]
 
+/*
+ * Escape Sequences
+ */
+
+ESCAPE = "\\"
+
+ESCAPED_CHARACTER = {ESCAPE} .
+ESCAPED_CHARACTER_CODE = {ESCAPE} "x{" {HEXADECIMAL_DIGIT}{1,6} "}" |
+                         {ESCAPE} "x" {HEXADECIMAL_DIGIT}{1,2}
+
+VALID_ESCAPE_SEQUENCE = {ESCAPED_CHARACTER_CODE} |
+                        {ESCAPED_CHARACTER}
+
+/*
+ * Char tokens
+ */
+
+CHAR_TOKEN = "?" ({VALID_ESCAPE_SEQUENCE} | .)
+
 /*
  * White Space
  */
@@ -308,19 +327,6 @@ SIGIL_HEREDOC_TERMINATOR = ({SIGIL_DOUBLE_QUOTES_TERMINATOR}|{SIGIL_SINGLE_QUOTE
 GROUP_TERMINATOR = {QUOTE_TERMINATOR}|{SIGIL_TERMINATOR}
 GROUP_HEREDOC_TERMINATOR = {QUOTE_HEREDOC_TERMINATOR}|{SIGIL_HEREDOC_TERMINATOR}
 
-/*
- * Escape Sequences
- */
-
-ESCAPE = "\\"
-
-ESCAPED_CHARACTER = {ESCAPE} .
-ESCAPED_CHARACTER_CODE = {ESCAPE} "x{" {HEXADECIMAL_DIGIT}{1,6} "}" |
-                         {ESCAPE} "x" {HEXADECIMAL_DIGIT}{1,2}
-
-VALID_ESCAPE_SEQUENCE = {ESCAPED_CHARACTER_CODE} |
-                        {ESCAPED_CHARACTER}
-
 /*
  *  States - Ordered lexigraphically
  */
@@ -350,6 +356,8 @@ VALID_ESCAPE_SEQUENCE = {ESCAPED_CHARACTER_CODE} |
   // This rule is only meant to match whitespace surrounded by other tokens as the above rule will handle blank lines.
   {WHITE_SPACE}+                            { return TokenType.WHITE_SPACE; }
 
+  {CHAR_TOKEN}                              { return ElixirTypes.CHAR_TOKEN; }
+
   {COLON}                                   { pushAndBegin(ATOM_START);
                                               return ElixirTypes.COLON; }
 

src/org/elixir_lang/ElixirColorSettingsPage.java

@@ -21,8 +21,9 @@
 public class ElixirColorSettingsPage implements ColorSettingsPage {
     private static final AttributesDescriptor[] DESCRIPTORS = new AttributesDescriptor[]{
             new AttributesDescriptor("Atom", ElixirSyntaxHighlighter.ATOM),
-            new AttributesDescriptor("Comment", ElixirSyntaxHighlighter.COMMENT),
             new AttributesDescriptor("Character List", ElixirSyntaxHighlighter.CHAR_LIST),
+            new AttributesDescriptor("Character Token", ElixirSyntaxHighlighter.CHAR_TOKEN),
+            new AttributesDescriptor("Comment", ElixirSyntaxHighlighter.COMMENT),
             new AttributesDescriptor("Escape Sequence", ElixirSyntaxHighlighter.VALID_ESCAPE_SEQUENCE),
             new AttributesDescriptor("Expression Substitution Mark", ElixirSyntaxHighlighter.EXPRESSION_SUBSTITUTION_MARK),
             new AttributesDescriptor("Number", ElixirSyntaxHighlighter.NUMBER),