Remove separate syntax heads for each operator

This replaces all the specialized operator heads by a single K"Operator"
head that encodes the precedence level in its flags (except for operators
that are also used for non-operator purposes). The operators are
already K"Identifier" in the final parse tree. There is very little
reason to spend all of the extra effort separating them into separate
heads only to undo this later. Moreover, I think it's actively misleading,
because it makes people think that they can query things about an operator
by looking at the head, which doesn't work for suffixed operators.

Additionally, this removes the `op=` token, replacing it by two tokens,
one K"Operator" with a special precendence level and one `=`. This then
removes the last use of `bump_split` (since this PR is on top of #573).

As a free bonus this prepares us for having compound assignment syntax
for suffixed operators, which was infeasible in the flips parser. That
syntax change is not part of this PR but would be trivial (this PR
makes it an explicit error).

Fixes #334

Author: Keno

src/JuliaSyntax.jl

@@ -41,7 +41,11 @@ export SourceFile
 @_public source_line_range
 
 # Expression predicates, kinds and flags
-export @K_str, kind
+export @K_str, kind, PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT,
+    PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND,
+    PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS,
+    PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL,
+    PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN, generic_operators_by_level
 @_public Kind
 
 @_public flags,

src/core/parse_stream.jl

@@ -376,7 +376,10 @@ function _buffer_lookahead_tokens(lexer, lookahead)
         was_whitespace = is_whitespace(k)
         had_whitespace |= was_whitespace
         f = EMPTY_FLAGS
-        raw.suffix     && (f |= SUFFIXED_FLAG)
+        if k == K"Operator" && raw.op_precedence != Tokenize.PREC_NONE
+            # Store operator precedence in numeric flags
+            f |= set_numeric_flags(Int(raw.op_precedence))
+        end
         push!(lookahead, SyntaxToken(SyntaxHead(k, f), k,
                                      had_whitespace, raw.endbyte + 2))
         token_count += 1

src/integration/expr.jl

@@ -341,20 +341,28 @@ end
     elseif k == K"dots"
         n = numeric_flags(flags(nodehead))
         return n == 2 ? :(..) : :(...)
-    elseif k == K"op=" && length(args) == 3
-        lhs = args[1]
-        op = args[2]
-        rhs = args[3]
-        headstr = string(args[2], '=')
-        retexpr.head = Symbol(headstr)
-        retexpr.args = Any[lhs, rhs]
-    elseif k == K".op=" && length(args) == 3
-        lhs = args[1]
-        op = args[2]
-        rhs = args[3]
-        headstr = '.' * string(args[2], '=')
-        retexpr.head = Symbol(headstr)
-        retexpr.args = Any[lhs, rhs]
+    elseif k == K"op="
+        if length(args) == 3
+            lhs = args[1]
+            op = args[2]
+            rhs = args[3]
+            headstr = string(args[2], '=')
+            retexpr.head = Symbol(headstr)
+            retexpr.args = Any[lhs, rhs]
+        elseif length(args) == 1
+            return Symbol(string(args[1], '='))
+        end
+    elseif k == K".op="
+        if length(args) == 3
+            lhs = args[1]
+            op = args[2]
+            rhs = args[3]
+            headstr = '.' * string(args[2], '=')
+            retexpr.head = Symbol(headstr)
+            retexpr.args = Any[lhs, rhs]
+        else
+            return Symbol(string('.', args[1], '='))
+        end
     elseif k == K"macrocall"
         if length(args) >= 2
             a2 = args[2]

src/julia/julia_parse_stream.jl

@@ -262,45 +262,6 @@ function validate_tokens(stream::ParseStream)
     sort!(stream.diagnostics, by=first_byte)
 end
 
-"""
-    bump_split(stream, token_spec1, [token_spec2 ...])
-
-Bump the next token, splitting it into several pieces
-
-Tokens are defined by a number of `token_spec` of shape `(nbyte, kind, flags)`.
-If all `nbyte` are positive, the sum must equal the token length. If one
-`nbyte` is negative, that token is given `tok_len + nbyte` bytes and the sum of
-all `nbyte` must equal zero.
-
-This is a hack which helps resolves the occasional lexing ambiguity. For
-example
-* Whether .+ should be a single token or the composite (. +) which is used for
-  standalone operators.
-* Whether ... is splatting (most of the time) or three . tokens in import paths
-
-TODO: Are these the only cases?  Can we replace this general utility with a
-simpler one which only splits preceding dots?
-"""
-function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N}
-    tok = stream.lookahead[stream.lookahead_index]
-    stream.lookahead_index += 1
-    start_b = _next_byte(stream)
-    toklen = tok.next_byte - start_b
-    prev_b = start_b
-    for (i, (nbyte, k, f)) in enumerate(split_spec)
-        h = SyntaxHead(k, f)
-        actual_nbyte = nbyte < 0 ? (toklen + nbyte) : nbyte
-        orig_k = k == K"." ? K"." : kind(tok)
-        node = RawGreenNode(h, actual_nbyte, orig_k)
-        push!(stream.output, node)
-        prev_b += actual_nbyte
-        stream.next_byte += actual_nbyte
-    end
-    @assert tok.next_byte == prev_b
-    stream.peek_count = 0
-    return position(stream)
-end
-
 function peek_dotted_op_token(ps, allow_whitespace=false)
     # Peek the next token, but if it is a dot, peek the next one as well
     t = peek_token(ps)
@@ -318,7 +279,12 @@ function peek_dotted_op_token(ps, allow_whitespace=false)
             t = t2
         end
     end
-    return (isdotted, t)
+    isassign = false
+    if !allow_whitespace && is_operator(t)
+        t3 = peek_token(ps, 2+isdotted)
+        isassign = kind(t3) == K"=" && !preceding_whitespace(t3)
+    end
+    return (isdotted, isassign, t)
 end
 
 function bump_dotted(ps, isdot, t, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None")