Refactor parser to support error recovery

implants/lib/eldritchv2/eldritch-core/src/ast.rs

@@ -614,6 +614,7 @@ pub enum ExprKind {
         then_branch: Box<Expr>,
         else_branch: Box<Expr>,
     },
+    Error(String),
 }
 
 #[derive(Debug, Clone)]
@@ -634,4 +635,5 @@ pub enum StmtKind {
     Break,
     Continue,
     Pass,
+    Error(String),
 }

implants/lib/eldritchv2/eldritch-core/src/interpreter/core.rs

@@ -187,10 +187,14 @@ impl Interpreter {
         }
 
         let mut parser = Parser::new(tokens);
-        let stmts = match parser.parse() {
-            Ok(s) => s,
-            Err(e) => return Err(format!("Parser Error: {e}")),
-        };
+        let (stmts, errors) = parser.parse();
+
+        if !errors.is_empty() {
+            // If we have parsing errors, we return the first one formatted
+            // Or maybe a combined list? Usually first is enough to abort execution.
+            // The prompt says "interpreter does not attempt to evaluate / execute statements from ASTs with error tokens"
+            return Err(self.format_error(input, errors[0].clone()));
+        }
 
         let mut last_val = Value::None;
 

implants/lib/eldritchv2/eldritch-core/src/interpreter/eval/mod.rs

@@ -77,5 +77,9 @@ pub fn evaluate(interp: &mut Interpreter, expr: &Expr) -> Result<Value, Eldritch
                 evaluate(interp, else_branch)
             }
         }
+        ExprKind::Error(msg) => interp.runtime_error(
+            &format!("Runtime encountered syntax error node: {}", msg),
+            span,
+        ),
     }
 }

implants/lib/eldritchv2/eldritch-core/src/interpreter/exec.rs

@@ -144,6 +144,12 @@ pub fn execute(interp: &mut Interpreter, stmt: &Stmt) -> Result<(), EldritchErro
         StmtKind::Break => interp.flow = Flow::Break,
         StmtKind::Continue => interp.flow = Flow::Continue,
         StmtKind::Pass => {} // Do nothing
+        StmtKind::Error(msg) => {
+            return interp.runtime_error(
+                &format!("Runtime encountered syntax error node: {}", msg),
+                stmt.span,
+            );
+        }
     }
     Ok(())
 }

implants/lib/eldritchv2/eldritch-core/src/lib.rs

@@ -12,7 +12,7 @@ mod parser;
 mod token;
 
 // Re-export core types
-pub use ast::{Environment, ForeignValue, Value};
+pub use ast::{Environment, ExprKind, ForeignValue, StmtKind, Value};
 pub use interpreter::{BufferPrinter, Interpreter, Printer, StdoutPrinter};
 pub use lexer::Lexer;
 pub use token::{Span, TokenKind};

implants/lib/eldritchv2/eldritch-core/src/parser/expr.rs

@@ -626,68 +626,135 @@ impl Parser {
                 return Ok(self.make_expr(ExprKind::List(Vec::new()), span, end));
             }
 
-            let first_expr = self.expression()?;
+            // Attempt to parse first expression.
+            // If it fails, we assume it's a List of expressions and try to recover.
+            // But if it succeeds, we check for comprehension syntax.
+            let first_expr_res = self.expression();
 
-            if self.match_token(&[TokenKind::For]) {
-                let (var, _) = {
-                    let t = self.consume(
-                        |t| matches!(t, TokenKind::Identifier(_)),
-                        "Expected iteration variable.",
-                    )?;
-                    let v = if let TokenKind::Identifier(s) = &t.kind {
-                        s.clone()
-                    } else {
-                        unreachable!()
+            // If first expression is valid, check for comprehension
+            if let Ok(first_expr) = first_expr_res {
+                if self.match_token(&[TokenKind::For]) {
+                    let (var, _) = {
+                        let t = self.consume(
+                            |t| matches!(t, TokenKind::Identifier(_)),
+                            "Expected iteration variable.",
+                        )?;
+                        let v = if let TokenKind::Identifier(s) = &t.kind {
+                            s.clone()
+                        } else {
+                            unreachable!()
+                        };
+                        (v, t.span)
                     };
-                    (v, t.span)
-                };
 
-                self.consume(|t| matches!(t, TokenKind::In), "Expected 'in'.")?;
-                // Use logic_or to avoid consuming the 'if' of the comprehension
-                let iterable = self.logic_or()?;
-                let mut cond = None;
-                if self.match_token(&[TokenKind::If]) {
-                    // Condition can be a full expression (ternary allowed inside it if parenthesized? no, generally allowed)
-                    // The condition of a comprehension: [x for x in y if (a if b else c)]
-                    // If we use expression(), it might consume 'else' if there was one after the comprehension?
-                    // But comprehensions are closed by ']'.
-                    // So expression() is safe here.
-                    cond = Some(Box::new(self.expression()?));
+                    self.consume(|t| matches!(t, TokenKind::In), "Expected 'in'.")?;
+                    let iterable = self.logic_or()?;
+                    let mut cond = None;
+                    if self.match_token(&[TokenKind::If]) {
+                        cond = Some(Box::new(self.expression()?));
+                    }
+                    let end = self
+                        .consume(|t| matches!(t, TokenKind::RBracket), "Expected ']'.")?
+                        .span;
+                    return Ok(self.make_expr(
+                        ExprKind::ListComp {
+                            body: Box::new(first_expr),
+                            var,
+                            iterable: Box::new(iterable),
+                            cond,
+                        },
+                        span,
+                        end,
+                    ));
+                }
+
+                // Not a comprehension, so it's a list literal starting with first_expr
+                let mut elements = vec![first_expr];
+                if self.match_token(&[TokenKind::Comma]) && !self.check(&TokenKind::RBracket) {
+                    loop {
+                        if self.check(&TokenKind::RBracket) {
+                            break;
+                        }
+                        match self.expression() {
+                            Ok(e) => elements.push(e),
+                            Err(e) => {
+                                self.errors.push(e.clone());
+                                // Recover: skip until comma or RBracket
+                                while !self.check(&TokenKind::Comma)
+                                    && !self.check(&TokenKind::RBracket)
+                                    && !self.is_at_end()
+                                {
+                                    self.advance();
+                                }
+                                elements.push(self.make_expr(
+                                    ExprKind::Error(e.message),
+                                    e.span,
+                                    e.span,
+                                ));
+                            }
+                        }
+
+                        if !self.match_token(&[TokenKind::Comma]) {
+                            break;
+                        }
+                    }
                 }
                 let end = self
-                    .consume(|t| matches!(t, TokenKind::RBracket), "Expected ']'.")?
+                    .consume(
+                        |t| matches!(t, TokenKind::RBracket),
+                        "Expected ']' after list.",
+                    )?
                     .span;
-                return Ok(self.make_expr(
-                    ExprKind::ListComp {
-                        body: Box::new(first_expr),
-                        var,
-                        iterable: Box::new(iterable),
-                        cond,
-                    },
-                    span,
-                    end,
-                ));
-            }
+                return Ok(self.make_expr(ExprKind::List(elements), span, end));
+            } else {
+                // First expression failed. Must be a list literal with error at start.
+                let e = first_expr_res.unwrap_err();
+                self.errors.push(e.clone());
+                // Recover: skip until comma or RBracket
+                while !self.check(&TokenKind::Comma)
+                    && !self.check(&TokenKind::RBracket)
+                    && !self.is_at_end()
+                {
+                    self.advance();
+                }
+                let err_expr = self.make_expr(ExprKind::Error(e.message), e.span, e.span);
+                let mut elements = vec![err_expr];
 
-            let mut elements = vec![first_expr];
-            if self.match_token(&[TokenKind::Comma]) && !self.check(&TokenKind::RBracket) {
-                loop {
-                    if self.check(&TokenKind::RBracket) {
-                        break;
-                    }
-                    elements.push(self.expression()?);
-                    if !self.match_token(&[TokenKind::Comma]) {
-                        break;
+                if self.match_token(&[TokenKind::Comma]) && !self.check(&TokenKind::RBracket) {
+                    loop {
+                        if self.check(&TokenKind::RBracket) {
+                            break;
+                        }
+                        match self.expression() {
+                            Ok(e) => elements.push(e),
+                            Err(e) => {
+                                self.errors.push(e.clone());
+                                while !self.check(&TokenKind::Comma)
+                                    && !self.check(&TokenKind::RBracket)
+                                    && !self.is_at_end()
+                                {
+                                    self.advance();
+                                }
+                                elements.push(self.make_expr(
+                                    ExprKind::Error(e.message),
+                                    e.span,
+                                    e.span,
+                                ));
+                            }
+                        }
+                        if !self.match_token(&[TokenKind::Comma]) {
+                            break;
+                        }
                     }
                 }
+                let end = self
+                    .consume(
+                        |t| matches!(t, TokenKind::RBracket),
+                        "Expected ']' after list.",
+                    )?
+                    .span;
+                return Ok(self.make_expr(ExprKind::List(elements), span, end));
             }
-            let end = self
-                .consume(
-                    |t| matches!(t, TokenKind::RBracket),
-                    "Expected ']' after list.",
-                )?
-                .span;
-            return Ok(self.make_expr(ExprKind::List(elements), span, end));
         }
 
         if self.match_token(&[TokenKind::LBrace]) {

implants/lib/eldritchv2/eldritch-core/src/parser/mod.rs

@@ -9,11 +9,16 @@ pub mod stmt;
 pub struct Parser {
     pub(crate) tokens: Vec<Token>,
     pub(crate) current: usize,
+    pub(crate) errors: Vec<EldritchError>,
 }
 
 impl Parser {
     pub fn new(tokens: Vec<Token>) -> Self {
-        Parser { tokens, current: 0 }
+        Parser {
+            tokens,
+            current: 0,
+            errors: Vec::new(),
+        }
     }
 
     pub(crate) fn peek(&self) -> &Token {
@@ -70,6 +75,29 @@ impl Parser {
         }
     }
 
+    pub(crate) fn synchronize(&mut self) {
+        self.advance();
+
+        while !self.is_at_end() {
+            if self.tokens[self.current - 1].kind == TokenKind::Newline {
+                return;
+            }
+
+            match self.peek().kind {
+                TokenKind::Def
+                | TokenKind::If
+                | TokenKind::For
+                | TokenKind::Return
+                | TokenKind::Pass
+                | TokenKind::Break
+                | TokenKind::Continue => return,
+                _ => {}
+            }
+
+            self.advance();
+        }
+    }
+
     pub(crate) fn match_token(&mut self, kinds: &[TokenKind]) -> bool {
         for k in kinds {
             if core::mem::discriminant(&self.peek().kind) == core::mem::discriminant(k) {
@@ -87,7 +115,7 @@ impl Parser {
         matches!(self.tokens[self.current].kind, TokenKind::Eof)
     }
 
-    pub fn parse(&mut self) -> Result<Vec<Stmt>, EldritchError> {
+    pub fn parse(&mut self) -> (Vec<Stmt>, Vec<EldritchError>) {
         let mut statements = Vec::new();
         while !self.is_at_end() {
             while matches!(self.peek().kind, TokenKind::Newline) {
@@ -96,9 +124,22 @@ impl Parser {
             if self.is_at_end() {
                 break;
             }
-            statements.push(self.declaration()?);
+            match self.declaration() {
+                Ok(stmt) => statements.push(stmt),
+                Err(err) => {
+                    self.errors.push(err.clone());
+                    self.synchronize();
+                    // Create an Error stmt to fill the gap
+                    use super::ast::StmtKind;
+                    let span = err.span;
+                    statements.push(Stmt {
+                        kind: StmtKind::Error(err.message),
+                        span,
+                    });
+                }
+            }
         }
-        Ok(statements)
+        (statements, self.errors.clone())
     }
 
     #[allow(clippy::only_used_in_recursion)]