Implement lookaround compilation

These changes implement the compilation of lookaround assertions from
HIR to NFA. Subexpressions of lookaround assertions are patched to
a top level reverse union. This is necessary so that the NFA will
explore the innermost subexpression first and thereby make sure that
all subexpression results are available when they need to be checked.
I.e. any `WriteLookaround` state must be visited before any
`CheckLookaround` state with the same index.

Author: Multimodcrafter

regex-automata/src/nfa/thompson/builder.rs

@@ -41,7 +41,9 @@ enum State {
     },
     /// A state that only transitions to another state if the current input
     /// byte is in a particular range of bytes.
-    ByteRange { trans: Transition },
+    ByteRange {
+        trans: Transition,
+    },
     /// A state with possibly many transitions, represented in a sparse
     /// fashion. Transitions must be ordered lexicographically by input range
     /// and be non-overlapping. As such, this may only be used when every
@@ -55,10 +57,15 @@ enum State {
     /// that `Sparse` is used for via `Union`. But this creates a more bloated
     /// NFA with more epsilon transitions than is necessary in the special case
     /// of character classes.
-    Sparse { transitions: Vec<Transition> },
+    Sparse {
+        transitions: Vec<Transition>,
+    },
     /// A conditional epsilon transition satisfied via some sort of
     /// look-around.
-    Look { look: Look, next: StateID },
+    Look {
+        look: Look,
+        next: StateID,
+    },
     /// An empty state that records the start of a capture location. This is an
     /// unconditional epsilon transition like `Empty`, except it can be used to
     /// record position information for a capture group when using the NFA for
@@ -91,10 +98,20 @@ enum State {
         /// The next state that this state should transition to.
         next: StateID,
     },
+    WriteLookaround {
+        lookaround_index: usize,
+    },
+    CheckLookaround {
+        lookaround_index: usize,
+        positive: bool,
+        next: StateID,
+    },
     /// An alternation such that there exists an epsilon transition to all
     /// states in `alternates`, where matches found via earlier transitions
     /// are preferred over later transitions.
-    Union { alternates: Vec<StateID> },
+    Union {
+        alternates: Vec<StateID>,
+    },
     /// An alternation such that there exists an epsilon transition to all
     /// states in `alternates`, where matches found via later transitions are
     /// preferred over earlier transitions.
@@ -110,7 +127,9 @@ enum State {
     /// to be amortized constant time. But if we used a `Union`, we'd need to
     /// prepend the state, which takes O(n) time. There are other approaches we
     /// could use to solve this, but this seems simple enough.
-    UnionReverse { alternates: Vec<StateID> },
+    UnionReverse {
+        alternates: Vec<StateID>,
+    },
     /// A state that cannot be transitioned out of. This is useful for cases
     /// where you want to prevent matching from occurring. For example, if your
     /// regex parser permits empty character classes, then one could choose a
@@ -124,7 +143,9 @@ enum State {
     ///
     /// `pattern_id` refers to the ID of the pattern itself, which corresponds
     /// to the pattern's index (starting at 0).
-    Match { pattern_id: PatternID },
+    Match {
+        pattern_id: PatternID,
+    },
 }
 
 impl State {
@@ -154,7 +175,9 @@ impl State {
             | State::CaptureStart { .. }
             | State::CaptureEnd { .. }
             | State::Fail
-            | State::Match { .. } => 0,
+            | State::Match { .. }
+            | State::CheckLookaround { .. }
+            | State::WriteLookaround { .. } => 0,
             State::Sparse { ref transitions } => {
                 transitions.len() * mem::size_of::<Transition>()
             }
@@ -470,6 +493,22 @@ impl Builder {
                 State::Look { look, next } => {
                     remap[sid] = nfa.add(nfa::State::Look { look, next });
                 }
+                State::WriteLookaround { lookaround_index } => {
+                    remap[sid] = nfa.add(nfa::State::WriteLookaround {
+                        look_idx: lookaround_index,
+                    });
+                }
+                State::CheckLookaround {
+                    lookaround_index,
+                    positive,
+                    next,
+                } => {
+                    remap[sid] = nfa.add(nfa::State::CheckLookaround {
+                        look_idx: lookaround_index,
+                        positive,
+                        next,
+                    });
+                }
                 State::CaptureStart { pattern_id, group_index, next } => {
                     // We can't remove this empty state because of the side
                     // effect of capturing an offset for this capture slot.
@@ -693,6 +732,30 @@ impl Builder {
         self.add(State::Empty { next: StateID::ZERO })
     }
 
+    /// Add a state which will record that the lookaround with the given index
+    /// is satisfied at the current position.
+    pub fn add_write_lookaround(
+        &mut self,
+        index: usize,
+    ) -> Result<StateID, BuildError> {
+        self.add(State::WriteLookaround { lookaround_index: index })
+    }
+
+    /// Add a state which will check whether the lookaround with the given
+    /// index is satisfied at the current position.
+    pub fn add_check_lookaround(
+        &mut self,
+        index: usize,
+        positive: bool,
+        next: StateID,
+    ) -> Result<StateID, BuildError> {
+        self.add(State::CheckLookaround {
+            lookaround_index: index,
+            positive,
+            next,
+        })
+    }
+
     /// Add a "union" NFA state.
     ///
     /// A "union" NFA state that contains zero or more unconditional epsilon
@@ -1159,6 +1222,9 @@ impl Builder {
             State::Look { ref mut next, .. } => {
                 *next = to;
             }
+            State::CheckLookaround { ref mut next, .. } => {
+                *next = to;
+            }
             State::Union { ref mut alternates } => {
                 alternates.push(to);
                 self.memory_states += mem::size_of::<StateID>();
@@ -1173,6 +1239,7 @@ impl Builder {
             State::CaptureEnd { ref mut next, .. } => {
                 *next = to;
             }
+            State::WriteLookaround { .. } => {}
             State::Fail => {}
             State::Match { .. } => {}
         }

regex-automata/src/nfa/thompson/compiler.rs

@@ -3,7 +3,7 @@ use core::{borrow::Borrow, cell::RefCell};
 use alloc::{sync::Arc, vec, vec::Vec};
 
 use regex_syntax::{
-    hir::{self, Hir},
+    hir::{self, Hir, LookAround},
     utf8::{Utf8Range, Utf8Sequences},
     ParserBuilder,
 };
@@ -711,6 +711,7 @@ pub struct Compiler {
     /// State used for caching common suffixes when compiling reverse UTF-8
     /// automata (for Unicode character classes).
     utf8_suffix: RefCell<Utf8SuffixMap>,
+    lookaround_alt: RefCell<Option<StateID>>,
 }
 
 impl Compiler {
@@ -723,6 +724,7 @@ impl Compiler {
             utf8_state: RefCell::new(Utf8State::new()),
             trie_state: RefCell::new(RangeTrie::new()),
             utf8_suffix: RefCell::new(Utf8SuffixMap::new(1000)),
+            lookaround_alt: RefCell::new(None),
         }
     }
 
@@ -977,11 +979,20 @@ impl Compiler {
 
         let compiled = self.c_alt_iter(exprs.iter().map(|e| {
             let _ = self.start_pattern()?;
+            let lookaround_prefix =
+                self.c_at_least(&Hir::dot(hir::Dot::AnyByte), false, 0)?;
+            let lookaround_alt = self.add_union_reverse()?;
+            self.patch(lookaround_prefix.end, lookaround_alt)?;
+            let top_level_alt = self.add_union()?;
+            self.patch(top_level_alt, lookaround_prefix.start)?;
+            self.lookaround_alt.borrow_mut().replace(lookaround_alt);
             let one = self.c_cap(0, None, e.borrow())?;
             let match_state_id = self.add_match()?;
             self.patch(one.end, match_state_id)?;
-            let _ = self.finish_pattern(one.start)?;
-            Ok(ThompsonRef { start: one.start, end: match_state_id })
+            self.patch(top_level_alt, one.start)?;
+            let _ = self.finish_pattern(top_level_alt)?;
+            self.lookaround_alt.borrow_mut().take();
+            Ok(ThompsonRef { start: top_level_alt, end: match_state_id })
         }))?;
         self.patch(unanchored_prefix.end, compiled.start)?;
         let nfa = self
@@ -1003,14 +1014,39 @@ impl Compiler {
             Class(Class::Bytes(ref c)) => self.c_byte_class(c),
             Class(Class::Unicode(ref c)) => self.c_unicode_class(c),
             Look(ref look) => self.c_look(look),
-            LookAround(_) => todo!("implement lookaround NFA compilation"),
+            LookAround(ref lookaround) => self.c_lookaround(lookaround),
             Repetition(ref rep) => self.c_repetition(rep),
             Capture(ref c) => self.c_cap(c.index, c.name.as_deref(), &c.sub),
             Concat(ref es) => self.c_concat(es.iter().map(|e| self.c(e))),
             Alternation(ref es) => self.c_alt_slice(es),
         }
     }
 
+    fn c_lookaround(
+        &self,
+        lookaround: &LookAround,
+    ) -> Result<ThompsonRef, BuildError> {
+        let sub = match lookaround {
+            LookAround::NegativeLookBehind(ref sub)
+            | LookAround::PositiveLookBehind(ref sub) => self.c(sub)?,
+        };
+        let pos = match lookaround {
+            LookAround::NegativeLookBehind(_) => false,
+            LookAround::PositiveLookBehind(_) => true,
+        };
+        let idx = todo!("get index");
+        let check = self.add_check_lookaround(idx, pos)?;
+        let write = self.add_write_lookaround(idx)?;
+        self.patch(sub.end, write)?;
+        self.patch(
+            self.lookaround_alt
+                .borrow()
+                .expect("Cannot compile lookaround outside pattern"),
+            sub.start,
+        )?;
+        Ok(ThompsonRef { start: check, end: check })
+    }
+
     /// Compile a concatenation of the sub-expressions yielded by the given
     /// iterator. If the iterator yields no elements, then this compiles down
     /// to an "empty" state that always matches.
@@ -1631,6 +1667,25 @@ impl Compiler {
         self.builder.borrow_mut().add_empty()
     }
 
+    fn add_write_lookaround(
+        &self,
+        index: usize,
+    ) -> Result<StateID, BuildError> {
+        self.builder.borrow_mut().add_write_lookaround(index)
+    }
+
+    fn add_check_lookaround(
+        &self,
+        index: usize,
+        positive: bool,
+    ) -> Result<StateID, BuildError> {
+        self.builder.borrow_mut().add_check_lookaround(
+            index,
+            positive,
+            StateID::ZERO,
+        )
+    }
+
     fn add_range(&self, start: u8, end: u8) -> Result<StateID, BuildError> {
         self.builder.borrow_mut().add_range(Transition {
             start,

regex-cli/cmd/generate/fowler.rs

@@ -421,5 +421,6 @@ fn count_capturing_groups_ast(ast: &regex_syntax::ast::Ast) -> usize {
         Ast::Concat(ref concat) => {
             concat.asts.iter().map(count_capturing_groups_ast).sum()
         }
+        Ast::LookAround(_) => todo!(),
     }
 }