Fix \b and \B with PCRE2_EXTRA_ASCII_BSW in the interpreters

Author: PhilipHazel

src/pcre2_compile.c

@@ -4950,6 +4950,8 @@ for (;;)
 
     case OP_WORD_BOUNDARY:
     case OP_NOT_WORD_BOUNDARY:
+    case OP_UCP_WORD_BOUNDARY:
+    case OP_NOT_UCP_WORD_BOUNDARY:
     if (!skipassert) return code;
     /* Fall through */
 
@@ -8032,23 +8034,41 @@ for (;; pptr++)
 
     /* For the rest (including \X when Unicode is supported - if not it's
     faulted at parse time), the OP value is the escape value when PCRE2_UCP is
-    not set; if it is set, these escapes do not show up here because they are
-    converted into Unicode property tests in parse_regex(). Note that \b and \B
-    do a one-character lookbehind, and \A also behaves as if it does. */
+    not set; if it is set, most of them do not show up here because they are
+    converted into Unicode property tests in parse_regex().
 
-    if (meta_arg == ESC_C) cb->external_flags |= PCRE2_HASBKC; /* Record */
-    if ((meta_arg == ESC_b || meta_arg == ESC_B || meta_arg == ESC_A) &&
-         cb->max_lookbehind == 0)
-      cb->max_lookbehind = 1;
+    In non-UTF mode, and for both 32-bit modes, we turn \C into OP_ALLANY
+    instead of OP_ANYBYTE so that it works in DFA mode and in lookbehinds.
+    There are special UCP codes for \B and \b which are used in UCP mode unless
+    "word" matching is being forced to ASCII.
 
-    /* In non-UTF mode, and for both 32-bit modes, we turn \C into OP_ALLANY
-    instead of OP_ANYBYTE so that it works in DFA mode and in lookbehinds. */
+    Note that \b and \B do a one-character lookbehind, and \A also behaves as
+    if it does. */
 
+    switch(meta_arg)
+      {
+      case ESC_C:
+      cb->external_flags |= PCRE2_HASBKC;  /* Record */
 #if PCRE2_CODE_UNIT_WIDTH == 32
-    *code++ = (meta_arg == ESC_C)? OP_ALLANY : meta_arg;
+      meta_arg = OP_ALLANY;
 #else
-    *code++ = (!utf && meta_arg == ESC_C)? OP_ALLANY : meta_arg;
+      if (!utf) meta_arg = OP_ALLANY;
 #endif
+      break;
+
+      case ESC_B:
+      case ESC_b:
+      if ((options & PCRE2_UCP) != 0 && (xoptions & PCRE2_EXTRA_ASCII_BSW) == 0)
+        meta_arg = (meta_arg == ESC_B)? OP_NOT_UCP_WORD_BOUNDARY :
+          OP_UCP_WORD_BOUNDARY;
+      /* Fall through */
+
+      case ESC_A:
+      if (cb->max_lookbehind == 0) cb->max_lookbehind = 1;
+      break;
+      }
+
+    *code++ = meta_arg;
     break;  /* End META_ESCAPE */
 
 

src/pcre2_dfa_match.c

@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
      Original API code Copyright (c) 1997-2012 University of Cambridge
-          New API code Copyright (c) 2016-2022 University of Cambridge
+          New API code Copyright (c) 2016-2023 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -187,7 +187,8 @@ static const uint8_t coptable[] = {
   0, 0, 0, 0,                    /* SKIP, SKIP_ARG, THEN, THEN_ARG         */
   0, 0,                          /* COMMIT, COMMIT_ARG                     */
   0, 0, 0,                       /* FAIL, ACCEPT, ASSERT_ACCEPT            */
-  0, 0, 0                        /* CLOSE, SKIPZERO, DEFINE                */
+  0, 0, 0,                       /* CLOSE, SKIPZERO, DEFINE                */
+  0, 0                           /* \B and \b in UCP mode                  */
 };
 
 /* This table identifies those opcodes that inspect a character. It is used to
@@ -264,7 +265,8 @@ static const uint8_t poptable[] = {
   0, 0, 0, 0,                    /* SKIP, SKIP_ARG, THEN, THEN_ARG         */
   0, 0,                          /* COMMIT, COMMIT_ARG                     */
   0, 0, 0,                       /* FAIL, ACCEPT, ASSERT_ACCEPT            */
-  0, 0, 0                        /* CLOSE, SKIPZERO, DEFINE                */
+  0, 0, 0,                       /* CLOSE, SKIPZERO, DEFINE                */
+  1, 1                           /* \B and \b in UCP mode                  */
 };
 
 /* These 2 tables allow for compact code for testing for \D, \d, \S, \s, \W,
@@ -1100,6 +1102,8 @@ for (;;)
       /*-----------------------------------------------------------------*/
       case OP_WORD_BOUNDARY:
       case OP_NOT_WORD_BOUNDARY:
+      case OP_NOT_UCP_WORD_BOUNDARY:
+      case OP_UCP_WORD_BOUNDARY:
         {
         int left_word, right_word;
 
@@ -1112,7 +1116,8 @@ for (;;)
 #endif
           GETCHARTEST(d, temp);
 #ifdef SUPPORT_UNICODE
-          if ((mb->poptions & PCRE2_UCP) != 0)
+          if (codevalue == OP_UCP_WORD_BOUNDARY ||
+              codevalue == OP_NOT_UCP_WORD_BOUNDARY)
             {
             if (d == '_') left_word = TRUE; else
               {
@@ -1137,7 +1142,8 @@ for (;;)
             mb->last_used_ptr = temp;
             }
 #ifdef SUPPORT_UNICODE
-          if ((mb->poptions & PCRE2_UCP) != 0)
+          if (codevalue == OP_UCP_WORD_BOUNDARY ||
+              codevalue == OP_NOT_UCP_WORD_BOUNDARY)
             {
             if (c == '_') right_word = TRUE; else
               {
@@ -1151,7 +1157,9 @@ for (;;)
           }
         else right_word = FALSE;
 
-        if ((left_word == right_word) == (codevalue == OP_NOT_WORD_BOUNDARY))
+        if ((left_word == right_word) ==
+            (codevalue == OP_NOT_WORD_BOUNDARY ||
+             codevalue == OP_NOT_UCP_WORD_BOUNDARY))
           { ADD_ACTIVE(state_offset + 1, 0); }
         }
       break;

src/pcre2_internal.h

@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
      Original API code Copyright (c) 1997-2012 University of Cambridge
-          New API code Copyright (c) 2016-2022 University of Cambridge
+          New API code Copyright (c) 2016-2023 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -1372,8 +1372,8 @@ enum {
   OP_SOD,            /* 1 Start of data: \A */
   OP_SOM,            /* 2 Start of match (subject + offset): \G */
   OP_SET_SOM,        /* 3 Set start of match (\K) */
-  OP_NOT_WORD_BOUNDARY,  /*  4 \B */
-  OP_WORD_BOUNDARY,      /*  5 \b */
+  OP_NOT_WORD_BOUNDARY,  /*  4 \B -- see also OP_NOT_UCP_WORD_BOUNDARY */
+  OP_WORD_BOUNDARY,      /*  5 \b -- see also OP_UCP_WORD_BOUNDARY */
   OP_NOT_DIGIT,          /*  6 \D */
   OP_DIGIT,              /*  7 \d */
   OP_NOT_WHITESPACE,     /*  8 \S */
@@ -1620,6 +1620,12 @@ enum {
 
   OP_DEFINE,         /* 167 */
 
+  /* These opcodes replace their normal counterparts in UCP mode when
+  PCRE2_EXTRA_ASCII_BSW is not set. */
+
+  OP_NOT_UCP_WORD_BOUNDARY, /* 168 */
+  OP_UCP_WORD_BOUNDARY,     /* 169 */
+
   /* This is not an opcode, but is used to check that tables indexed by opcode
   are the correct length, in order to catch updating errors - there have been
   some in the past. */
@@ -1679,7 +1685,7 @@ some cases doesn't actually use these names at all). */
   "*MARK", "*PRUNE", "*PRUNE", "*SKIP", "*SKIP",                  \
   "*THEN", "*THEN", "*COMMIT", "*COMMIT", "*FAIL",                \
   "*ACCEPT", "*ASSERT_ACCEPT",                                    \
-  "Close", "Skip zero", "Define"
+  "Close", "Skip zero", "Define", "\\B (ucp)", "\\b (ucp)"
 
 
 /* This macro defines the length of fixed length operations in the compiled
@@ -1775,7 +1781,8 @@ in UTF-8 mode. The code that uses this table must know about such things. */
   1, 3,                          /* COMMIT, COMMIT_ARG                     */ \
   1, 1, 1,                       /* FAIL, ACCEPT, ASSERT_ACCEPT            */ \
   1+IMM2_SIZE, 1,                /* CLOSE, SKIPZERO                        */ \
-  1                              /* DEFINE                                 */
+  1,                             /* DEFINE                                 */ \
+  1, 1                           /* \B and \b in UCP mode                  */
 
 /* A magic value for OP_RREF to indicate the "any recursion" condition. */
 

src/pcre2_match.c

@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
      Original API code Copyright (c) 1997-2012 University of Cambridge
-          New API code Copyright (c) 2015-2022 University of Cambridge
+          New API code Copyright (c) 2015-2023 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -6060,6 +6060,8 @@ fprintf(stderr, "++ op=%d\n", *Fecode);
 
     case OP_NOT_WORD_BOUNDARY:
     case OP_WORD_BOUNDARY:
+    case OP_NOT_UCP_WORD_BOUNDARY:
+    case OP_UCP_WORD_BOUNDARY:
     if (Feptr == mb->check_subject) prev_is_word = FALSE; else
       {
       PCRE2_SPTR lastptr = Feptr - 1;
@@ -6074,7 +6076,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode);
       fc = *lastptr;
       if (lastptr < mb->start_used_ptr) mb->start_used_ptr = lastptr;
 #ifdef SUPPORT_UNICODE
-      if ((mb->poptions & PCRE2_UCP) != 0)
+      if (Fop == OP_UCP_WORD_BOUNDARY || Fop == OP_NOT_UCP_WORD_BOUNDARY)
         {
         if (fc == '_') prev_is_word = TRUE; else
           {
@@ -6108,7 +6110,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode);
       fc = *Feptr;
       if (nextptr > mb->last_used_ptr) mb->last_used_ptr = nextptr;
 #ifdef SUPPORT_UNICODE
-      if ((mb->poptions & PCRE2_UCP) != 0)
+      if (Fop == OP_UCP_WORD_BOUNDARY || Fop == OP_NOT_UCP_WORD_BOUNDARY)
         {
         if (fc == '_') cur_is_word = TRUE; else
           {
@@ -6123,7 +6125,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode);
 
     /* Now see if the situation is what we want */
 
-    if ((*Fecode++ == OP_WORD_BOUNDARY)?
+    if ((*Fecode++ == OP_WORD_BOUNDARY || Fop == OP_UCP_WORD_BOUNDARY)?
          cur_is_word == prev_is_word : cur_is_word != prev_is_word)
       RRETURN(MATCH_NOMATCH);
     break;
@@ -6853,7 +6855,7 @@ if (heapframes_size / 1024 > mb->heap_limit)
   if (max_size < frame_size) return PCRE2_ERROR_HEAPLIMIT;
   heapframes_size = max_size;
   }
-  
+
 /* If an existing frame vector in the match_data block is large enough, we can
 use it. Otherwise, free any pre-existing vector and get a new one. */
 

src/pcre2_study.c

@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
      Original API code Copyright (c) 1997-2012 University of Cambridge
-          New API code Copyright (c) 2016-2021 University of Cambridge
+          New API code Copyright (c) 2016-2023 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -273,6 +273,8 @@ for (;;)
     case OP_DOLLM:
     case OP_NOT_WORD_BOUNDARY:
     case OP_WORD_BOUNDARY:
+    case OP_NOT_UCP_WORD_BOUNDARY:
+    case OP_UCP_WORD_BOUNDARY:
     cc += PRIV(OP_lengths)[*cc];
     break;
 
@@ -1101,6 +1103,8 @@ do
 
       case OP_WORD_BOUNDARY:
       case OP_NOT_WORD_BOUNDARY:
+      case OP_UCP_WORD_BOUNDARY:
+      case OP_NOT_UCP_WORD_BOUNDARY:
       tcode++;
       break;
 

testdata/testinput5

@@ -2411,6 +2411,21 @@
 \= Expect no match     
     \x{660}\x{c0}\x{c0}
 
+# WORD BOUNDARY
+
+/\bABC\b/utf
+    \x{c0}ABC\x{d0}
+
+/\bABC\b/utf,ucp
+\= Expect no match
+    \x{c0}ABC\x{d0}
+
+/\bABC\b/utf,ucp,ascii_bsw
+    \x{c0}ABC\x{d0}\=no_jit
+
+/\bABC\b/utf,ucp,ascii_all
+    \x{c0}ABC\x{d0}\=no_jit
+    
 # POSIX 
 
 /[[:digit:]]+/utf,ucp

testdata/testoutput5

@@ -4016,7 +4016,7 @@ MK: a\x{12345}b\x{09}(d)c
 /(*UCP)(*UTF)[[:>:]]X/B
 ------------------------------------------------------------------
         Bra
-        \b
+        \b (ucp)
         Assert back
         Reverse
         prop Xwd
@@ -5334,6 +5334,25 @@ No match
     \x{660}\x{c0}\x{c0}
 No match
 
+# WORD BOUNDARY
+
+/\bABC\b/utf
+    \x{c0}ABC\x{d0}
+ 0: ABC
+
+/\bABC\b/utf,ucp
+\= Expect no match
+    \x{c0}ABC\x{d0}
+No match
+
+/\bABC\b/utf,ucp,ascii_bsw
+    \x{c0}ABC\x{d0}\=no_jit
+ 0: ABC
+
+/\bABC\b/utf,ucp,ascii_all
+    \x{c0}ABC\x{d0}\=no_jit
+ 0: ABC
+    
 # POSIX 
 
 /[[:digit:]]+/utf,ucp