Merge pull request #3807 from secondlife/marchcat/lua_editor

SLua editor: multi-line strings and comments

Author: marchcat

indra/llui/llkeywords.cpp

@@ -28,6 +28,7 @@
 
 #include <iostream>
 #include <fstream>
+#include <regex>
 
 #include "llkeywords.h"
 #include "llsdserialize.h"
@@ -80,6 +81,8 @@ LLKeywords::~LLKeywords()
     mLineTokenList.clear();
     std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer());
     mDelimiterTokenList.clear();
+    std::for_each(mRegexTokenList.begin(), mRegexTokenList.end(), DeletePointer());
+    mRegexTokenList.clear();
 }
 
 // Add the token as described
@@ -123,6 +126,10 @@ void LLKeywords::addToken(LLKeywordToken::ETokenType type,
         mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
         break;
 
+    case LLKeywordToken::TT_REGEX_MATCH:
+        mRegexTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
+        break;
+
     default:
         llassert(0);
     }
@@ -235,25 +242,27 @@ void LLKeywords::processTokens()
 
     // Add 'standard' stuff: Quotes, Comments, Strings, Labels, etc. before processing the LLSD
     std::string delimiter;
-    addToken(LLKeywordToken::TT_LABEL, "@", getColorGroup("misc-flow-label"), "Label\nTarget for jump statement", delimiter );
-    addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "\"", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "\"" );
+    addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "\"", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "\"");
 
     if (mLuauLanguage)
     {
         // Add Lua-style comments
         addToken(LLKeywordToken::TT_ONE_SIDED_DELIMITER, "--", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style single-line)\nNon-functional commentary or disabled code", delimiter);
-        addToken(LLKeywordToken::TT_TWO_SIDED_DELIMITER, "--[[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", "]]");
+        // Add Lua multi-line comments
+        addToken(LLKeywordToken::TT_REGEX_MATCH, "^--\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", "\\]\\1\\]");
+        // Add Lua multi-line strings
+        addToken(LLKeywordToken::TT_REGEX_MATCH, "^\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal (Lua-style multi-line)", "\\]\\1\\]");
     }
     else
     {
+        addToken(LLKeywordToken::TT_LABEL, "@", getColorGroup("misc-flow-label"), "Label\nTarget for jump statement", delimiter);
         // Add LSL-style comments
         addToken(LLKeywordToken::TT_ONE_SIDED_DELIMITER, "//", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (single-line)\nNon-functional commentary or disabled code", delimiter);
         addToken(LLKeywordToken::TT_TWO_SIDED_DELIMITER, "/*", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (multi-line)\nNon-functional commentary or disabled code", "*/");
     }
 
-
     LLSD::map_iterator itr = mSyntax.beginMap();
-    for ( ; itr != mSyntax.endMap(); ++itr)
+    for (; itr != mSyntax.endMap(); ++itr)
     {
         if (itr->first == "llsd-lsl-syntax-version")
         {
@@ -578,7 +587,121 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 
         while( *cur && *cur != '\n' )
         {
-            // Check against delimiters
+            // Check for regex matches first
+            bool regex_matched = false;
+            if (!mRegexTokenList.empty())
+            {
+                S32 seg_start = (S32)(cur - base);
+
+                std::string text_to_search(wtext.begin() + seg_start, wtext.end());
+
+                for (token_list_t::iterator iter = mRegexTokenList.begin();
+                    iter != mRegexTokenList.end(); ++iter)
+                {
+                    LLKeywordToken* regex_token = *iter;
+                    std::string start_pattern(regex_token->getToken().begin(), regex_token->getToken().end());
+                    std::string end_pattern(regex_token->getDelimiter().begin(), regex_token->getDelimiter().end());
+
+                    try
+                    {
+                        std::regex start_regex_pattern(start_pattern);
+                        std::smatch start_match;
+
+                        if (std::regex_search(text_to_search, start_match, start_regex_pattern) && !start_match.empty())
+                        {
+                            if (start_match.position() == 0) // Match starts at current position
+                            {
+                                // Calculate segment boundaries for start pattern
+                                S32 start_match_length = static_cast<S32>(start_match.str().length());
+                                S32 start_seg_end = seg_start + start_match_length;
+
+                                if (end_pattern.empty())
+                                {
+                                    // If no end pattern is provided, treat the entire regex match as a single segment
+                                    // Move cursor past the matched segment
+                                    cur = base + start_seg_end;
+
+                                    // Insert the matched segment
+                                    insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor);
+                                }
+                                else
+                                {
+                                    // Look for the end pattern after the start pattern
+                                    std::string remaining_text = text_to_search.substr(start_match_length);
+
+                                    // Process end pattern - replace any capture group references
+                                    std::string actual_end_pattern = end_pattern;
+
+                                    // Handle capture groups in the end pattern (replace \1, \2, etc. with their matched content)
+                                    for (size_t i = 1; i < start_match.size(); ++i)
+                                    {
+                                        std::string capture = start_match[i].str();
+                                        std::string placeholder = "\\" + std::to_string(i);
+
+                                        // Replace all occurrences of the placeholder with the captured content
+                                        size_t pos = 0;
+                                        while ((pos = actual_end_pattern.find(placeholder, pos)) != std::string::npos)
+                                        {
+                                            actual_end_pattern.replace(pos, placeholder.length(), capture);
+                                            pos += capture.length();
+                                        }
+                                    }
+
+                                    try
+                                    {
+                                        std::regex end_regex_pattern(actual_end_pattern);
+                                        std::smatch end_match;
+
+                                        S32 seg_end = start_seg_end;
+
+                                        if (std::regex_search(remaining_text, end_match, end_regex_pattern) && !end_match.empty())
+                                        {
+                                            // Calculate position of end match relative to the original text
+                                            S32 end_match_position = static_cast<S32>(end_match.position());
+                                            S32 end_match_length = static_cast<S32>(end_match.str().length());
+
+                                            // Calculate the total length including both patterns and text between
+                                            seg_end += end_match_position + end_match_length;
+                                        }
+                                        else
+                                        {
+                                            // End pattern not found, treat everything up to EOF as the segment
+                                            seg_end += static_cast<S32>(remaining_text.length());
+                                        }
+
+                                        // Move cursor past the entire matched segment (start + content + end)
+                                        cur = base + seg_end;
+
+                                        // Insert the matched segment
+                                        insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, seg_end, style, editor);
+                                    }
+                                    catch (const std::regex_error& e)
+                                    {
+                                        LL_WARNS() << "Regex error in end pattern: " << e.what() << " in pattern: " << actual_end_pattern << LL_ENDL;
+                                        // Fall back to treating the start match as the entire segment
+                                        cur = base + start_seg_end;
+                                        insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor);
+                                    }
+                                }
+
+                                regex_matched = true;
+                                break;
+                            }
+                        }
+                    }
+                    catch (const std::regex_error& e)
+                    {
+                        LL_WARNS() << "Regex error in start pattern: " << e.what() << " in pattern: " << start_pattern << LL_ENDL;
+                    }
+                }
+
+                if (regex_matched)
+                {
+                    continue;
+                }
+            }
+
+            // If no regex match, check against delimiters
             {
                 S32 seg_start = 0;
                 LLKeywordToken* cur_delimiter = NULL;
@@ -602,7 +725,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                     cur += cur_delimiter->getLengthHead();
 
                     LLKeywordToken::ETokenType type = cur_delimiter->getType();
-                    if( type == LLKeywordToken::TT_TWO_SIDED_DELIMITER || type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS )
+                    if(type == LLKeywordToken::TT_TWO_SIDED_DELIMITER || type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS)
                     {
                         while( *cur && !cur_delimiter->isTail(cur))
                         {
@@ -664,12 +787,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                         seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
                     }
 
-                    insertSegments(wtext, *seg_list,cur_delimiter, text_len, seg_start, seg_end, style, editor);
-                    /*
-                    LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_delimiter->getColor(), seg_start, seg_end, editor );
-                    text_segment->setToken( cur_delimiter );
-                    insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
-                    */
+                    insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
                     // Note: we don't increment cur, since the end of one delimited seg may be immediately
                     // followed by the start of another one.
                     continue;
@@ -866,6 +984,14 @@ void LLKeywords::dump()
         LLKeywordToken* delimiter_token = *iter;
         delimiter_token->dump();
     }
+
+    LL_INFOS() << "LLKeywords::sRegexTokenList" << LL_ENDL;
+    for (token_list_t::iterator iter = mRegexTokenList.begin();
+         iter != mRegexTokenList.end(); ++iter)
+    {
+        LLKeywordToken* regex_token = *iter;
+        regex_token->dump();
+    }
 }
 
 void LLKeywordToken::dump()

indra/llui/llkeywords.h

@@ -53,6 +53,11 @@ class LLKeywordToken
      * - TT_ONE_SIDED_DELIMITER are for open-ended delimiters which are terminated by EOL.
      * - TT_TWO_SIDED_DELIMITER are for delimiters that end with a different delimiter than they open with.
      * - TT_DOUBLE_QUOTATION_MARKS are for delimiting areas using the same delimiter to open and close.
+     * - TT_REGEX_MATCH are for pattern-based matching using regular expressions.
+     *      For TT_REGEX_MATCH: mToken contains the start pattern, mDelimiter contains the end pattern (if any).
+     *      If mDelimiter is empty, the entire match is considered one segment.
+     *      If mDelimiter contains capture group references (e.g. \1, \2), these will be replaced with
+     *      the corresponding capture groups from the start pattern match.
      */
     typedef enum e_token_type
     {
@@ -62,6 +67,7 @@ class LLKeywordToken
         TT_TWO_SIDED_DELIMITER,
         TT_ONE_SIDED_DELIMITER,
         TT_DOUBLE_QUOTATION_MARKS,
+        TT_REGEX_MATCH,
         // Following constants are more specific versions of the preceding ones
         TT_CONSTANT,                        // WORD
         TT_CONTROL,                         // WORD
@@ -194,6 +200,7 @@ class LLKeywords
     typedef std::deque<LLKeywordToken*> token_list_t;
     token_list_t mLineTokenList;
     token_list_t mDelimiterTokenList;
+    token_list_t mRegexTokenList;
 
     typedef  std::map<std::string, std::string, std::less<>> element_attributes_t;
     typedef element_attributes_t::const_iterator attribute_iterator_t;

indra/newview/app_settings/keywords_lua_default.xml

@@ -104,47 +104,42 @@
          <key>boolean</key>
          <map>
             <key>tooltip</key>
-            <string>Lua boolean: represents a true or false value.</string>
+            <string>Boolean: represents a true or false value.</string>
          </map>
-         <key>float</key>
+         <key>number</key>
          <map>
             <key>tooltip</key>
-            <string>32 bit floating point value.\nThe range is 1.175494351E-38 to 3.402823466E+38.</string>
+            <string>Double‑precision floating point number.</string>
          </map>
-         <key>integer</key>
-         <map>
-            <key>tooltip</key>
-            <string>32 bit integer value.\n−2,147,483,648 and +2,147,483,647 (that is 0x80000000 to 0x7FFFFFFF in hex).</string>
-         </map>
-         <key>key</key>
+         <key>string</key>
          <map>
             <key>tooltip</key>
-            <string>A 128 bit unique identifier (UUID).\nThe key is represented as hexidecimal characters (A-F and 0-9), grouped into sections (8,4,4,4,12 characters) and separated by hyphens (for a total of 36 characters). e.g. "A822FF2B-FF02-461D-B45D-DCD10A2DE0C2".</string>
+            <string>Text data (UTF‑8).</string>
          </map>
-         <key>list</key>
+         <key>table</key>
          <map>
             <key>tooltip</key>
-            <string>A collection of other data types.\nLists are signified by square brackets surrounding their elements; the elements inside are separated by commas. e.g. [0, 1, 2, 3, 4] or ["Yes", "No", "Perhaps"].</string>
+            <string>Collection of key‑value pairs.</string>
          </map>
-         <key>quaternion</key>
+         <key>thread</key>
          <map>
             <key>tooltip</key>
-            <string>The quaternion type is a left over from way back when LSL was created. It was later renamed to &lt;rotation&gt; to make it more user friendly, but it appears someone forgot to remove it ;-)</string>
+            <string>Represents a coroutine.</string>
          </map>
-         <key>rotation</key>
+         <key>userdata</key>
          <map>
             <key>tooltip</key>
-            <string>The rotation type is one of several ways to represent an orientation in 3D.\nIt is a mathematical object called a quaternion. You can think of a quaternion as four numbers (x, y, z, w), three of which represent the direction an object is facing and a fourth that represents the object's banking left or right around that direction.</string>
+            <string>Opaque external data.</string>
          </map>
-         <key>string</key>
+         <key>vector</key>
          <map>
             <key>tooltip</key>
-            <string>Text data.\nThe editor accepts UTF-8 encoded text.</string>
+            <string>A vector is a data type that contains a set of three float values.\nVectors are used to represent colors (RGB), positions, and directions/velocities.</string>
          </map>
-         <key>vector</key>
+         <key>uuid</key>
          <map>
             <key>tooltip</key>
-            <string>A vector is a data type that contains a set of three float values.\nVectors are used to represent colors (RGB), positions, and directions/velocities.</string>
+            <string>A 128‑bit unique identifier formatted as 36 hexadecimal characters (8‑4‑4‑4‑12), e.g. "A822FF2B-FF02-461D-B45D-DCD10A2DE0C2".</string>
          </map>
       </map>
       <key>constants</key>

indra/newview/skins/default/xui/de/floater_about.xml

@@ -23,6 +23,8 @@ mit Open-Source-Beiträgen von:</text>
         Havok.com(TM) Copyright (C) 1999-2001, Telekinesys Research Limited.
         jpeg2000 Copyright (C) 2001, David Taubman, The University of New South Wales (UNSW).
         jpeglib Copyright (C) 1991-1998, Thomas G. Lane.
+        luau Copyright (c) 1994–2019 Lua.org, PUC-Rio / Copyright (c) 2019-2024 Roblox Corporation
+        meshoptimizer Copyright (c) 2016-2021 Arseny Kapoulkine
         ogg/vorbis Copyright (C) 2002, Xiphophorus.
         OpenSSL Copyright (C) 1998-2008 The OpenSSL Project.
         PCRE Copyright (c) 1997-2012 University of Cambridge.

indra/newview/skins/default/xui/en/floater_about.xml

@@ -105,6 +105,7 @@ Dummy Name replaced at run time
         Havok.com(TM) Copyright (C) 1999-2001, Telekinesys Research Limited.
         jpeg2000 Copyright (C) 2001, David Taubman, The University of New South Wales (UNSW)
         jpeglib Copyright (C) 1991-1998, Thomas G. Lane.
+        luau Copyright (c) 1994–2019 Lua.org, PUC-Rio / Copyright (c) 2019-2024 Roblox Corporation
         meshoptimizer Copyright (c) 2016-2021 Arseny Kapoulkine
         ogg/vorbis Copyright (C) 2002, Xiphophorus
         OpenSSL Copyright (C) 1998-2008 The OpenSSL Project.

indra/newview/skins/default/xui/es/floater_about.xml

@@ -23,6 +23,8 @@ con contribuciones de código abierto de:</text>
         Havok.com(TM) Copyright (C) 1999-2001, Telekinesys Research Limited.
         jpeg2000 Copyright (C) 2001, David Taubman, The University of New South Wales (UNSW)
         jpeglib Copyright (C) 1991-1998, Thomas G. Lane.
+        luau Copyright (c) 1994–2019 Lua.org, PUC-Rio / Copyright (c) 2019-2024 Roblox Corporation
+        meshoptimizer Copyright (c) 2016-2021 Arseny Kapoulkine
         ogg/vorbis Copyright (C) 2002, Xiphophorus
         OpenSSL Copyright (C) 1998-2008 The OpenSSL Project.
         PCRE Copyright (c) 1997-2012 University of Cambridge

indra/newview/skins/default/xui/fr/floater_about.xml

@@ -23,6 +23,8 @@ avec les contributions Open Source de :</text>
         Havok.com(TM) Copyright (C) 1999-2001, Telekinesys Research Limited.
         jpeg2000 Copyright (C) 2001, David Taubman, The University of New South Wales (UNSW)
         jpeglib Copyright (C) 1991-1998, Thomas G. Lane.
+        luau Copyright (c) 1994–2019 Lua.org, PUC-Rio / Copyright (c) 2019-2024 Roblox Corporation
+        meshoptimizer Copyright (c) 2016-2021 Arseny Kapoulkine
         ogg/vorbis Copyright (C) 2002, Xiphophorus
         OpenSSL Copyright (C) 1998-2008 The OpenSSL Project.
         PCRE Copyright (c) 1997-2012 University of Cambridge

indra/newview/skins/default/xui/it/floater_about.xml

@@ -23,6 +23,8 @@ con contributi open source da:</text>
         Havok.com(TM) Copyright (C) 1999-2001, Telekinesys Research Limited.
         jpeg2000 Copyright (C) 2001, David Taubman, The University of New South Wales (UNSW)
         jpeglib Copyright (C) 1991-1998, Thomas G. Lane.
+        luau Copyright (c) 1994–2019 Lua.org, PUC-Rio / Copyright (c) 2019-2024 Roblox Corporation
+        meshoptimizer Copyright (c) 2016-2021 Arseny Kapoulkine
         ogg/vorbis Copyright (C) 2002, Xiphophorus
         OpenSSL Copyright (C) 1998-2008 The OpenSSL Project.
         PCRE Copyright (c) 1997-2012 University of Cambridge