fix(builtins): ARK-350, string:ord need to check it get only one utf8 char ; add string:utf8len

Author: SuperFola

CHANGELOG.md

@@ -12,11 +12,13 @@
 - `append`, `concat`, and `pop` can be used as values
 - new `ptr` command for the debugger, printing the VM pointers (ip, pp, sp)
 - compile time arity check when performing a tail call
+- `string:utf8len` to compute the number of codepoints in a string
 
 ### Changed
 - all paths inside `if` should return a value, when used as an expression. If an `else` branch is missing, `nil` will be returned
 - new compile time error when trying to use `append!`, `concat!`, `pop!`, `@=` and `@@=` as values
 - arguments in tail calls are loaded by value and not by reference
+- `string:ord` checks that it get only 1 utf8 character
 
 ### Removed
 

include/Ark/Builtins/Builtins.hpp

@@ -78,6 +78,7 @@ namespace Ark::internal::Builtins
         ARK_BUILTIN(format);
         ARK_BUILTIN(findSubStr);
         ARK_BUILTIN(removeAtStr);
+        ARK_BUILTIN(utf8len);
         ARK_BUILTIN(ord);
         ARK_BUILTIN(chr);
         ARK_BUILTIN(setStringAt);

include/utf8.hpp

@@ -133,7 +133,7 @@ namespace utf8
     /**
      * @brief Check the validity of a given string in UTF8
      * @param str
-     * @return true if the given string is a valid UTF88 string
+     * @return true if the given string is a valid UTF8 string
      */
     inline bool isValid(const char* str)
     {
@@ -183,46 +183,69 @@ namespace utf8
         return true;
     }
 
+    inline std::size_t length(const char* str)
+    {
+        std::size_t count = 0;
+        const char* s = str;
+
+        if (str == nullptr)
+            return 0;
+
+        while (*s != 0)
+        {
+            if (0xf0 == (0xf8 & *s))
+            {
+                ++count;
+                s += 4;
+            }
+            else if (0xe0 == (0xf0 & *s))
+            {
+                ++count;
+                s += 3;
+            }
+            else if (0xc0 == (0xe0 & *s))
+            {
+                ++count;
+                s += 2;
+            }
+            else if (0x00 == (0x80 & *s))
+            {
+                ++count;
+                s += 1;
+            }
+            else
+                break;
+        }
+
+        return count;
+    }
+
     /**
      * @brief Compute the UTF8 codepoint for a given UTF8 char
      * @param str
      * @return UTF8 codepoint if valid, -1 otherwise
      */
     inline int32_t codepoint(const char* str)
     {
-        int32_t codepoint = 0;
         const char* s = str;
 
         if (isValid(str))
         {
-            while (*s != 0)
-            {
-                if (0xf0 == (0xf8 & *s))
-                {
-                    codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | ((0x3f & s[2]) << 6) | (0x3f & s[3]);
-                    s += 4;
-                }
-                else if (0xe0 == (0xf0 & *s))
-                {
-                    codepoint = ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]);
-                    s += 3;
-                }
-                else if (0xc0 == (0xe0 & *s))
-                {
-                    codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]);
-                    s += 2;
-                }
-                else if (0x00 == (0x80 & *s))
-                {
-                    codepoint = s[0];
-                    ++s;
-                }
-                else
-                    return -1;
-            }
-        }
+            int32_t c = 0;
 
-        return codepoint;
+            if (0xf0 == (0xf8 & *s))
+                c = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | ((0x3f & s[2]) << 6) | (0x3f & s[3]);
+            else if (0xe0 == (0xf0 & *s))
+                c = ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]);
+            else if (0xc0 == (0xe0 & *s))
+                c = ((0x1f & s[0]) << 6) | (0x3f & s[1]);
+            else if (0x00 == (0x80 & *s))
+                c = s[0];
+            else
+                return -1;
+            return c;
+        }
+        return -1;
     }
 
     /**

lib/std

@@ -63,6 +63,7 @@ namespace Ark::internal::Builtins
         { "format", Value(String::format) },
         { "builtin__string:find", Value(String::findSubStr) },
         { "builtin__string:removeAt", Value(String::removeAtStr) },
+        { "builtin__string:utf8len", Value(String::utf8len) },
         { "builtin__string:ord", Value(String::ord) },
         { "builtin__string:chr", Value(String::chr) },
         { "builtin__string:setAt", Value(String::setStringAt) },

src/arkreactor/Builtins/Builtins.cpp

@@ -288,6 +288,18 @@ namespace Ark::internal::Builtins::String
             throw std::runtime_error(fmt::format("string:removeAt: index {} out of range (length: {})", num, n[0].stringRef().size()));
     }
 
+    Value utf8len(std::vector<Value>& n, VM* vm [[maybe_unused]])
+    {
+        if (!types::check(n, ValueType::String))
+            throw types::TypeCheckingError(
+                "string:utf8len",
+                { { types::Contract { { types::Typedef("string", ValueType::String) } } } },
+                n);
+
+        const std::size_t len = utf8::length(n[0].stringRef().c_str());
+        return Value(static_cast<double>(len));
+    }
+
     Value ord(std::vector<Value>& n, VM* vm [[maybe_unused]])
     {
         if (!types::check(n, ValueType::String))
@@ -296,7 +308,13 @@ namespace Ark::internal::Builtins::String
                 { { types::Contract { { types::Typedef("string", ValueType::String) } } } },
                 n);
 
-        return Value(utf8::codepoint(n[0].stringRef().c_str()));
+        if (const std::size_t len = utf8::length(n[0].stringRef().c_str()); len != 1)
+            throw std::runtime_error(fmt::format("string:ord: invalid string '{}', expected a single character, got {}", n[0].string(), len));
+
+        const int32_t codepoint = utf8::codepoint(n[0].stringRef().c_str());
+        if (codepoint == -1)
+            throw std::runtime_error(fmt::format("string:ord: invalid string '{}'", n[0].string()));
+        return Value(codepoint);
     }
 
     // cppcheck-suppress constParameterReference

src/arkreactor/Builtins/String.cpp

@@ -1,6 +1,5 @@
 page_0
 	PUSH_RETURN_ADDRESS L0
-	BUILTIN 69
 	BUILTIN 70
 	BUILTIN 71
 	BUILTIN 72
@@ -25,6 +24,7 @@ page_0
 	BUILTIN 91
 	BUILTIN 92
 	BUILTIN 93
+	BUILTIN 94
 	CALL_BUILTIN 9, 25
 .L0:
 	POP 0

tests/unittests/resources/CompilerSuite/ir/operators_as_builtins.expected

@@ -5,7 +5,7 @@ page_0
 	HALT 0
 
 page_1
-	CALL_BUILTIN_WITHOUT_RETURN_ADDRESS 55, 1
+	CALL_BUILTIN_WITHOUT_RETURN_ADDRESS 56, 1
 .L0:
 	RET 0
 	HALT 0

tests/unittests/resources/CompilerSuite/optimized_ir/builtins.expected

@@ -0,0 +1 @@
+(print (builtin__string:ord ""))

tests/unittests/resources/DiagnosticsSuite/runtime/string_ord_empty_str.ark

@@ -0,0 +1,6 @@
+string:ord: invalid string '', expected a single character, got 0
+
+In file tests/unittests/resources/DiagnosticsSuite/runtime/string_ord_empty_str.ark:1
+    1 | (print (builtin__string:ord ""))
+      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    2 |