Feat/unicode

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Change Log
 
+## [Unreleased changes] - 2026-MM-DD
+### Breaking changes
+
+### Deprecations
+- `std.Range` has been entirely deprecated and is scheduled for removal
+
+### Added
+- standard library:
+  - list:slice1
+  - list:first
+  - list:last
+  - list:cumulativeSum
+  - list:cumulativeProduct
+  - list:zeros
+  - list:ones
+  - string:lpad
+  - string:rpad
+  - string:ascii?
+  - string:first
+  - string:last
+  - string:codepoints
+- new builtin `builtin__string:codepoints` returning the Unicode codepoints of a string as a list of numbers
+
+### Changed
+- `$repr` shows the correct representation for macros
+- `$symcat` accepts symbols and strings as its first argument
+- `math:pow` uses a more precise way of computing values when inputs are integers
+
+### Removed
+
 ## [4.5.2] - 2026-05-06
 ### Changed
 - fix compiler segfault when there is no variable to mark as unreachable

include/Ark/Builtins/Builtins.hpp

@@ -38,7 +38,6 @@ namespace Ark::internal::Builtins
     {
         ARK_BUILTIN(reverseList);
         ARK_BUILTIN(findInList);
-        ARK_BUILTIN(sliceList);
         ARK_BUILTIN(sort_);
         ARK_BUILTIN(fill);
         ARK_BUILTIN(setListAt);
@@ -82,6 +81,7 @@ namespace Ark::internal::Builtins
         ARK_BUILTIN(ord);
         ARK_BUILTIN(chr);
         ARK_BUILTIN(setStringAt);
+        ARK_BUILTIN(codepoints);
     }
 
     namespace Mathematics

src/arkreactor/Builtins/Builtins.cpp

@@ -30,7 +30,7 @@ namespace Ark::internal::Builtins
         // List
         { "builtin__list:reverse", Value(List::reverseList) },
         { "builtin__list:find", Value(List::findInList) },
-        { "builtin__list:slice", Value(List::sliceList) },
+        { "builtin__list:slice", Value(slice) },
         { "builtin__list:sort", Value(List::sort_) },
         { "builtin__list:fill", Value(List::fill) },
         { "builtin__list:setAt", Value(List::setListAt) },
@@ -67,6 +67,7 @@ namespace Ark::internal::Builtins
         { "builtin__string:ord", Value(String::ord) },
         { "builtin__string:chr", Value(String::chr) },
         { "builtin__string:setAt", Value(String::setStringAt) },
+        { "builtin__string:codepoints", Value(String::codepoints) },
 
         // Mathematics
         { "builtin__math:exp", Value(Mathematics::exponential) },

src/arkreactor/Builtins/List.cpp

@@ -33,38 +33,6 @@ namespace Ark::internal::Builtins::List
         return Value(-1);
     }
 
-    Value sliceList(std::vector<Value>& n, VM* vm [[maybe_unused]])
-    {
-        if (!types::check(n, ValueType::List, ValueType::Number, ValueType::Number, ValueType::Number))
-            throw types::TypeCheckingError(
-                "list:slice",
-                { { types::Contract { { types::Typedef("list", ValueType::List),
-                                        types::Typedef("start", ValueType::Number),
-                                        types::Typedef("end", ValueType::Number),
-                                        types::Typedef("step", ValueType::Number) } } } },
-                n);
-
-        const long step = static_cast<long>(n[3].number());
-        if (step <= 0)
-            throw std::runtime_error("list:slice: step can not be null or negative");
-
-        auto start = static_cast<long>(n[1].number());
-        auto end = static_cast<long>(n[2].number());
-
-        if (start > end)
-            throw std::runtime_error(fmt::format("list:slice: start position ({}) must be less or equal to the end position ({})", start, end));
-        if (start < 0)
-            throw std::runtime_error(fmt::format("list:slice: start index {} can not be less than 0", start));
-        if (std::cmp_greater(end, n[0].list().size()))
-            throw std::runtime_error(fmt::format("list:slice: end index {} out of range (length: {})", end, n[0].list().size()));
-
-        std::vector<Value> list;
-        for (auto i = static_cast<std::size_t>(start); std::cmp_less(i, end); i += static_cast<std::size_t>(step))
-            list.push_back(n[0].list()[i]);
-
-        return Value(std::move(list));
-    }
-
     Value sort_(std::vector<Value>& n, VM* vm [[maybe_unused]])
     {
         if (!types::check(n, ValueType::List))

src/arkreactor/Builtins/String.cpp

@@ -12,6 +12,7 @@
 
 #include <Ark/TypeChecker.hpp>
 #include <Ark/VM/VM.hpp>
+#include <Ark/Compiler/AST/utf8_char.hpp>
 
 struct value_wrapper
 {
@@ -353,4 +354,25 @@ namespace Ark::internal::Builtins::String
         string[static_cast<std::size_t>(idx)] = n[2].string()[0];
         return n[0];
     }
+
+    Value codepoints(std::vector<Value>& n, VM* vm [[maybe_unused]])
+    {
+        if (!types::check(n, ValueType::String))
+            throw types::TypeCheckingError(
+                "string:codepoints",
+                { { types::Contract { { types::Typedef("string", ValueType::String) } } } },
+                n);
+
+        Value data(ValueType::List);
+        auto it = n[0].stringRef().begin();
+        const auto end = n[0].stringRef().end();
+        while (it != end)
+        {
+            auto [next, sym] = utf8_char_t::at(it, end);
+            data.push_back(Value(sym.codepoint()));
+            it = next;
+        }
+
+        return data;
+    }
 }

src/arkreactor/Compiler/Macros/Processor.cpp

@@ -528,7 +528,8 @@ namespace Ark::internal
             {
                 if (node.list().size() <= 2)
                     throwMacroProcessingError(fmt::format("When expanding `{}', expected at least 2 arguments, got {} arguments", Language::Symcat, argcount), node);
-                checkMacroTypeError(Language::Symcat.data(), "symbol", NodeType::Symbol, node.list()[1]);
+                if (node.list()[1].nodeType() != NodeType::Symbol && node.list()[1].nodeType() != NodeType::String)
+                    checkMacroTypeError(Language::Symcat.data(), "symbol", NodeType::Symbol, node.list()[1]);
 
                 std::string sym = node.list()[1].string();
 
@@ -584,7 +585,15 @@ namespace Ark::internal
                 checkMacroArgCountEq(node, 1, Language::Repr.data(), true);
 
                 const Node arg = node.constList()[1];
-                node.updateValueAndType(Node(NodeType::String, arg.repr()));
+                if (arg.nodeType() == NodeType::Symbol)
+                {
+                    if (const Node* arg_as_macro = findNearestMacro(arg.string()); arg_as_macro != nullptr)
+                        node.updateValueAndType(Node(NodeType::String, arg_as_macro->repr()));
+                    else
+                        node.updateValueAndType(Node(NodeType::String, arg.repr()));
+                }
+                else
+                    node.updateValueAndType(Node(NodeType::String, arg.repr()));
             }
             else if (name == Language::AsIs)
             {

tests/unittests/resources/CompilerSuite/ir/operators_as_builtins.expected

@@ -1,6 +1,5 @@
 page_0
 	PUSH_RETURN_ADDRESS L0
-	BUILTIN 70
 	BUILTIN 71
 	BUILTIN 72
 	BUILTIN 73
@@ -25,6 +24,7 @@ page_0
 	BUILTIN 92
 	BUILTIN 93
 	BUILTIN 94
+	BUILTIN 95
 	CALL_BUILTIN 9, 25
 .L0:
 	POP 0

tests/unittests/resources/CompilerSuite/optimized_ir/builtins.expected

@@ -5,7 +5,7 @@ page_0
 	HALT 0
 
 page_1
-	CALL_BUILTIN_WITHOUT_RETURN_ADDRESS 56, 1
+	CALL_BUILTIN_WITHOUT_RETURN_ADDRESS 57, 1
 .L0:
 	RET 0
 	HALT 0

tests/unittests/resources/DiagnosticsSuite/runtime/backtrace_builtin.expected

@@ -6,16 +6,16 @@ Signature
 Arguments
   → `list' (expected List), got "live" (String)
 
-In file /lib/std/List.ark:51
-   48 | # (list:sort [4 2 3])  # [1 2 4]
-   49 | # =end
-   50 | # @author https://github.com/SuperFola
-   51 | (let sort (fun (_L) (builtin__list:sort _L)))
+In file /lib/std/List.ark:63
+   60 | # (list:sort [4 2 3])  # [1 2 4]
+   61 | # =end
+   62 | # @author https://github.com/SuperFola
+   63 | (let sort (fun (_L) (builtin__list:sort _L)))
       | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-   52 |
-   53 | # @brief Generate a List of n copies of an element
+   64 |
+   65 | # @brief Generate a List of n copies of an element
 
-[   2] In function `list:sort' (/lib/std/List.ark:51)
+[   2] In function `list:sort' (/lib/std/List.ark:63)
 [   1] In global scope (tests/unittests/resources/DiagnosticsSuite/runtime/backtrace_builtin.ark:3)
 
-Current scope variables values:
+Current scope variables values:

tests/unittests/resources/DiagnosticsSuite/typeChecking/builtin_codepoints_num.ark

@@ -0,0 +1 @@
+(builtin__string:codepoints 1)

tests/unittests/resources/DiagnosticsSuite/typeChecking/builtin_codepoints_num.expected

@@ -0,0 +1,12 @@
+Function string:codepoints expected 1 argument
+Call
+  ↳ (string:codepoints 1)
+Signature
+  ↳ (string:codepoints string)
+Arguments
+  → `string' (expected String), got 1 (Number)
+
+In file tests/unittests/resources/DiagnosticsSuite/typeChecking/builtin_codepoints_num.ark:1
+    1 | (builtin__string:codepoints 1)
+      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    2 |

tests/unittests/resources/DiagnosticsSuite/typeChecking/listslice_str_num_bool_nil.expected

@@ -1,10 +1,19 @@
-Function list:slice expected 4 arguments
+Function slice expected between 3 arguments and 4 arguments
 Call
-  ↳ (list:slice "hello" 1 true nil)
+  ↳ (slice "hello" 1 true nil)
 Signature
-  ↳ (list:slice list start end step)
+  ↳ (slice container start end)
 Arguments
-  → `list' (expected List), got "hello" (String)
+  → `container' (expected List, or String) ✓
+  → `start' (expected Number) ✓
+  → `end' (expected Number), got true (Bool)
+  → unexpected additional args: nil (Nil)
+
+Alternative 2:
+Signature
+  ↳ (slice container start end step)
+Arguments
+  → `container' (expected List, or String) ✓
   → `start' (expected Number) ✓
   → `end' (expected Number), got true (Bool)
   → `step' (expected Number), got nil (Nil)

tests/unittests/resources/LangSuite/macro-tests.ark

@@ -130,8 +130,14 @@
     (test:case "builtin macros" {
         (macro nil-type ($type nil))
         (macro bool-type ($type true))
-        (test:eq "Nil" nil-type)
-        (test:eq "Bool" bool-type) })
+        (macro show-repr (v) ($repr v))
+        (test:eq nil-type "Nil")
+        (test:eq bool-type "Bool")
+        (test:eq (show-repr (macro a 5)) "(macro a 5)")
+        # function macros are evaluated on instantiation
+        (test:eq (show-repr show-repr) "(macro show-repr (v) ($repr v))")
+        # constant macros are evaluated on declaration
+        (test:eq (show-repr nil-type) "(macro nil-type \"Nil\")") })
 
     (test:case "generate valid arkscript code with macros" {
         (macro make-func (retval) (fun () retval))