Allow unicode

docs/docs/models-and-databases/reference/fields.md

@@ -249,7 +249,6 @@ The `max_size` argument is optional and defaults to 50 characters. It allows to
 The `slugify` argument allows specifying the field from which the slug should be generated. This is useful when you want the slug to be automatically derived from another field.
 
 ```crystal
-
 class Article < Marten::Model
   field :title, :string
   field :slug, :slug, slugify: :title
@@ -262,6 +261,20 @@ article.slug # => "my-article-48e810f2" - the suffix is a random string
 
 When an Article object is saved, the slug field will automatically generate a slug based on the title field if no custom slug is provided.
 
+:::warning
+The slugification functionality also transforms Unicode characters and symbols. When filtering a model by the query parameter, it may be necessary to decode the slug parameter first, because although the browser may show you the unicode characters, it will send the encoded characters in the HTTP request:
+
+```crystal
+class ArticleDetailsHandler < Marten::Handler
+  def get
+    article = Article.get(slug: URI.decode(params[:slug].to_s))
+
+    # …
+  end
+end
+```
+:::
+
 :::info
 As slug fields are usually used to query records, they are indexed by default. You can use the [`index`](#index) option (`index: false`) to disable auto-indexing.
 :::

spec/marten/core/sluggable_spec.cr

@@ -12,6 +12,11 @@ describe Marten::Core::Sluggable do
       slug.should eq("testtitle123")
     end
 
+    it "handles emojis" do
+      value_with_emojis = "🚀 TRAVEL & PLACES"
+      slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_emojis, max_size)
+      slug.should eq("🚀-travel-places")
+    end
     it "converts the string to lowercase" do
       g_slug.should eq("test-title-123545434")
     end
@@ -31,7 +36,7 @@ describe Marten::Core::Sluggable do
     it "removes non-ASCII characters" do
       value_with_non_ascii = "Test Títle 123"
       slug = Marten::Core::SluggableSpec::SlugGenerator.new.generate_slug(value_with_non_ascii, max_size)
-      slug.should eq("test-ttle-123")
+      slug.should eq("test-títle-123")
     end
 
     it "limits the slug length to max_size" do

spec/marten/db/field/slug_spec.cr

@@ -85,15 +85,15 @@ describe Marten::DB::Field::Slug do
 
       article.save
 
-      article.slug.not_nil!.should eq("berraschungsmoment")
+      article.slug.not_nil!.should eq("überraschungsmoment")
     end
 
     it "removes emoji and special characters and slugifies the title" do
       article = Marten::DB::Field::SlugSpec::Article.new(title: "🚀 TRAVEL & PLACES")
 
       article.save
 
-      article.slug.not_nil!.should eq("travel-places")
+      article.slug.not_nil!.should eq("🚀-travel-places")
     end
 
     it "trims leading and trailing whitespace and slugifies the title" do

src/marten/core/sluggable.cr

@@ -2,23 +2,20 @@ module Marten
   module Core
     # The Sluggable module provides functionality for generating URL-friendly slugs from strings.
     module Sluggable
-      private NON_ALPHANUMERIC_RE  = /[^\w\s-]/
+      private NON_ALPHANUMERIC_RE  = /[^\p{L}\p{N}\p{So}\s-]/
       private WHITESPACE_HYPHEN_RE = /[-\s]+/
-      private NON_ASCII_RE         = /[^\x00-\x7F]/
 
       # Generates a slug from the given value, ensuring the resulting slug does not exceed the specified max_size.
       #
       # The slug is created by:
-      # 1. Removing non-alphanumeric characters (except whitespace and hyphens).
+      # 1. Removing non-alphanumeric characters (except for Unicode letters, numbers, symbols, whitespace, and hyphens).
       # 2. Converting the string to lowercase.
       # 3. Replacing sequences of whitespace and hyphens with a single hyphen.
-      # 4. Removing non-ASCII characters.
-      # 5. Stripping trailing hyphens and underscores of the slug.
-      # 6. Truncate the slug to the maximum size
+      # 4. Stripping trailing hyphens and underscores from the slug.
+      # 5. Truncating the slug to the specified max_size.
       def generate_slug(value, max_size)
         slug = value.gsub(NON_ALPHANUMERIC_RE, "").downcase
         slug = slug.gsub(WHITESPACE_HYPHEN_RE, "-")
-        slug = slug.gsub(NON_ASCII_RE, "")
         slug[...(max_size)].strip("-_")
       end
     end