feat: target_script="arabic" and "hebrew" (#792)

CHANGELOG.md

@@ -110,6 +110,27 @@ from 12 to 13, which is source- and binary-breaking for anyone constructing one
 
 ### Added
 
+- **`target_script="arabic"` and `"hebrew"` (#792).** Generation drops an equivalence class
+  entirely when no member belongs to the target script, so a class whose members are all
+  Arabic folded to nothing under either shipped table — 948 of TR39's 1,007 strong-RTL
+  sources were in that position (#791). These give them somewhere to land: **373 rows** and
+  **261 rows** of cross-script punctuation, letterlike symbols and digits.
+
+  Opt-in, like `cyrillic`. No preset consumes a non-Latin target, so this adds a view
+  rather than changing any existing answer — which is what resolved the blocking question
+  #792 §1 raised about colliding with #735.
+
+  **They do not reach an intra-Arabic pair.** #792 was filed believing an Arabic target
+  would fold Persian keheh onto Arabic kaf; prototyping it first showed all four code
+  points in its motivating table absent from the generated table, because both members of
+  each pair are already in the target script. TR39 does put them in one class — the data is
+  not the problem, the cross-script model is. Split out as #848, which needs the generator
+  to stop discarding same-script classes, and which is #831's machinery one script over.
+
+  `is_suspicious_hostname` is unaffected and says so: it computes whole-script-confusable
+  against Latin with the fold's target hardcoded, so an Arabic label whose skeleton stays
+  Arabic cannot qualify whatever these tables hold.
+
 - **`UNICODE_VERSION` and `KEY_SCHEMA_VERSION`, on all seven surfaces (#645, #642, #644).**
   #641, #642 and #644 were filed separately and are one failure repeated: disarm knows
   something an integrator needs and has no channel to say it. `CONFUSABLES_VERSION` (#560)

bindings/cabi/src/lib.rs

@@ -107,7 +107,7 @@ fn build_transliterate(
 
 // ── Confusables & normalization (fallible) ──────────────────────────────────────
 
-/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"`).
+/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"` | `"arabic"` | `"hebrew"`).
 #[ffi_export]
 fn disarm_normalize_confusables(text: char_p::Ref<'_>, target: char_p::Ref<'_>) -> DisarmResult {
     build_normalize_confusables(

bindings/java/rust/src/lib.rs

@@ -388,7 +388,7 @@ fn new_untranslatable<'l>(env: &mut Env<'l>, u: &api::Untranslatable) -> JniResu
 
 // ── Confusables (TR39) ──────────────────────────────────────────────────────────
 
-/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"`).
+/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"` | `"arabic"` | `"hebrew"`).
 #[jni_mangle("dev.disarm.internal.Native")]
 pub fn normalizeConfusables<'l>(
     mut env: EnvUnowned<'l>,

bindings/node/src/lib.rs

@@ -141,7 +141,7 @@ pub struct Untranslatable {
 
 // ── Confusables (TR39) ────────────────────────────────────────────────────────
 
-/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"`).
+/// Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"` | `"arabic"` | `"hebrew"`).
 #[napi]
 pub fn normalize_confusables(
     text: String,

bindings/ruby/ext/disarm/src/lib.rs

@@ -204,7 +204,7 @@ fn transliterate_opts(
 
 // ── Confusables (TR39) ────────────────────────────────────────────────────────
 
-/// `Disarm._normalize_confusables(text, "latin" | "cyrillic")`.
+/// `Disarm._normalize_confusables(text, "latin" | "cyrillic" | "arabic" | "hebrew")`.
 fn normalize_confusables(
     text: Wtf8Text,
     target: String,
@@ -215,7 +215,7 @@ fn normalize_confusables(
     Ok(api::normalize_confusables_with(&text, target, digit_policy).into_owned())
 }
 
-/// `Disarm._confusable?(text, "latin" | "cyrillic")`.
+/// `Disarm._confusable?(text, "latin" | "cyrillic" | "arabic" | "hebrew")`.
 fn is_confusable(text: Wtf8Text, target: String) -> Result<bool, Error> {
     let target: api::TargetScript = target.parse().map_err(|e| map_err(&e))?;
     Ok(api::is_confusable(&text, target))

build.rs

@@ -17,6 +17,19 @@ use std::fs;
 use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
 
+/// Every bundled confusables table: `(script token, TSV file name, generated ident)`.
+///
+/// One list, because there were three before this: the Latin and Cyrillic tables were
+/// built by hand-written blocks, Arabic and Hebrew by a loop, and the upstream-version
+/// agreement check compared only the first two. #792 added two tables and the check did
+/// not notice (#849 review). Adding a target here now reaches the version check for free.
+const CONFUSABLE_TABLES: [(&str, &str, &str); 4] = [
+    ("latin", "confusables_to_latin.tsv", "TO_LATIN"),
+    ("cyrillic", "confusables_to_cyrillic.tsv", "TO_CYRILLIC"),
+    ("arabic", "confusables_to_arabic.tsv", "TO_ARABIC"),
+    ("hebrew", "confusables_to_hebrew.tsv", "TO_HEBREW"),
+];
+
 fn main() {
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
     let data_dir = Path::new("src/tables/data");
@@ -131,6 +144,24 @@ fn main() {
         fs::write(out_dir.join("confusables_to_cyrillic_phf.rs"), code).unwrap();
     }
 
+    // --- Confusables (Arabic and Hebrew targets, #792) ---
+    // The RTL targets. 948 of TR39's 1,007 strong-RTL sources were unmapped under both
+    // existing targets (#791), because generation drops a class entirely when no member
+    // belongs to the target script — so a class whose members are all Arabic survived
+    // into neither table. No `inject_folding_singleton_rows` here: that pass (#481) is
+    // about a canonical singleton resolving as its target, and both of these tables are
+    // generated from classes whose target is already the script's own letter.
+    for &(script, table, ident) in &CONFUSABLE_TABLES[2..] {
+        let entries = read_char_str_tsv(&data_dir.join(table));
+        assert!(!entries.is_empty(), "{table}: expected ≥1 entries, got 0");
+        let code = build_char_str_map(&entries, ident, "");
+        fs::write(
+            out_dir.join(format!("confusables_to_{script}_phf.rs")),
+            code,
+        )
+        .unwrap();
+    }
+
     // --- Bundled confusables.txt version (#560) ---
     // The upstream version is already written in the TSV header line that
     // `read_char_str_tsv` skips as a comment. Parse it here and emit it as a const so a
@@ -139,15 +170,28 @@ fn main() {
     // shape, which is the build-time assertion the acceptance criteria ask for: the
     // constant cannot silently go stale, and the version is never typed a second time.
     {
-        let latin = read_confusables_version(&data_dir.join("confusables_to_latin.tsv"));
-        let cyrillic = read_confusables_version(&data_dir.join("confusables_to_cyrillic.tsv"));
-        assert_eq!(
-            latin, cyrillic,
-            "confusables tables disagree on the upstream version (latin {latin}, cyrillic \
-             {cyrillic}); both are generated from one confusables.txt release, so a single \
-             CONFUSABLES_VERSION const no longer covers them. Either regenerate both from the \
-             same release, or split the const per table."
-        );
+        // Every bundled confusables table, not just the first two. #792 added Arabic and
+        // Hebrew, and this check kept comparing Latin against Cyrillic — so a mixed
+        // upstream release would have shipped with one CONFUSABLES_VERSION describing
+        // four tables it no longer described (#849 review). Derived from the table list
+        // above rather than written out again, so the next target is covered by adding it
+        // in one place.
+        let versions: Vec<(&str, String)> = CONFUSABLE_TABLES
+            .iter()
+            .map(|(script, table, _)| (*script, read_confusables_version(&data_dir.join(table))))
+            .collect();
+        let (first_script, first) = &versions[0];
+        for (script, version) in &versions[1..] {
+            assert_eq!(
+                first, version,
+                "confusables tables disagree on the upstream version ({first_script} \
+                 {first}, {script} {version}); all are generated from one confusables.txt \
+                 release, so a single CONFUSABLES_VERSION const no longer covers them. \
+                 Either regenerate them all from the same release, or split the const per \
+                 table."
+            );
+        }
+        let latin = first.clone();
         fs::write(
             out_dir.join("confusables_version.rs"),
             format!(

docs/provenance.md

@@ -16,7 +16,7 @@ against for byte-stable behavior.
 
 | Surface / table | Standard | Version |
 |---|---|---|
-| Confusables — `confusables_to_latin.tsv`, `confusables_to_cyrillic.tsv` | Unicode **UTS #39** `confusables.txt` | **17.0.0**, plus disarm additions (#336) — see below |
+| Confusables — `confusables_to_latin.tsv`, `confusables_to_cyrillic.tsv`, `confusables_to_arabic.tsv`, `confusables_to_hebrew.tsv` | Unicode **UTS #39** `confusables.txt` | **17.0.0**, plus disarm additions (#336) — see below |
 | Case folding — `case_folding.tsv` | Unicode `CaseFolding.txt` (status C + F) | **16.0** |
 | East Asian width — `char_width.tsv` | UCD `EastAsianWidth.txt` | **15.1.0** |
 | Emoji presentation — `emoji_presentation.tsv` | UCD `emoji-data.txt` | **15.1.0** |

docs/user-guide/confusables.md

@@ -243,15 +243,21 @@ By default, confusables are normalized to Latin. You can specify a different tar
 |--------|----------|-------------|
 | `"latin"` (default) | 2,273 | Non-Latin → Latin. Cyrillic а→a, Greek Ρ→P, etc. |
 | `"cyrillic"` | 1,349 | Non-Cyrillic → Cyrillic. Latin A→А, p→р, etc. |
+| `"arabic"` | 373 | Non-Arabic → Arabic. `⸮`→`؟`, `𞣉`→`٣`, etc. |
+| `"hebrew"` | 261 | Non-Hebrew → Hebrew. `ℵ`→`א`, `∸`→`﬩`, etc. |
 
 Characters without a confusable equivalent in the target script pass through unchanged. This is pure visual mapping — not transliteration. Latin `f` has no Cyrillic lookalike, so it stays as `f`.
 
-### These are the only two views, not two views of one table
+### The targets are not views of one table
 
 The table above reads like a menu, and it is worth being precise about what it is not.
 Generation keeps the members of an equivalence class that belong to the target script and
-**drops the class entirely when no member does**. So a class whose members are all Arabic,
-or all CJK, survives into neither table — there is no third view to select.
+**drops the class entirely when no member does**. So a class whose members are all CJK
+survives into none of the four tables, and adding a target does not open a view onto rows
+the others hide — it builds a table from the classes that have a member in that script.
+
+`"arabic"` and `"hebrew"` (#792) were added for exactly this reason, and the section below
+says what they reach and what they do not.
 
 Measured against the bundled `confusables.txt` (Unicode 17.0.0):
 
@@ -303,6 +309,32 @@ The same holds for Farsi yeh against Arabic yeh. So a caller comparing identitie
 `search_key` or `catalog_key` is not exposed to the intra-Arabic gap that
 `normalize_confusables` has, and a caller using the fold directly is.
 
+### The RTL targets, and what they do not reach
+
+`"arabic"` and `"hebrew"` exist because generation **drops an equivalence class entirely**
+when no member belongs to the target script, so a class whose members are all Arabic folded
+to nothing under either of the original two. 948 of TR39's 1,007 strong-RTL sources were in
+that position (#791). These give them somewhere to land.
+
+They fold **toward** Arabic and Hebrew from other scripts, and that is the limit of what a
+target-script table can do. An **intra-Arabic** pair — Persian keheh against Arabic kaf,
+which TR39 puts in one equivalence class — is not reachable, because both members are
+already in the target script:
+
+```python
+from disarm import normalize_confusables
+
+# unchanged: a cross-script table cannot express a same-script pair
+assert normalize_confusables("\u06a9", target_script="arabic") == "\u06a9"
+```
+
+Tracked as [#848](https://github.com/raeq/disarm/issues/848), which needs the generator to
+stop discarding same-script classes — a different change from adding a target.
+
+`is_suspicious_hostname` is unaffected too, and deliberately: it computes
+whole-script-confusable against Latin and calls the fold with `"latin"` hardcoded, so an
+Arabic label whose skeleton stays Arabic cannot qualify whatever these tables hold.
+
 ## Script detection
 
 Identify which Unicode scripts are present in a string:

python/disarm/_api.py

@@ -790,8 +790,15 @@ def normalize_confusables(
     Args:
         text: Input string potentially containing homoglyphs.
         target_script: Script to normalize toward. Supported values:
-            ``"latin"`` (default, 2,273 mappings) and ``"cyrillic"``
-            (1,349 mappings).
+            ``"latin"`` (default, 2,273 mappings), ``"cyrillic"`` (1,349 mappings),
+            ``"arabic"`` (373 mappings) and ``"hebrew"`` (261 mappings).
+
+            The two RTL targets exist because generation drops an equivalence
+            class entirely when no member belongs to the target script, so a
+            class whose members are all Arabic folded to nothing under either of
+            the first two (#791/#792). They do **not** reach an intra-Arabic pair
+            such as ``"\u06a9"`` against ``"\u0643"``: both members are already in
+            the target script, which a cross-script table cannot express (#848).
         digit_policy: How non-Latin **digits** fold (#561).
 
             ``"numeric"`` (default) sends them to the ASCII digit — ``०`` becomes
@@ -2079,8 +2086,10 @@ def unmapped_confusables(*, target_script: str | Script = "latin") -> frozenset[
     as coverage it does not have.
 
     Args:
-        target_script: Which bundled table to report against — ``"latin"`` (default)
-            or ``"cyrillic"``. The two have genuinely different coverage.
+        target_script: Which bundled table to report against — ``"latin"`` (default),
+            ``"cyrillic"``, ``"arabic"`` or ``"hebrew"``. They have genuinely
+            different coverage, and the residue is largest for the RTL targets
+            because most of TR39 has no Arabic or Hebrew member at all.
 
     Returns:
         A frozenset of single-character strings.

scripts/gen_confusables.py

@@ -371,6 +371,29 @@ def _close_under_case(fold: dict[str, str]) -> dict[str, str]:
 # Target script definitions
 # ---------------------------------------------------------------------------
 
+
+def is_arabic(cp: int) -> bool:
+    """True if codepoint is in an Arabic block (#792).
+
+    Includes the presentation-form blocks. They are removed by the NFKC step every preset
+    runs before the fold, so a row keyed on one is unreachable through a preset — but
+    `normalize_confusables` is callable directly, and the caller who does that is exactly
+    the one who has not normalized first.
+    """
+    return (
+        (0x0600 <= cp <= 0x06FF)  # Arabic
+        or (0x0750 <= cp <= 0x077F)  # Arabic Supplement
+        or (0x08A0 <= cp <= 0x08FF)  # Arabic Extended-A
+        or (0xFB50 <= cp <= 0xFDFF)  # Arabic Presentation Forms-A
+        or (0xFE70 <= cp <= 0xFEFF)  # Arabic Presentation Forms-B
+    )
+
+
+def is_hebrew(cp: int) -> bool:
+    """True if codepoint is in a Hebrew block (#792)."""
+    return (0x0590 <= cp <= 0x05FF) or (0xFB1D <= cp <= 0xFB4F)
+
+
 SCRIPTS = {
     "latin": {
         "is_target": is_latin,
@@ -380,6 +403,23 @@ def _close_under_case(fold: dict[str, str]) -> dict[str, str]:
         "is_target": is_cyrillic,
         "is_target_or_common": lambda cp: is_cyrillic(cp) or is_combining_mark(cp),
     },
+    # #792: the RTL targets. 948 of the 1,007 strong-RTL sources in TR39 are unmapped
+    # under both existing targets (#791), because generation drops a class entirely when
+    # no member belongs to the target script — so a class whose members are all Arabic
+    # survives into neither table. These give those classes somewhere to land.
+    #
+    # They do NOT reach an intra-Arabic pair such as `\u06a9` / `\u0643`: both members are
+    # already in the target script, which `filter_direct` skips and `filter_via_classes`
+    # has nothing to map from. That is #848, and it needs the generator to stop discarding
+    # same-script classes rather than a new target.
+    "arabic": {
+        "is_target": is_arabic,
+        "is_target_or_common": lambda cp: is_arabic(cp) or is_combining_mark(cp),
+    },
+    "hebrew": {
+        "is_target": is_hebrew,
+        "is_target_or_common": lambda cp: is_hebrew(cp) or is_combining_mark(cp),
+    },
 }
 
 

src/api/metadata.rs

@@ -239,7 +239,7 @@ mod tests {
         );
     }
 
-    /// Both bundled tables are folded from one upstream release, which is what lets a
+    /// All four bundled tables are folded from one upstream release, which is what lets a
     /// single const cover them. build.rs asserts it at build time; assert it here too so
     /// the reason for the single const is visible in the test suite.
     #[test]