OPENNLP-1991: Add a feedforward neural dependency parser

opennlp-api/src/main/java/opennlp/tools/util/ParamChecks.java

@@ -38,6 +38,7 @@ public final class ParamChecks {
   private static final String NOT_BLANK_SUFFIX = " must not be null or blank";
   private static final String NOT_NEGATIVE_SUFFIX = " must not be negative";
   private static final String FINITE_SUFFIX = " must be finite";
+  private static final String POSITIVE_SUFFIX = " must be positive";
 
   private ParamChecks() {
     // utility class
@@ -172,6 +173,41 @@ public static double requireNonNegative(double value, String name) {
     return value;
   }
 
+  /**
+   * Returns {@code value} if it is greater than zero.
+   * <p>
+   * The message of a thrown exception is {@code "<name> must be positive"}.
+   *
+   * @param value The argument to check.
+   * @param name The parameter name used in the exception message.
+   * @return {@code value}, always greater than zero.
+   * @throws IllegalArgumentException Thrown if {@code value} is zero or negative.
+   */
+  public static int requirePositive(int value, String name) {
+    if (value <= 0) {
+      throw new IllegalArgumentException(name + POSITIVE_SUFFIX);
+    }
+    return value;
+  }
+
+  /**
+   * Returns {@code value} if it is finite and greater than zero.
+   * <p>
+   * The message of a thrown exception is {@code "<name> must be positive"}.
+   *
+   * @param value The argument to check.
+   * @param name The parameter name used in the exception message.
+   * @return {@code value}, always finite and greater than zero.
+   * @throws IllegalArgumentException Thrown if {@code value} is zero, negative, infinite,
+   *     or {@code NaN}.
+   */
+  public static double requirePositive(double value, String name) {
+    if (value <= 0 || !Double.isFinite(value)) {
+      throw new IllegalArgumentException(name + POSITIVE_SUFFIX);
+    }
+    return value;
+  }
+
   /**
    * Returns {@code value} if it is finite.
    * <p>

opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java

@@ -544,30 +544,145 @@ public static boolean isUnicodeBlank(CharSequence input) {
     return true;
   }
 
+  /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point whose lowering depends on context. */
+  private static final int GREEK_CAPITAL_SIGMA = 0x03A3;
+
+  /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the lowering of the capital at the end of a word. */
+  private static final int GREEK_SMALL_FINAL_SIGMA = 0x03C2;
+
   /**
    * Converts a {@link CharSequence} to lower case, independent of the current
-   * {@link java.util.Locale} via {@link Character#toLowerCase(int)} which uses
-   * mapping information from the UnicodeData file.
+   * {@link java.util.Locale}. Each code point maps through
+   * {@link Character#toLowerCase(int)}, the UnicodeData mapping, so no character
+   * expands. The one contextual rule of the Unicode
+   * <a href="https://www.unicode.org/Public/UCD/latest/ucd/SpecialCasing.txt">SpecialCasing</a>
+   * data applies as well: a Greek capital sigma at the end of a word, after a cased
+   * letter and before no further cased letter, becomes the final sigma U+03C2.
    *
    * @param string The {@link CharSequence} to transform.
    * @return The lower-cased String.
    */
   public static String toLowerCase(CharSequence string) {
-    int[] cp = string.codePoints().map(Character::toLowerCase).toArray();
+    int[] cp = toLowerCaseCodePoints(string);
     return new String(cp, 0, cp.length);
   }
 
+  /**
+   * Converts a {@link CharSequence} to lower case into a {@link CharBuffer}, with the
+   * mapping of {@link #toLowerCase(CharSequence)}.
+   *
+   * @param sequence The {@link CharSequence} to transform.
+   * @return The lower-cased characters, positioned at the start.
+   */
   public static CharBuffer toLowerCaseCharBuffer(CharSequence sequence) {
     CharBuffer result = CharBuffer.allocate(sequence.length());
-    for (int cp : sequence.codePoints().toArray()) {
-      for (char c : Character.toChars(Character.toLowerCase(cp))) {
-        result.append(c);
+    for (int cp : toLowerCaseCodePoints(sequence)) {
+      if (Character.isBmpCodePoint(cp)) {
+        result.put((char) cp);
+      } else {
+        result.put(Character.highSurrogate(cp));
+        result.put(Character.lowSurrogate(cp));
       }
     }
     result.clear();
     return result;
   }
 
+  /**
+   * Lowers every code point of a sequence, applying the Final_Sigma context.
+   *
+   * @param sequence The {@link CharSequence} to transform.
+   * @return The lower-cased code points.
+   */
+  private static int[] toLowerCaseCodePoints(CharSequence sequence) {
+    final int[] source = sequence.codePoints().toArray();
+    final int[] lowered = new int[source.length];
+    for (int i = 0; i < source.length; i++) {
+      lowered[i] = source[i] == GREEK_CAPITAL_SIGMA && isFinalSigma(source, i)
+          ? GREEK_SMALL_FINAL_SIGMA : Character.toLowerCase(source[i]);
+    }
+    return lowered;
+  }
+
+  /**
+   * The Final_Sigma condition of the Unicode SpecialCasing data: the code point is
+   * preceded by a cased letter and then zero or more case-ignorable code points, and
+   * is not followed by zero or more case-ignorable code points and then a cased letter.
+   *
+   * @param cp The code points of the sequence.
+   * @param index The position of the capital sigma.
+   * @return {@code true} if the sigma at {@code index} is word-final.
+   */
+  private static boolean isFinalSigma(int[] cp, int index) {
+    int before = index - 1;
+    while (before >= 0 && isCaseIgnorable(cp[before])) {
+      before--;
+    }
+    if (before < 0 || !isCased(cp[before])) {
+      return false;
+    }
+    int after = index + 1;
+    while (after < cp.length && isCaseIgnorable(cp[after])) {
+      after++;
+    }
+    return after == cp.length || !isCased(cp[after]);
+  }
+
+  /**
+   * @param cp The code point to test.
+   * @return {@code true} if the code point has the Unicode Cased property: an upper,
+   *         lower, or title case letter, including the Other_Uppercase and
+   *         Other_Lowercase code points.
+   */
+  private static boolean isCased(int cp) {
+    return Character.isUpperCase(cp) || Character.isLowerCase(cp) || Character.isTitleCase(cp);
+  }
+
+  /**
+   * @param cp The code point to test.
+   * @return {@code true} if the code point has the Unicode Case_Ignorable property: the
+   *         general categories Mn, Me, Cf, Lm, and Sk, and the
+   *         <a href="https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt">Word_Break</a>
+   *         values MidLetter, MidNumLet, and Single_Quote.
+   */
+  private static boolean isCaseIgnorable(int cp) {
+    switch (Character.getType(cp)) {
+      case Character.NON_SPACING_MARK:
+      case Character.ENCLOSING_MARK:
+      case Character.FORMAT:
+      case Character.MODIFIER_LETTER:
+      case Character.MODIFIER_SYMBOL:
+        return true;
+      default:
+        break;
+    }
+    switch (cp) {
+      // Single_Quote
+      case 0x0027:
+      // MidLetter
+      case 0x003A:
+      case 0x00B7:
+      case 0x0387:
+      case 0x055F:
+      case 0x05F4:
+      case 0x2027:
+      case 0xFE13:
+      case 0xFE55:
+      case 0xFF1A:
+      // MidNumLet
+      case 0x002E:
+      case 0x2018:
+      case 0x2019:
+      case 0x2024:
+      case 0xFE52:
+      case 0xFF07:
+      case 0xFF0E:
+        return true;
+      default:
+        return false;
+    }
+  }
+
   /*
   public static CharBuffer toLowerCaseCharBuffer(CharSequence string) {
     int[] cp = string.codePoints().map(Character::toLowerCase).toArray();

opennlp-api/src/test/java/opennlp/tools/util/ParamChecksTest.java

@@ -176,6 +176,35 @@ void testNegativeDoubleIsRejected(double value) {
     assertEquals("weight must not be negative", e.getMessage());
   }
 
+  @ParameterizedTest
+  @ValueSource(ints = {1, 2, Integer.MAX_VALUE})
+  void testPositiveIntIsReturnedUnchanged(int value) {
+    assertEquals(value, ParamChecks.requirePositive(value, "size"));
+  }
+
+  @ParameterizedTest
+  @ValueSource(ints = {0, -1, Integer.MIN_VALUE})
+  void testNonPositiveIntIsRejected(int value) {
+    final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+        () -> ParamChecks.requirePositive(value, "size"));
+    assertEquals("size must be positive", e.getMessage());
+  }
+
+  @ParameterizedTest
+  @ValueSource(doubles = {Double.MIN_VALUE, 1.5, Double.MAX_VALUE})
+  void testPositiveDoubleIsReturnedUnchanged(double value) {
+    assertEquals(value, ParamChecks.requirePositive(value, "rate"));
+  }
+
+  @ParameterizedTest
+  @ValueSource(doubles = {0.0, -0.0, -1.0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY,
+      Double.NaN})
+  void testNonPositiveOrNonFiniteDoubleIsRejected(double value) {
+    final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+        () -> ParamChecks.requirePositive(value, "rate"));
+    assertEquals("rate must be positive", e.getMessage());
+  }
+
   @ParameterizedTest
   @ValueSource(doubles = {0.0, -1.5, Double.MAX_VALUE, -Double.MAX_VALUE})
   void testFiniteDoubleIsReturnedUnchanged(double value) {

opennlp-api/src/test/java/opennlp/tools/util/StringUtilLocaleTest.java

@@ -48,20 +48,25 @@ void testGetShortestEditScriptIsIndependentOfDefaultLocale() {
 
   /**
    * Guards the choice of {@link Locale#ROOT} over the code point based
-   * {@link StringUtil#toLowerCase(CharSequence)}: the two disagree on context sensitive
-   * mappings such as the Greek final sigma. Existing models were trained with
-   * {@link Locale#ROOT} semantics, so only {@link Locale#ROOT} keeps them readable.
+   * {@link StringUtil#toLowerCase(CharSequence)}: the results are not the same for
+   * mappings that change the length of a string, such as the dotted capital I. Existing
+   * models were trained with {@link Locale#ROOT} semantics, so only {@link Locale#ROOT}
+   * can read them.
    */
   @Test
   void testGetShortestEditScriptUsesRootFoldingNotCodePointFolding() {
-    // Locale.ROOT maps the trailing capital sigma to the final form 'ς', so both inputs
-    // fold to the same string and no edit is required.
-    Assertions.assertEquals("O", StringUtil.getShortestEditScript("ΟΔΟΣ",
-        "οδος"));
+    // Both lowerings map the trailing capital sigma to the final form, so both inputs
+    // lower to the same string and no edit is required.
+    Assertions.assertEquals("O", StringUtil.getShortestEditScript("\u039F\u0394\u039F\u03A3",
+        "\u03BF\u03B4\u03BF\u03C2"));
+    Assertions.assertEquals("\u03BF\u03B4\u03BF\u03C2",
+        StringUtil.toLowerCase("\u039F\u0394\u039F\u03A3"));
 
-    // Character#toLowerCase is context free and would yield the non final 'σ' instead,
-    // which is a different string and hence a different, model breaking outcome label.
-    Assertions.assertNotEquals("οδος",
-        StringUtil.toLowerCase("ΟΔΟΣ"));
+    // Locale.ROOT expands the dotted capital I to 'i' plus COMBINING DOT ABOVE, while the
+    // code point mapping gives a plain 'i': a different string and hence a different,
+    // model breaking outcome label.
+    Assertions.assertEquals("i\u0307", "\u0130".toLowerCase(Locale.ROOT));
+    Assertions.assertEquals("i", StringUtil.toLowerCase("\u0130"));
+    Assertions.assertNotEquals("O", StringUtil.getShortestEditScript("\u0130", "i"));
   }
 }

opennlp-api/src/test/java/opennlp/tools/util/StringUtilTest.java

@@ -20,6 +20,7 @@
 import java.nio.CharBuffer;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -888,6 +889,56 @@ void testToLowerCase() {
     Assertions.assertEquals("simple", StringUtil.toLowerCase("SIMPLE"));
   }
 
+  /**
+   * @return Words with a Greek capital sigma and their lowering under the Final_Sigma
+   *         context of the Unicode SpecialCasing data. Never {@code null}.
+   */
+  private static Stream<Arguments> finalSigmaWords() {
+    return Stream.of(
+        // word-final after a cased letter
+        Arguments.of("\u039F\u0394\u039F\u03A3", "\u03BF\u03B4\u03BF\u03C2"),
+        // punctuation after the sigma is case-ignorable, so it stays final
+        Arguments.of("\u039F\u0394\u039F\u03A3.", "\u03BF\u03B4\u03BF\u03C2."),
+        // a combining mark after the sigma is case-ignorable as well
+        Arguments.of("\u0391\u03A3\u0301", "\u03B1\u03C2\u0301"),
+        // a cased letter follows: not final, even across an apostrophe
+        Arguments.of("\u0391\u03A3'\u0391", "\u03B1\u03C3'\u03B1"),
+        Arguments.of("\u03A3\u039F\u03A6\u0399\u0391", "\u03C3\u03BF\u03C6\u03B9\u03B1"),
+        // no cased letter before: a lone sigma or one after a digit is not final
+        Arguments.of("\u03A3", "\u03C3"),
+        Arguments.of("1\u03A3", "1\u03C3"),
+        // two sigmas: only the last is final; two words: each has its own final
+        Arguments.of("\u0391\u03A3\u03A3", "\u03B1\u03C3\u03C2"),
+        Arguments.of("\u039F\u0394\u039F\u03A3 \u039F\u0394\u039F\u03A3",
+            "\u03BF\u03B4\u03BF\u03C2 \u03BF\u03B4\u03BF\u03C2"),
+        // a supplementary cased letter before the sigma counts as cased
+        Arguments.of(new String(new int[] {DESERET_CAPITAL_BEE, 0x03A3}, 0, 2),
+            new String(new int[] {DESERET_SMALL_BEE, 0x03C2}, 0, 2)));
+  }
+
+  /**
+   * Verifies the Final_Sigma context of {@link StringUtil#toLowerCase(CharSequence)} and
+   * {@link StringUtil#toLowerCaseCharBuffer(CharSequence)} against the root-locale
+   * lowering of the JDK, which applies the same SpecialCasing rule.
+   *
+   * @param input The word to lower.
+   * @param expected The expected lowering.
+   */
+  @ParameterizedTest
+  @MethodSource("finalSigmaWords")
+  void testToLowerCaseAppliesTheFinalSigmaContext(String input, String expected) {
+    Assertions.assertEquals(expected, input.toLowerCase(Locale.ROOT), "oracle");
+    Assertions.assertEquals(expected, StringUtil.toLowerCase(input));
+    Assertions.assertEquals(expected, StringUtil.toLowerCaseCharBuffer(input).toString());
+  }
+
+  /** Verifies that the per-code-point mapping still never expands a character. */
+  @Test
+  void testToLowerCaseDoesNotExpandDottedCapitalI() {
+    Assertions.assertEquals("istanbul", StringUtil.toLowerCase("\u0130STANBUL"));
+    Assertions.assertEquals("istanbul", StringUtil.toLowerCaseCharBuffer("\u0130STANBUL").toString());
+  }
+
   @Test
   void testToUpperCase() {
     Assertions.assertEquals("TEST", StringUtil.toUpperCase("test"));