Skip to content

Commit

Permalink
LUCENE-9724: Hunspell: tolerate existing aff/dic file typos (apache#2307
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@donnerpeter
donnerpeter authored Feb 7, 2021
1 parent 1852d7a commit 1cc26b6
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 125 deletions.
94 changes: 54 additions & 40 deletions lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder)
Arrays.sort(ignore);
needsInputCleaning = true;
} else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {
int num = Integer.parseInt(singleArgument(reader, line));
int num = parseNum(reader, line);
FST<CharsRef> res = parseConversions(reader, num);
if (line.startsWith("I")) {
iconv = res;
Expand All @@ -397,9 +397,9 @@ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder)
} else if ("TRY".equals(firstWord)) {
tryChars = singleArgument(reader, line);
} else if ("REP".equals(firstWord)) {
int count = Integer.parseInt(singleArgument(reader, line));
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
String[] parts = splitBySpace(reader, reader.readLine(), 3);
String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);
repTable.add(new RepEntry(parts[1], parts[2]));
}
} else if ("KEY".equals(firstWord)) {
Expand All @@ -409,11 +409,11 @@ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder)
} else if ("FORBIDDENWORD".equals(firstWord)) {
forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDMIN".equals(firstWord)) {
compoundMin = Math.max(1, Integer.parseInt(singleArgument(reader, line)));
compoundMin = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDWORDMAX".equals(firstWord)) {
compoundMax = Math.max(1, Integer.parseInt(singleArgument(reader, line)));
compoundMax = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDRULE".equals(firstWord)) {
compoundRules = parseCompoundRules(reader, Integer.parseInt(singleArgument(reader, line)));
compoundRules = parseCompoundRules(reader, parseNum(reader, line));
} else if ("COMPOUNDFLAG".equals(firstWord)) {
compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDBEGIN".equals(firstWord)) {
Expand All @@ -437,7 +437,7 @@ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder)
} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {
simplifiedTriple = true;
} else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {
int count = Integer.parseInt(singleArgument(reader, line));
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
checkCompoundPatterns.add(
new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));
Expand Down Expand Up @@ -481,16 +481,24 @@ static String extractLanguageCode(String isoCode) {
return underscore < 0 ? isoCode : isoCode.substring(0, underscore);
}

private int parseNum(LineNumberReader reader, String line) throws ParseException {
return Integer.parseInt(splitBySpace(reader, line, 2, Integer.MAX_VALUE)[1]);
}

private String singleArgument(LineNumberReader reader, String line) throws ParseException {
return splitBySpace(reader, line, 2)[1];
}

private String[] splitBySpace(LineNumberReader reader, String line, int expectedParts)
throws ParseException {
return splitBySpace(reader, line, expectedParts, expectedParts);
}

private String[] splitBySpace(LineNumberReader reader, String line, int minParts, int maxParts)
throws ParseException {
String[] parts = line.split("\\s+");
if (parts.length < expectedParts
|| parts.length > expectedParts && !parts[expectedParts].startsWith("#")) {
throw new ParseException("Invalid syntax", reader.getLineNumber());
if (parts.length < minParts || parts.length > maxParts && !parts[maxParts].startsWith("#")) {
throw new ParseException("Invalid syntax: " + line, reader.getLineNumber());
}
return parts;
}
Expand All @@ -509,7 +517,7 @@ private Breaks parseBreaks(LineNumberReader reader, String line)
Set<String> starting = new LinkedHashSet<>();
Set<String> ending = new LinkedHashSet<>();
Set<String> middle = new LinkedHashSet<>();
int num = Integer.parseInt(singleArgument(reader, line));
int num = parseNum(reader, line);
for (int i = 0; i < num; i++) {
String breakStr = singleArgument(reader, reader.readLine());
if (breakStr.startsWith("^")) {
Expand Down Expand Up @@ -590,15 +598,8 @@ private void parseAffix(

for (int i = 0; i < numLines; i++) {
String line = reader.readLine();
String[] ruleArgs = line.split("\\s+");

// from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]
// condition is optional
if (ruleArgs.length < 4) {
throw new ParseException(
"The affix file contains a rule with less than four elements: " + line,
reader.getLineNumber());
}
String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);

char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);
String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];
Expand Down Expand Up @@ -654,9 +655,11 @@ private void parseAffix(
"Too many patterns, please report this to dev@lucene.apache.org");
}
seenPatterns.put(regex, patternIndex);
CharacterRunAutomaton pattern =
new CharacterRunAutomaton(new RegExp(regex, RegExp.NONE).toAutomaton());
patterns.add(pattern);
try {
patterns.add(new CharacterRunAutomaton(conditionRegexp(regex).toAutomaton()));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("On line " + reader.getLineNumber() + ": " + line, e);
}
}

Integer stripOrd = seenStrips.get(strip);
Expand Down Expand Up @@ -706,6 +709,17 @@ private void parseAffix(
}
}

private static RegExp conditionRegexp(String regex) {
try {
return new RegExp(regex, RegExp.NONE);
} catch (IllegalArgumentException e) {
if (e.getMessage().contains("expected ']'")) {
return conditionRegexp(regex + "]");
}
throw e;
}
}

char affixData(int affixIndex, int offset) {
return affixData[affixIndex * 4 + offset];
}
Expand Down Expand Up @@ -752,6 +766,8 @@ private void readConfig(BufferedInputStream stream) throws IOException, ParseExc
LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, streamCharset));
String line;
while ((line = reader.readLine()) != null) {
if (line.isBlank()) continue;

String firstWord = line.split("\\s")[0];
if ("SET".equals(firstWord)) {
decoder = getDecoder(singleArgument(reader, line));
Expand All @@ -767,11 +783,12 @@ private void readConfig(BufferedInputStream stream) throws IOException, ParseExc
*
* @return {@code true} if the sequence matched and has been consumed.
*/
@SuppressWarnings("SameParameterValue")
private static boolean maybeConsume(BufferedInputStream stream, byte[] bytes) throws IOException {
stream.mark(bytes.length);
for (int i = 0; i < bytes.length; i++) {
for (byte b : bytes) {
int nextByte = stream.read();
if (nextByte != (bytes[i] & 0xff)) { // covers EOF (-1) as well.
if (nextByte != (b & 0xff)) { // covers EOF (-1) as well.
stream.reset();
return false;
}
Expand Down Expand Up @@ -1344,6 +1361,9 @@ boolean hasFlag(IntsRef forms, char flag, BytesRef scratch) {

/** Abstraction of the process of parsing flags taken from the affix and dic files */
abstract static class FlagParsingStrategy {
// we don't check the flag count, as Hunspell accepts longer sequences
// https://github.com/hunspell/hunspell/issues/707
static final boolean checkFlags = false;

/**
* Parses the given String into a single flag
Expand All @@ -1353,7 +1373,7 @@ abstract static class FlagParsingStrategy {
*/
char parseFlag(String rawFlag) {
char[] flags = parseFlags(rawFlag);
if (flags.length != 1) {
if (checkFlags && flags.length != 1) {
throw new IllegalArgumentException("expected only one flag, got: " + rawFlag);
}
return flags[0];
Expand Down Expand Up @@ -1406,7 +1426,8 @@ public char[] parseFlags(String rawFlags) {
continue;
}
int flag = Integer.parseInt(replacement);
if (flag == FLAG_UNSET || flag >= Character.MAX_VALUE) { // read default flags as well
if (flag >= Character.MAX_VALUE) { // read default flags as well
// accept 0 due to https://github.com/hunspell/hunspell/issues/708
throw new IllegalArgumentException(
"Num flags should be between 0 and " + DEFAULT_FLAGS + ", found " + flag);
}
Expand All @@ -1428,28 +1449,21 @@ private static class DoubleASCIIFlagParsingStrategy extends FlagParsingStrategy

@Override
public char[] parseFlags(String rawFlags) {
if (rawFlags.length() == 0) {
return new char[0];
}

StringBuilder builder = new StringBuilder();
if (rawFlags.length() % 2 == 1) {
if (checkFlags && rawFlags.length() % 2 == 1) {
throw new IllegalArgumentException(
"Invalid flags (should be even number of characters): " + rawFlags);
}
for (int i = 0; i < rawFlags.length(); i += 2) {
char f1 = rawFlags.charAt(i);
char f2 = rawFlags.charAt(i + 1);

char[] flags = new char[rawFlags.length() / 2];
for (int i = 0; i < flags.length; i++) {
char f1 = rawFlags.charAt(i * 2);
char f2 = rawFlags.charAt(i * 2 + 1);
if (f1 >= 256 || f2 >= 256) {
throw new IllegalArgumentException(
"Invalid flags (LONG flags must be double ASCII): " + rawFlags);
}
char combined = (char) (f1 << 8 | f2);
builder.append(combined);
flags[i] = (char) (f1 << 8 | f2);
}

char[] flags = new char[builder.length()];
builder.getChars(0, builder.length(), flags, 0);
return flags;
}
}
Expand Down
84 changes: 20 additions & 64 deletions lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,7 @@
public class TestDictionary extends LuceneTestCase {

public void testSimpleDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("simple.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
Directory tempDir = getDirectory();

Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
Dictionary dictionary = loadDictionary("simple.aff", "simple.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
Expand All @@ -60,85 +56,44 @@ public void testSimpleDictionary() throws Exception {
assertEquals(1, ordList.length);
flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);

affixStream.close();
dictStream.close();
tempDir.close();
}

public void testCompressedDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");

Directory tempDir = getDirectory();
Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
Dictionary dictionary = loadDictionary("compressed.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);

affixStream.close();
dictStream.close();
tempDir.close();
}

public void testCompressedBeforeSetDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
Directory tempDir = getDirectory();

Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
Dictionary dictionary = loadDictionary("compressed-before-set.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);

affixStream.close();
dictStream.close();
tempDir.close();
}

public void testCompressedEmptyAliasDictionary() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
Directory tempDir = getDirectory();

Dictionary dictionary = new Dictionary(tempDir, "dictionary", affixStream, dictStream);
Dictionary dictionary = loadDictionary("compressed-empty-alias.aff", "compressed.dic");
assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);
assertEquals(1, dictionary.lookupPrefix(new char[] {'s'}).length);
IntsRef ordList = dictionary.lookupWord(new char[] {'o', 'l', 'r'}, 0, 3);
BytesRef ref = new BytesRef();
char[] flags = dictionary.decodeFlags(ordList.ints[0], ref);
assertEquals(1, flags.length);

affixStream.close();
dictStream.close();
tempDir.close();
}

// malformed rule causes ParseException
public void testInvalidData() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
Directory tempDir = getDirectory();

public void testInvalidData() {
ParseException expected =
expectThrows(
ParseException.class,
() -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));
assertTrue(
expected
.getMessage()
.startsWith("The affix file contains a rule with less than four elements"));
expectThrows(ParseException.class, () -> loadDictionary("broken.aff", "simple.dic"));
assertTrue(expected.getMessage().startsWith("Invalid syntax"));
assertEquals(24, expected.getErrorOffset());

affixStream.close();
dictStream.close();
tempDir.close();
}

public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseException {
Expand All @@ -155,20 +110,21 @@ public void testUsingFlagsBeforeFlagDirective() throws IOException, ParseExcepti
assertEquals(42, dictionary.keepcase);
}

// malformed flags causes ParseException
public void testInvalidFlags() throws Exception {
InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");
InputStream dictStream = getClass().getResourceAsStream("simple.dic");
Directory tempDir = getDirectory();
public void testForgivableErrors() throws Exception {
Dictionary dictionary = loadDictionary("forgivable-errors.aff", "simple.dic");
assertEquals(1, dictionary.repTable.size());
assertEquals(2, dictionary.compoundMax);

Exception expected =
expectThrows(
Exception.class, () -> new Dictionary(tempDir, "dictionary", affixStream, dictStream));
assertTrue(expected.getMessage().startsWith("expected only one flag"));
loadDictionary("forgivable-errors-long.aff", "single-word.dic");
loadDictionary("forgivable-errors-num.aff", "single-word.dic");
}

affixStream.close();
dictStream.close();
tempDir.close();
private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {
try (InputStream affixStream = getClass().getResourceAsStream(aff);
InputStream dicStream = getClass().getResourceAsStream(dic);
Directory tempDir = getDirectory()) {
return new Dictionary(tempDir, "dictionary", affixStream, dicStream);
}
}

private static class CloseCheckInputStream extends FilterInputStream {
Expand Down
21 changes: 0 additions & 21 deletions lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/broken-flags.aff
View file

This file was deleted.

4 changes: 4 additions & 0 deletions ...e/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-long.aff
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FLAG long

SFX A10 Y 1
SFX A10 nout l .
4 changes: 4 additions & 0 deletions ...ne/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forgivable-errors-num.aff
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FLAG num

SFX 0 Y 1
SFX 0 nout l .
Loading

0 comments on commit 1cc26b6

Please sign in to comment.