java: Use a generated keyword matcher to improve performance

CHANGELOG.md

@@ -9,6 +9,7 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt
 ## [Unreleased]
 ### Fixed
 - [Java] Optimize `GherkinLine.substringTrimmed` ([#444](https://github.com/cucumber/gherkin/pull/444))
+- [Java] Improve performance with a generated keyword matcher ([#445](https://github.com/cucumber/gherkin/pull/445))
 
 ### Changed
 - Fixed Afrikaans translation for "rule" ([#428](https://github.com/cucumber/gherkin/pull/428))

java/pom.xml

@@ -42,6 +42,13 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>org.assertj</groupId>
+                <artifactId>assertj-bom</artifactId>
+                <version>3.27.3</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -76,6 +83,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.freemarker</groupId>
             <artifactId>freemarker</artifactId>
@@ -96,6 +109,11 @@
             <version>1.37</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -152,20 +170,21 @@
                         <goals>
                             <goal>java</goal>
                         </goals>
+                        <configuration>
+                            <classpathScope>test</classpathScope>
+                            <addOutputToClasspath>false</addOutputToClasspath>
+                            <addResourcesToClasspath>false</addResourcesToClasspath>
+                            <additionalClasspathElements>
+                                ${project.build.directory}/codegen-classes
+                            </additionalClasspathElements>
+                            <mainClass>Generate</mainClass>
+                            <arguments>
+                                <argument>${project.build.directory}/generated-sources/gherkin</argument>
+                                <argument>io/cucumber/gherkin</argument>
+                            </arguments>
+                        </configuration>
                     </execution>
                 </executions>
-                <configuration>
-                    <classpathScope>test</classpathScope>
-                    <addOutputToClasspath>false</addOutputToClasspath>
-                    <addResourcesToClasspath>false</addResourcesToClasspath>
-                    <additionalClasspathElements>${project.build.directory}/codegen-classes
-                    </additionalClasspathElements>
-                    <mainClass>GenerateGherkinDialects</mainClass>
-                    <arguments>
-                        <argument>${project.build.directory}/generated-sources/gherkin-dialects</argument>
-                        <argument>io/cucumber/gherkin</argument>
-                    </arguments>
-                </configuration>
             </plugin>
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
@@ -179,7 +198,7 @@
                         </goals>
                         <configuration>
                             <sources>
-                                <source>${project.build.directory}/generated-sources/gherkin-dialects</source>
+                                <source>${project.build.directory}/generated-sources/gherkin</source>
                             </sources>
                         </configuration>
                     </execution>

java/src/codegen/java/Generate.java

@@ -0,0 +1,12 @@
+public class Generate {
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 2) {
+            throw new IllegalArgumentException("Usage: <baseDirectory> <packagePath>");
+        }
+        String baseDirectory = args[0];
+        String packagePath = args[1];
+        GenerateGherkinDialects.generate(baseDirectory, packagePath);
+        GenerateKeywordMatchers.generate(baseDirectory, packagePath);
+    }
+}

java/src/codegen/java/GenerateGherkinDialects.java

@@ -25,16 +25,9 @@
  * This class generates the GherkinDialects class using the FreeMarker
  * template engine and provided templates.
  */
-public class GenerateGherkinDialects {
-
-    public static void main(String[] args) throws Exception {
-        if (args.length != 2) {
-            throw new IllegalArgumentException("Usage: <baseDirectory> <packagePath>");
-        }
-
-        String baseDirectory = args[0];
-        String packagePath = args[1];
+class GenerateGherkinDialects {
 
+    static void generate(String baseDirectory, String packagePath) throws Exception {
         Path path = Paths.get(baseDirectory, packagePath,  "GherkinDialects.java");
 
         Template dialectsSource = readTemplate();

java/src/codegen/java/GenerateKeywordMatchers.java

@@ -0,0 +1,234 @@
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+import freemarker.template.TemplateException;
+import freemarker.template.TemplateExceptionHandler;
+import io.cucumber.messages.types.StepKeywordType;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.BinaryOperator;
+import java.util.stream.Collectors;
+
+import static io.cucumber.messages.types.StepKeywordType.ACTION;
+import static io.cucumber.messages.types.StepKeywordType.CONJUNCTION;
+import static io.cucumber.messages.types.StepKeywordType.CONTEXT;
+import static io.cucumber.messages.types.StepKeywordType.OUTCOME;
+import static io.cucumber.messages.types.StepKeywordType.UNKNOWN;
+import static java.nio.file.Files.newBufferedReader;
+import static java.nio.file.Files.newBufferedWriter;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.util.Comparator.naturalOrder;
+import static java.util.Map.Entry.comparingByKey;
+
+/*
+ * This class generates the KeywordMatchers class using the FreeMarker
+ * template engine and provided templates.
+ */
+class GenerateKeywordMatchers {
+
+    private static final Comparator<String> LONGEST_TO_SHORTEST_COMPARATOR =
+            (s1, s2) -> Integer.compare(s2.length(), s1.length());
+
+    static void generate(String baseDirectory, String packagePath) throws Exception {
+        Path path = Paths.get(baseDirectory, packagePath, "KeywordMatchers.java");
+
+        Template dialectsSource = readTemplate();
+
+        Map<String, Map<String, Object>> binding = new LinkedHashMap<>();
+        Map<String, Object> dialects = readGherkinLanguages();
+        Map<String, Object> matcherModels = createMatcherModels(dialects);
+        binding.put("matchers", matcherModels);
+
+        try {
+            Files.createDirectories(path.getParent());
+            dialectsSource.process(binding, newBufferedWriter(path, CREATE, TRUNCATE_EXISTING));
+        } catch (IOException | TemplateException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static Template readTemplate() throws IOException {
+        Configuration cfg = new Configuration(Configuration.VERSION_2_3_21);
+        cfg.setClassForTemplateLoading(GenerateKeywordMatchers.class, "templates");
+        cfg.setDefaultEncoding("UTF-8");
+        cfg.setLocale(Locale.US);
+        cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
+        return cfg.getTemplate("keyword-matchers.java.ftl");
+    }
+
+    private static Map<String, Object> readGherkinLanguages() throws IOException {
+        ObjectMapper objectMapper = new ObjectMapper();
+        TypeReference<Map<String, Object>> mapObjectType = new TypeReference<Map<String, Object>>() {
+        };
+        try (Reader reader = newBufferedReader(Paths.get("../gherkin-languages.json"))) {
+            Map<String, Object> sorted = new TreeMap<>(naturalOrder());
+            sorted.putAll(objectMapper.readValue(reader, mapObjectType));
+            return sorted;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> createMatcherModels(Map<String, Object> dialects) {
+        return dialects.entrySet()
+                .stream()
+                .sorted(comparingByKey())
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey, entry -> matcherModel(
+                                entry.getKey(),
+                                (Map<String, Object>) entry.getValue()),
+                        uniqueKeyCondition(),
+                        LinkedHashMap::new
+                ));
+    }
+
+    private static BinaryOperator<Object> uniqueKeyCondition() {
+        return (a, b) ->  {
+            throw new IllegalStateException("Duplicate keys " + a + " and " + b);
+        };
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> matcherModel(String language, Map<String, Object> dialect) {
+        String normalizedLanguage = getNormalizedLanguage(language);
+        Map<String, Object> model = new HashMap<>();
+        model.put("className", capitalize(normalizedLanguage));
+
+        List<String> featureKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("feature")
+        );
+        model.put("features", featureKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        List<String> backgroundKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("background")
+        );
+        model.put("backgrounds", backgroundKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        List<String> ruleKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("rule")
+        );
+        model.put("rules", ruleKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        List<String> scenarioKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("scenario"),
+                (List<String>) dialect.get("scenarioOutline")
+        );
+        model.put("scenarios", scenarioKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        List<String> exampleKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("examples")
+        );
+        model.put("examples", exampleKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        Map<String, StepKeywordType> aggregateKeywordTypes = aggregateKeywordTypes(
+                (List<String>) dialect.get("given"),
+                (List<String>) dialect.get("when"),
+                (List<String>) dialect.get("then"),
+                (List<String>) dialect.get("and"),
+                (List<String>) dialect.get("but")
+        );
+        List<String> stepKeywords = distinctSortedKeywords(
+                (List<String>) dialect.get("given"),
+                (List<String>) dialect.get("when"),
+                (List<String>) dialect.get("then"),
+                (List<String>) dialect.get("and"),
+                (List<String>) dialect.get("but")
+        );
+        model.put("steps", stepKeywords.stream().map(keyword -> {
+            Map<String, Object> entry = new HashMap<>();
+            entry.put("keyword", keyword);
+            entry.put("length", keyword.length());
+            entry.put("keywordType", aggregateKeywordTypes.get(keyword).name());
+            return entry;
+        }).collect(Collectors.toList()));
+
+        return model;
+    }
+
+    private static String capitalize(String str) {
+        return str.substring(0, 1).toUpperCase() + str.substring(1);
+    }
+
+    private static String getNormalizedLanguage(String language) {
+        return language.replaceAll("[\\s-]", "_").toLowerCase();
+    }
+
+    @SafeVarargs
+    private static List<String> distinctSortedKeywords(List<String>... keywords) {
+        // french is the largest dialect with 32 keywords, so we build the sorting hashset with this max size
+        Set<String> uniqueKeywords = new HashSet<>(32);
+        for (List<String> keyword : keywords) {
+            uniqueKeywords.addAll(keyword);
+        }
+        List<String> sortedKeywords = new ArrayList<>(uniqueKeywords);
+        sortedKeywords.sort(LONGEST_TO_SHORTEST_COMPARATOR);
+        return sortedKeywords;
+    }
+
+    private static Map<String, StepKeywordType> aggregateKeywordTypes(
+            List<String> givenKeywords,
+            List<String> whenKeywords,
+            List<String> thenKeywords,
+            List<String> andKeywords,
+            List<String> butKeywords
+    ) {
+        Map<String, StepKeywordType> stepKeywordsTypes = new HashMap<>();
+        mergeKeywordTypes(stepKeywordsTypes, CONTEXT, givenKeywords);
+        mergeKeywordTypes(stepKeywordsTypes, ACTION, whenKeywords);
+        mergeKeywordTypes(stepKeywordsTypes, OUTCOME, thenKeywords);
+        mergeKeywordTypes(stepKeywordsTypes, CONJUNCTION, distinctSortedKeywords(andKeywords, butKeywords));
+        return stepKeywordsTypes;
+    }
+
+    private static void mergeKeywordTypes(Map<String, StepKeywordType> accumulator, StepKeywordType type, List<String> keywords) {
+        for (String keyword : keywords) {
+            StepKeywordType existing = accumulator.get(keyword);
+            if (existing == null) {
+                accumulator.put(keyword, type);
+            } else {
+                // Type is unknown if there are multiple applicable types.
+                accumulator.put(keyword, UNKNOWN);
+            }
+        }
+    }
+}