Merge pull request #2979 from idodeclare/feature/uctags

Feature/uctags

Author: tarzanek

dev/checkstyle/suppressions.xml

@@ -39,6 +39,8 @@ Portions Copyright (c) 2018-2019, Chris Fraire <cfraire@me.com>.
         |JFlexXrefUtils\.java|FileAnalyzerFactory\.java|SearchController\.java|
         |Context\.java|HistoryContext\.java|Suggester\.java" />
 
+    <suppress checks="FileLength" files="RuntimeEnvironment\.java" />
+
     <suppress checks="MethodLength" files="Indexer\.java" />
 
     <suppress checks="JavadocStyle" files="MellonHeaderDecoder\.java|CustomSloppyPhraseScorer\.java|

dev/install-universal_ctags.sh

@@ -2,8 +2,6 @@
 
 git clone https://github.com/universal-ctags/ctags.git
 cd ctags
-# need to use last working version until issue #2977 is resolved.
-git reset --hard 1295de210df49c979f27e185106a7ebc55146c57
 ./autogen.sh
 ./configure && make && make install
 /usr/local/bin/ctags --version

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java

@@ -34,6 +34,7 @@
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -58,20 +59,25 @@ public class Ctags implements Resettable {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(Ctags.class);
 
+    private final RuntimeEnvironment env;
     private volatile boolean closing;
     private final LangTreeMap defaultLangMap = new LangTreeMap();
     private LangMap langMap;
+    private List<String> command;
     private Process ctags;
     private OutputStreamWriter ctagsIn;
     private BufferedReader ctagsOut;
     private static final String CTAGS_FILTER_TERMINATOR = "__ctags_done_with_file__";
-    private String binary;
     private String CTagsExtraOptionsFile = null;
     private int tabSize;
     private Duration timeout = Duration.ofSeconds(10);
 
     private boolean junit_testing = false;
 
+    public Ctags() {
+        env = RuntimeEnvironment.getInstance();
+    }
+
     /**
      * Gets a value indicating if a subprocess of ctags was started and it is
      * not alive.
@@ -82,14 +88,6 @@ public boolean isClosed() {
         return ctags != null && !ctags.isAlive();
     }
 
-    public String getBinary() {
-        return binary;
-    }
-
-    public void setBinary(String binary) {
-        this.binary = binary;
-    }
-
     public void setLangMap(LangMap langMap) {
         this.langMap = langMap;
     }
@@ -136,22 +134,31 @@ public void close() {
         }
     }
 
-    private void initialize() throws IOException {
-        ProcessBuilder processBuilder;
-        List<String> command = new ArrayList<>();
+    /**
+     * Gets the command-line arguments used to run ctags.
+     * @return a defined (immutable) list
+     */
+    public List<String> getArgv() {
+        initialize();
+        return Collections.unmodifiableList(command);
+    }
+
+    private void initialize() {
+        env.validateUniversalCtags();
 
-        command.add(binary);
-        command.add("--c-kinds=+l");
+        command = new ArrayList<>();
+        command.add(env.getCtags());
+        command.add("--kinds-c=+l");
 
         // Workaround for bug #14924: Don't get local variables in Java
         // code since that creates many false positives.
         // CtagsTest : bug14924 "too many methods" guards for this
         // universal ctags are however safe, so enabling for them
-        command.add("--java-kinds=+l");
+        command.add("--kinds-java=+l");
 
-        command.add("--sql-kinds=+l");
-        command.add("--Fortran-kinds=+L");
-        command.add("--C++-kinds=+l");
+        command.add("--kinds-sql=+l");
+        command.add("--kinds-Fortran=+L");
+        command.add("--kinds-C++=+l");
         command.add("--file-scope=yes");
         command.add("-u");
         command.add("--filter=yes");
@@ -201,18 +208,20 @@ private void initialize() throws IOException {
 
         /* Add extra command line options for ctags. */
         if (CTagsExtraOptionsFile != null) {
-            LOGGER.log(Level.INFO, "Adding extra options to ctags");
+            LOGGER.log(Level.FINER, "Adding extra options to ctags");
             command.add("--options=" + CTagsExtraOptionsFile);
         }
+    }
 
+    private void run() throws IOException {
         StringBuilder sb = new StringBuilder();
         for (String s : command) {
             sb.append(s).append(" ");
         }
         String commandStr = sb.toString();
         LOGGER.log(Level.FINE, "Executing ctags command [{0}]", commandStr);
 
-        processBuilder = new ProcessBuilder(command);
+        ProcessBuilder processBuilder = new ProcessBuilder(command);
 
         ctags = processBuilder.start();
         ctagsIn = new OutputStreamWriter(
@@ -242,7 +251,9 @@ private void initialize() throws IOException {
     }
 
     private void addRustSupport(List<String> command) {
-        command.add("--langdef=rust");
+        if (!env.getCtagsLanguages().contains("Rust")) { // Built-in would be capitalized.
+            command.add("--langdef=rust"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".RS", "rust"); // Upper-case file spec
 
         // The following are not supported yet in Universal Ctags b13cb551
@@ -256,9 +267,12 @@ private void addRustSupport(List<String> command) {
     }
 
     private void addPowerShellSupport(List<String> command) {
-        command.add("--langdef=powershell");
+        if (!env.getCtagsLanguages().contains("PowerShell")) { // Built-in would be capitalized.
+            command.add("--langdef=powershell"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".PS1", "powershell"); // Upper-case file spec
         defaultLangMap.add(".PSM1", "powershell"); // Upper-case file spec
+
         command.add("--regex-powershell=/\\$(\\{[^}]+\\})/\\1/v,variable/");
         command.add("--regex-powershell=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
         command.add("--regex-powershell=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");
@@ -275,8 +289,11 @@ private void addPowerShellSupport(List<String> command) {
     }
 
     private void addPascalSupport(List<String> command) {
-        command.add("--langdef=pascal");
+        if (!env.getCtagsLanguages().contains("Pascal")) { // Built-in would be capitalized.
+            command.add("--langdef=pascal"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".PAS", "pascal"); // Upper-case file spec
+
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*\\([[:space:]]*[[:alnum:]_][[:space:]]*\\)/\\1/t,Type/");
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*class[[:space:]]*[^;]*$/\\1/c,Class/");
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*interface[[:space:]]*[^;]*$/\\1/i,interface/");
@@ -290,7 +307,9 @@ private void addPascalSupport(List<String> command) {
     }
 
     private void addSwiftSupport(List<String> command) {
-        command.add("--langdef=swift");
+        if (!env.getCtagsLanguages().contains("Swift")) { // Built-in would be capitalized.
+            command.add("--langdef=swift"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".SWIFT", "swift"); // Upper-case file spec
         command.add("--regex-swift=/enum[[:space:]]+([^\\{\\}]+).*$/\\1/n,enum,enums/");
         command.add("--regex-swift=/typealias[[:space:]]+([^:=]+).*$/\\1/t,typealias,typealiases/");
@@ -303,9 +322,12 @@ private void addSwiftSupport(List<String> command) {
     }
 
     private void addKotlinSupport(List<String> command) {
-        command.add("--langdef=kotlin");
+        if (!env.getCtagsLanguages().contains("Kotlin")) { // Built-in would be capitalized.
+            command.add("--langdef=kotlin"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".KT", "kotlin"); // Upper-case file spec
         defaultLangMap.add(".KTS", "kotlin"); // Upper-case file spec
+
         command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
                 "(private[^ ]*|protected)?[[:space:]]*class[[:space:]]+([[:alnum:]_:]+)/\\4/c,classes/");
         command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
@@ -326,8 +348,13 @@ private void addKotlinSupport(List<String> command) {
         command.add("--regex-kotlin=/^[[:space:]]*import[[:space:]]+([[:alnum:]_.:]+)/\\1/I,imports/");
     }
 
+    /**
+     * Override Clojure support with patterns from https://gist.github.com/kul/8704283.
+     */
     private void addClojureSupport(List<String> command) {
-        command.add("--langdef=clojure"); // clojure support (patterns are from https://gist.github.com/kul/8704283)
+        if (!env.getCtagsLanguages().contains("Clojure")) { // Built-in would be capitalized.
+            command.add("--langdef=clojure"); // Lower-case if user-defined.
+        }
         defaultLangMap.add(".CLJ", "clojure"); // Upper-case file spec
         defaultLangMap.add(".CLJS", "clojure"); // Upper-case file spec
         defaultLangMap.add(".CLJX", "clojure"); // Upper-case file spec
@@ -345,9 +372,12 @@ private void addClojureSupport(List<String> command) {
     }
 
     private void addHaskellSupport(List<String> command) {
-        command.add("--langdef=haskell"); // below was added with #912
+        if (!env.getCtagsLanguages().contains("Haskell")) { // Built-in would be capitalized.
+            command.add("--langdef=haskell"); // below added with #912. Lowercase if user-defined.
+        }
         defaultLangMap.add(".HS", "haskell"); // Upper-case file spec
         defaultLangMap.add(".HSC", "haskell"); // Upper-case file spec
+
         command.add("--regex-haskell=/^[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\1/c,classes/");
         command.add("--regex-haskell=/^[[:space:]]*data[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
         command.add("--regex-haskell=/^[[:space:]]*newtype[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
@@ -359,8 +389,11 @@ private void addHaskellSupport(List<String> command) {
     }
 
     private void addScalaSupport(List<String> command) {
-        command.add("--langdef=scala"); // below is bug 61 to get full scala support
+        if (!env.getCtagsLanguages().contains("Scala")) { // Built-in would be capitalized.
+            command.add("--langdef=scala"); // below is bug 61 to get full scala support. Lower-case
+        }
         defaultLangMap.add(".SCALA", "scala"); // Upper-case file spec
+
         command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
                 "(private|protected)?[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\4/c,classes/");
         command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
@@ -408,6 +441,7 @@ public Definitions doCtags(String file) throws IOException,
             }
         } else {
             initialize();
+            run();
         }
 
         CtagsReader rdr = new CtagsReader();
@@ -429,7 +463,7 @@ public Definitions doCtags(String file) throws IOException,
              * the ctags process completes so that the indexer can
              * make progress instead of hanging the whole operation.
              */
-            IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer();
+            IndexerParallelizer parallelizer = env.getIndexerParallelizer();
             ExecutorService executor = parallelizer.getCtagsWatcherExecutor();
             Future<Definitions> future = executor.submit(() -> {
                 readTags(rdr);

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/swift/SwiftAnalyzer.java

@@ -46,11 +46,11 @@ protected SwiftAnalyzer(AnalyzerFactory factory) {
     }
 
     /**
-     * @return {@code "Swift"} to match the OpenGrok-customized definitions
+     * @return {@code "swift"} to match the OpenGrok-customized definitions
      */
     @Override
     public String getCtagsLang() {
-        return "Swift";
+        return "swift";
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -34,6 +34,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -121,6 +122,11 @@ public final class RuntimeEnvironment {
 
     private transient File dtagsEftar = null;
 
+    private transient volatile Boolean ctagsFound;
+    private final transient Set<String> ctagsLanguages = new HashSet<>();
+
+    public WatchDogService watchDog;
+
     /**
      * Creates a new instance of RuntimeEnvironment. Private to ensure a
      * singleton anti-pattern.
@@ -180,8 +186,6 @@ public static RuntimeEnvironment getInstance() {
         return instance;
     }
 
-    public WatchDogService watchDog;
-
     public IndexerParallelizer getIndexerParallelizer() {
         return lzIndexerParallelizer.get();
     }
@@ -641,24 +645,42 @@ public void setHitsPerPage(int hitsPerPage) {
         setConfigurationValue("hitsPerPage", hitsPerPage);
     }
 
-    private transient Boolean ctagsFound;
-
     /**
      * Validate that there is a Universal ctags program.
      *
      * @return true if success, false otherwise
      */
     public boolean validateUniversalCtags() {
         if (ctagsFound == null) {
-            if (!CtagsUtil.validate(getCtags())) {
-                ctagsFound = false;
-            } else {
-                ctagsFound = true;
+            String ctagsBinary = getCtags();
+            configLock.writeLock().lock();
+            try {
+                if (ctagsFound == null) {
+                    ctagsFound = CtagsUtil.validate(ctagsBinary);
+                    if (ctagsFound) {
+                        List<String> languages = CtagsUtil.getLanguages(ctagsBinary);
+                        if (languages != null) {
+                            ctagsLanguages.addAll(languages);
+                        }
+                    }
+                }
+            } finally {
+                configLock.writeLock().unlock();
             }
         }
         return ctagsFound;
     }
 
+    /**
+     * Gets the base set of supported Ctags languages.
+     * @return a defined set which may be empty if
+     * {@link #validateUniversalCtags()} has not yet been called or if the call
+     * fails
+     */
+    public Set<String> getCtagsLanguages() {
+        return Collections.unmodifiableSet(ctagsLanguages);
+    }
+
     /**
      * Get the max time a SCM operation may use to avoid being cached.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

@@ -742,9 +742,7 @@ private void addFile(File file, String path, Ctags ctags)
         if (env.getCtagsTimeout() != 0) {
             ctags.setTimeout(env.getCtagsTimeout());
         }
-        if (ctags.getBinary() != null) {
-            fa.setCtags(ctags);
-        }
+        fa.setCtags(ctags);
         fa.setProject(Project.getProject(path));
         fa.setScopesEnabled(env.isScopesEnabled());
         fa.setFoldingEnabled(env.isFoldingEnabled());