Skip to content

Feature/ctags opts #2963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 8, 2019
Merged

Feature/ctags opts #2963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8904618
Add LangMap and tests
idodeclare Oct 18, 2019
973cf38
Add getCtagsLang() method
idodeclare Oct 17, 2019
6de4f5a
Use LangMap to customize Ctags from -A,--analyzer
idodeclare Oct 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AbstractAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
/*
* Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
* Portions Copyright (c) 2017-2018, Chris Fraire <cfraire@me.com>.
* Portions Copyright (c) 2017-2019, Chris Fraire <cfraire@me.com>.
*/
package org.opengrok.indexer.analysis;

Expand All @@ -34,7 +34,6 @@
import org.opengrok.indexer.configuration.Project;

/**
*
* @author Chandan
*/
public abstract class AbstractAnalyzer extends Analyzer {
Expand All @@ -52,16 +51,25 @@ public AbstractAnalyzer(ReuseStrategy reuseStrategy) {
super(reuseStrategy);
}

/**
* Subclasses should override to return the case-insensitive name aligning
* with either a built-in Universal Ctags language name or an OpenGrok
* custom language name.
* @return a defined instance or {@code null} if the analyzer has no aligned
* Universal Ctags language
*/
public abstract String getCtagsLang();

public abstract long getVersionNo();

/**
* Subclasses should override to produce a value relevant for the evolution
* of their analysis in each release.
*
* @return 0
* @return 0 since {@link AbstractAnalyzer} is not specialized
*/
protected int getSpecializedVersionNo() {
return 0; // FileAnalyzer is not specialized.
return 0;
}

public void setCtags(Ctags ctags) {
Expand Down
26 changes: 24 additions & 2 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,8 @@ public class AnalyzerGuru {
*/
private static final Map<String, Long> ANALYZER_VERSIONS = new HashMap<>();

private static final LangTreeMap langMap = new LangTreeMap();

/*
* If you write your own analyzer please register it here. The order is
* important for any factory that uses a FileAnalyzerFactory.Matcher
Expand Down Expand Up @@ -431,8 +433,10 @@ public static void addPrefix(String prefix, AnalyzerFactory factory) {
AnalyzerFactory oldFactory;
if (factory == null) {
oldFactory = pre.remove(prefix);
langMap.exclude(prefix);
} else {
oldFactory = pre.put(prefix, factory);
langMap.add(prefix, factory.getAnalyzer().getCtagsLang());
}

if (factoriesDifferent(factory, oldFactory)) {
Expand All @@ -448,21 +452,39 @@ public static void addPrefix(String prefix, AnalyzerFactory factory) {
* @param factory a factory which creates the analyzer to use for the given
* extension (if you pass null as the analyzer, you will disable the
* analyzer used for that extension)
* @throws IllegalArgumentException if {@code extension} contains a period
*/
public static void addExtension(String extension,
AnalyzerFactory factory) {
public static void addExtension(String extension, AnalyzerFactory factory) {
if (extension.contains(".")) {
throw new IllegalArgumentException("extension contains a '.'");
}

// LangMap fileSpec requires a leading period to indicate an extension.
String langMapExtension = "." + extension;

AnalyzerFactory oldFactory;
if (factory == null) {
oldFactory = ext.remove(extension);
langMap.exclude(langMapExtension);
} else {
oldFactory = ext.put(extension, factory);
langMap.add(langMapExtension, factory.getAnalyzer().getCtagsLang());
}

if (factoriesDifferent(factory, oldFactory)) {
addCustomizationKey("e:" + extension);
}
}

/**
* Gets an unmodifiable view of the language mappings resulting from
* {@link #addExtension(String, AnalyzerFactory)} and
* {@link #addPrefix(String, AnalyzerFactory)}.
*/
public static LangMap getLangMap() {
return langMap.unmodifiable();
}

/**
* Get the default Analyzer.
*
Expand Down
77 changes: 46 additions & 31 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ public class Ctags implements Resettable {
private static final Logger LOGGER = LoggerFactory.getLogger(Ctags.class);

private volatile boolean closing;
private final LangTreeMap defaultLangMap = new LangTreeMap();
private LangMap langMap;
private Process ctags;
private OutputStreamWriter ctagsIn;
private BufferedReader ctagsOut;
Expand Down Expand Up @@ -88,6 +90,10 @@ public void setBinary(String binary) {
this.binary = binary;
}

public void setLangMap(LangMap langMap) {
this.langMap = langMap;
}

public int getTabSize() {
return tabSize;
}
Expand Down Expand Up @@ -152,13 +158,15 @@ private void initialize() throws IOException {
command.add("--filter-terminator=" + CTAGS_FILTER_TERMINATOR + "\n");
command.add("--fields=-anf+iKnS");
command.add("--excmd=pattern");
command.add("--langmap=sh:+.kshlib"); // RFE #17849
command.add("--langmap=sql:+.plb"); // RFE #19208
command.add("--langmap=sql:+.pls"); // RFE #19208
command.add("--langmap=sql:+.pld"); // RFE #19208
command.add("--langmap=sql:+.pks"); // RFE #19208 ?
command.add("--langmap=sql:+.pkb"); // # 1763
command.add("--langmap=sql:+.pck"); // # 1763

defaultLangMap.clear();
defaultLangMap.add(".KSHLIB", "sh"); // RFE #17849. Upper-case file spec
defaultLangMap.add(".PLB", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PLS", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PLD", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PKS", "sql"); // RFE #19208 ? Upper-case file spec
defaultLangMap.add(".PKB", "sql"); // # 1763. Upper-case file spec
defaultLangMap.add(".PCK", "sql"); // # 1763. Upper-case file spec

//Ideally all below should be in ctags, or in outside config file,
//we might run out of command line SOON
Expand All @@ -185,6 +193,12 @@ private void initialize() throws IOException {

//PLEASE add new languages ONLY with POSIX syntax (see above wiki link)

if (langMap != null) {
command.addAll(langMap.mergeSecondary(defaultLangMap).getCtagsArgs());
} else {
command.addAll(defaultLangMap.getCtagsArgs());
}

/* Add extra command line options for ctags. */
if (CTagsExtraOptionsFile != null) {
LOGGER.log(Level.INFO, "Adding extra options to ctags");
Expand Down Expand Up @@ -229,7 +243,7 @@ private void initialize() throws IOException {

private void addRustSupport(List<String> command) {
command.add("--langdef=rust");
command.add("--langmap=rust:+.rs");
defaultLangMap.add(".RS", "rust"); // Upper-case file spec

// The following are not supported yet in Universal Ctags b13cb551
command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?(static|const)[[:space:]]+(mut[[:space:]]+)?" +
Expand All @@ -242,26 +256,27 @@ private void addRustSupport(List<String> command) {
}

private void addPowerShellSupport(List<String> command) {
command.add("--langdef=Posh");
command.add("--langmap=Posh:+.ps1,Posh:+.psm1");
command.add("--regex-Posh=/\\$(\\{[^}]+\\})/\\1/v,variable/");
command.add("--regex-Posh=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
command.add("--regex-Posh=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");

command.add("--_fielddef-Posh=signature,signatures");
command.add("--fields-Posh=+{signature}");
command.add("--langdef=powershell");
defaultLangMap.add(".PS1", "powershell"); // Upper-case file spec
defaultLangMap.add(".PSM1", "powershell"); // Upper-case file spec
command.add("--regex-powershell=/\\$(\\{[^}]+\\})/\\1/v,variable/");
command.add("--regex-powershell=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
command.add("--regex-powershell=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");

command.add("--_fielddef-powershell=signature,signatures");
command.add("--fields-powershell=+{signature}");
// escaped variable markers
command.add("--regex-Posh=/`\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
command.add("--regex-Posh=/`\\$(\\{[^}]+\\})/\\1//{exclusive}");
command.add("--regex-Posh=/#.*\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
command.add("--regex-Posh=/#.*\\$(\\{[^}]+\\})/\\1//{exclusive}");
command.add("--regex-Posh=/^[[:space:]]*(function|filter)[[:space:]]+([^({[:space:]]+)[[:space:]]*" +
command.add("--regex-powershell=/`\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
command.add("--regex-powershell=/`\\$(\\{[^}]+\\})/\\1//{exclusive}");
command.add("--regex-powershell=/#.*\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
command.add("--regex-powershell=/#.*\\$(\\{[^}]+\\})/\\1//{exclusive}");
command.add("--regex-powershell=/^[[:space:]]*(function|filter)[[:space:]]+([^({[:space:]]+)[[:space:]]*" +
"(\\(([^)]+)\\))?/\\2/f,function,functions/{icase}{exclusive}{_field=signature:(\\4)}");
}

private void addPascalSupport(List<String> command) {
command.add("--langdef=pascal");
command.add("--langmap=pascal:+.pas");
defaultLangMap.add(".PAS", "pascal"); // Upper-case file spec
command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*\\([[:space:]]*[[:alnum:]_][[:space:]]*\\)/\\1/t,Type/");
command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*class[[:space:]]*[^;]*$/\\1/c,Class/");
command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*interface[[:space:]]*[^;]*$/\\1/i,interface/");
Expand All @@ -276,7 +291,7 @@ private void addPascalSupport(List<String> command) {

private void addSwiftSupport(List<String> command) {
command.add("--langdef=swift");
command.add("--langmap=swift:+.swift");
defaultLangMap.add(".SWIFT", "swift"); // Upper-case file spec
command.add("--regex-swift=/enum[[:space:]]+([^\\{\\}]+).*$/\\1/n,enum,enums/");
command.add("--regex-swift=/typealias[[:space:]]+([^:=]+).*$/\\1/t,typealias,typealiases/");
command.add("--regex-swift=/protocol[[:space:]]+([^:\\{]+).*$/\\1/p,protocol,protocols/");
Expand All @@ -289,8 +304,8 @@ private void addSwiftSupport(List<String> command) {

private void addKotlinSupport(List<String> command) {
command.add("--langdef=kotlin");
command.add("--langmap=kotlin:+.kt");
command.add("--langmap=kotlin:+.kts");
defaultLangMap.add(".KT", "kotlin"); // Upper-case file spec
defaultLangMap.add(".KTS", "kotlin"); // Upper-case file spec
command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
"(private[^ ]*|protected)?[[:space:]]*class[[:space:]]+([[:alnum:]_:]+)/\\4/c,classes/");
command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
Expand All @@ -313,9 +328,9 @@ private void addKotlinSupport(List<String> command) {

private void addClojureSupport(List<String> command) {
command.add("--langdef=clojure"); // clojure support (patterns are from https://gist.github.com/kul/8704283)
command.add("--langmap=clojure:+.clj");
command.add("--langmap=clojure:+.cljs");
command.add("--langmap=clojure:+.cljx");
defaultLangMap.add(".CLJ", "clojure"); // Upper-case file spec
defaultLangMap.add(".CLJS", "clojure"); // Upper-case file spec
defaultLangMap.add(".CLJX", "clojure"); // Upper-case file spec

command.add("--regex-clojure=/\\([[:space:]]*create-ns[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/n,namespace/");
command.add("--regex-clojure=/\\([[:space:]]*def[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/d,definition/");
Expand All @@ -331,8 +346,8 @@ private void addClojureSupport(List<String> command) {

private void addHaskellSupport(List<String> command) {
command.add("--langdef=haskell"); // below was added with #912
command.add("--langmap=haskell:+.hs");
command.add("--langmap=haskell:+.hsc");
defaultLangMap.add(".HS", "haskell"); // Upper-case file spec
defaultLangMap.add(".HSC", "haskell"); // Upper-case file spec
command.add("--regex-haskell=/^[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\1/c,classes/");
command.add("--regex-haskell=/^[[:space:]]*data[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
command.add("--regex-haskell=/^[[:space:]]*newtype[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
Expand All @@ -345,7 +360,7 @@ private void addHaskellSupport(List<String> command) {

private void addScalaSupport(List<String> command) {
command.add("--langdef=scala"); // below is bug 61 to get full scala support
command.add("--langmap=scala:+.scala");
defaultLangMap.add(".SCALA", "scala"); // Upper-case file spec
command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
"(private|protected)?[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\4/c,classes/");
command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
Expand Down
10 changes: 9 additions & 1 deletion opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/FileAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
/*
* Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
* Portions Copyright (c) 2017-2018, Chris Fraire <cfraire@me.com>.
* Portions Copyright (c) 2017-2019, Chris Fraire <cfraire@me.com>.
*/
package org.opengrok.indexer.analysis;

Expand Down Expand Up @@ -59,6 +59,14 @@ public class FileAnalyzer extends AbstractAnalyzer {

private static final Logger LOGGER = LoggerFactory.getLogger(FileAnalyzer.class);

/**
* @return {@code null} as there is no aligned language
*/
@Override
public String getCtagsLang() {
return null;
}

/**
* Gets a version number to be used to tag processed documents so that
* re-analysis can be re-done later if a stored version number is different
Expand Down
89 changes: 89 additions & 0 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/LangMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2019, Chris Fraire <cfraire@me.com>.
*/

package org.opengrok.indexer.analysis;

import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Represents an API for mapping file specifications versus languages and
* getting the ctags options representation (--langmap-&lt;LANG&gt; or
* --map-&lt;LANG&gt;) thereof.
*/
public interface LangMap {

/**
* Removes all settings from this map.
*/
void clear();

/**
* Adds the specified mapping of a file specification to a language. Any
* matching exclusion via {@link #exclude(String)} is undone.
* @param fileSpec a value starting with a period ({@code '.'}) to specify
* a file extension; otherwise specifying a prefix.
* @throws IllegalArgumentException if {@code fileSpec} is {@code null} or
* is an extension (i.e. starting with a period) but contains any other
* periods, as that is not ctags-compatible
*/
void add(String fileSpec, String ctagsLang);

/**
* Exclude the specified mapping of a file specification to any language.
* Any matching addition via {@link #add(String, String)} is undone.
* @throws IllegalArgumentException if {@code fileSpec} is {@code null}
*/
void exclude(String fileSpec);

/**
* Gets the transformation of the instance's mappings to ctags arguments.
*/
List<String> getCtagsArgs();

/**
* Creates a new instance, merging the settings from the current instance
* overlaying a specified {@code other}. Additions from the current instance
* take precedence, and exclusions from the {@code other} only take effect
* if the current instance has no matching addition.
* @param other a defined instance
* @return a defined instance
*/
LangMap mergeSecondary(LangMap other);

/**
* Gets an unmodifiable view of the current instance.
*/
LangMap unmodifiable();

/**
* Gets an unmodifiable view of the current additions.
*/
Map<String, String> getAdditions();

/**
* Gets an unmodifiable view of the current exclusions.
*/
Set<String> getExclusions();
}
Loading