Skip to content

Commit

Permalink
feat: JDK 22 string templates
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtkiesel
jtkiesel committed Jan 22, 2024
1 parent 983a09d commit 2bad5eb
Show file tree
Hide file tree
Showing 10 changed files with 490 additions and 27 deletions.
65 changes: 65 additions & 0 deletions packages/java-parser/api.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,11 @@ export abstract class JavaCstVisitor<IN, OUT> implements ICstVisitor<IN, OUT> {
classLiteralSuffix(ctx: ClassLiteralSuffixCtx, param?: IN): OUT;
arrayAccessSuffix(ctx: ArrayAccessSuffixCtx, param?: IN): OUT;
methodReferenceSuffix(ctx: MethodReferenceSuffixCtx, param?: IN): OUT;
templateArgument(ctx: TemplateArgumentCtx, param?: IN): OUT;
template(ctx: TemplateCtx, param?: IN): OUT;
stringTemplate(ctx: StringTemplateCtx, param?: IN): OUT;
textBlockTemplate(ctx: TextBlockTemplateCtx, param?: IN): OUT;
embeddedExpression(ctx: EmbeddedExpressionCtx, param?: IN): OUT;
pattern(ctx: PatternCtx, param?: IN): OUT;
typePattern(ctx: TypePatternCtx, param?: IN): OUT;
recordPattern(ctx: RecordPatternCtx, param?: IN): OUT;
Expand Down Expand Up @@ -672,6 +677,11 @@ export abstract class JavaCstVisitorWithDefaults<IN, OUT>
classLiteralSuffix(ctx: ClassLiteralSuffixCtx, param?: IN): OUT;
arrayAccessSuffix(ctx: ArrayAccessSuffixCtx, param?: IN): OUT;
methodReferenceSuffix(ctx: MethodReferenceSuffixCtx, param?: IN): OUT;
templateArgument(ctx: TemplateArgumentCtx, param?: IN): OUT;
template(ctx: TemplateCtx, param?: IN): OUT;
stringTemplate(ctx: StringTemplateCtx, param?: IN): OUT;
textBlockTemplate(ctx: TextBlockTemplateCtx, param?: IN): OUT;
embeddedExpression(ctx: EmbeddedExpressionCtx, param?: IN): OUT;
pattern(ctx: PatternCtx, param?: IN): OUT;
typePattern(ctx: TypePatternCtx, param?: IN): OUT;
recordPattern(ctx: RecordPatternCtx, param?: IN): OUT;
Expand Down Expand Up @@ -2999,6 +3009,7 @@ export type PrimarySuffixCtx = {
unqualifiedClassInstanceCreationExpression?: UnqualifiedClassInstanceCreationExpressionCstNode[];
typeArguments?: TypeArgumentsCstNode[];
Identifier?: IToken[];
templateArgument?: TemplateArgumentCstNode[];
methodInvocationSuffix?: MethodInvocationSuffixCstNode[];
classLiteralSuffix?: ClassLiteralSuffixCstNode[];
arrayAccessSuffix?: ArrayAccessSuffixCstNode[];
Expand Down Expand Up @@ -3264,6 +3275,60 @@ export type MethodReferenceSuffixCtx = {
New?: IToken[];
};

export interface TemplateArgumentCstNode extends CstNode {
name: "templateArgument";
children: TemplateArgumentCtx;
}

export type TemplateArgumentCtx = {
template?: TemplateCstNode[];
StringLiteral?: IToken[];
TextBlock?: IToken[];
};

export interface TemplateCstNode extends CstNode {
name: "template";
children: TemplateCtx;
}

export type TemplateCtx = {
stringTemplate?: StringTemplateCstNode[];
textBlockTemplate?: TextBlockTemplateCstNode[];
};

export interface StringTemplateCstNode extends CstNode {
name: "stringTemplate";
children: StringTemplateCtx;
}

export type StringTemplateCtx = {
StringTemplateBegin: IToken[];
embeddedExpression: EmbeddedExpressionCstNode[];
StringTemplateMid?: IToken[];
StringTemplateEnd: IToken[];
};

export interface TextBlockTemplateCstNode extends CstNode {
name: "textBlockTemplate";
children: TextBlockTemplateCtx;
}

export type TextBlockTemplateCtx = {
TextBlockTemplateBegin: IToken[];
embeddedExpression: EmbeddedExpressionCstNode[];
TextBlockTemplateMid?: IToken[];
TextBlockTemplateEnd: IToken[];
};

export interface EmbeddedExpressionCstNode extends CstNode {
name: "embeddedExpression";
children: EmbeddedExpressionCtx;
}

export type EmbeddedExpressionCtx = {
expression?: ExpressionCstNode[];
};

export interface PatternCstNode extends CstNode {
name: "pattern";
children: PatternCtx;
Expand Down
65 changes: 57 additions & 8 deletions packages/java-parser/src/productions/expressions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ export function defineRules($, t) {
});
$.CONSUME(t.Identifier);
}
}
},
{ ALT: () => $.SUBRULE($.templateArgument) }
]);
}
},
Expand All @@ -247,13 +248,20 @@ export function defineRules($, t) {
$.RULE("fqnOrRefType", () => {
$.SUBRULE($.fqnOrRefTypePartFirst);

$.MANY2({
// ".class" is a classLiteralSuffix
GATE: () =>
// avoids ambiguity with ".this" and ".new" which are parsed as a primary suffix.
tokenMatcher(this.LA(2).tokenType, t.Class) === false &&
tokenMatcher(this.LA(2).tokenType, t.This) === false &&
tokenMatcher(this.LA(2).tokenType, t.New) === false,
$.MANY({
// avoids ambiguity with primary suffixes
GATE: () => {
const nextNextToken = $.LA(2);
return !(
tokenMatcher(nextNextToken, t.Class) ||
tokenMatcher(nextNextToken, t.This) ||
tokenMatcher(nextNextToken, t.New) ||
tokenMatcher(nextNextToken, t.StringLiteral) ||
tokenMatcher(nextNextToken, t.TextBlock) ||
tokenMatcher(nextNextToken, t.StringTemplateBegin) ||
tokenMatcher(nextNextToken, t.TextBlockTemplateBegin)
);
},
DEF: () => {
$.CONSUME(t.Dot);
$.SUBRULE2($.fqnOrRefTypePartRest);
Expand Down Expand Up @@ -505,6 +513,47 @@ export function defineRules($, t) {
]);
});

$.RULE("templateArgument", () => {
$.OR([
{ ALT: () => $.SUBRULE($.template) },
{ ALT: () => $.CONSUME(t.StringLiteral) },
{ ALT: () => $.CONSUME(t.TextBlock) }
]);
});

$.RULE("template", () => {
$.OR([
{ ALT: () => $.SUBRULE($.stringTemplate) },
{ ALT: () => $.SUBRULE($.textBlockTemplate) }
]);
});

$.RULE("stringTemplate", () => {
$.CONSUME(t.StringTemplateBegin);
$.SUBRULE($.embeddedExpression);
$.MANY(() => {
$.CONSUME(t.StringTemplateMid);
$.SUBRULE1($.embeddedExpression);
});
$.CONSUME(t.StringTemplateEnd);
});

$.RULE("textBlockTemplate", () => {
$.CONSUME(t.TextBlockTemplateBegin);
$.SUBRULE($.embeddedExpression);
$.MANY(() => {
$.CONSUME(t.TextBlockTemplateMid);
$.SUBRULE1($.embeddedExpression);
});
$.CONSUME(t.TextBlockTemplateEnd);
});

$.RULE("embeddedExpression", () => {
$.OPTION(() => {
$.SUBRULE($.expression);
});
});

// https://docs.oracle.com/javase/specs/jls/se21/html/jls-14.html#jls-Pattern
$.RULE("pattern", () => {
$.OR([
Expand Down
86 changes: 77 additions & 9 deletions packages/java-parser/src/tokens.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,11 @@ FRAGMENT("EscapeSequence", "\\\\[bstnfr\"'\\\\]|{{OctalEscape}}");
// Not using InputCharacter terminology there because CR and LF are already captured in EscapeSequence
FRAGMENT(
"StringCharacter",
"(?:(?:{{EscapeSequence}})|{{UnicodeInputCharacter}})"
'(?:(?:{{EscapeSequence}})|{{UnicodeEscape}}|(?!["\\\\]).)'
);
FRAGMENT(
"TextBlockCharacter",
"(?:(?:{{EscapeSequence}})|{{UnicodeEscape}}|(?!\\\\).|\\\\?{{LineTerminator}})"
);

function matchJavaIdentifier(text, startOffset) {
Expand Down Expand Up @@ -93,10 +97,18 @@ const Identifier = createTokenOrg({
)
});

const allTokens = [];
const allTokens = {
modes: {
global: [],
stringTemplate: [],
textBlockTemplate: []
},
defaultMode: "global"
};
const allModes = Object.keys(allTokens.modes);
const tokenDictionary = {};

function createToken(options) {
function createToken(options, modes = allModes) {
// TODO create a test to check all the tokenbs have a label defined
if (!options.label) {
// simple token (e.g operator)
Expand All @@ -110,7 +122,7 @@ function createToken(options) {
}

const newTokenType = createTokenOrg(options);
allTokens.push(newTokenType);
modes.forEach(mode => allTokens.modes[mode].push(newTokenType));
tokenDictionary[options.name] = newTokenType;
return newTokenType;
}
Expand Down Expand Up @@ -221,14 +233,62 @@ createToken({

createToken({
name: "TextBlock",
pattern: /"""\s*\n(\\"|\s|.)*?"""/
pattern: MAKE_PATTERN(
'"""[\\x09\\x20\\x0C]*{{LineTerminator}}{{TextBlockCharacter}}*?"""'
)
});

createToken({
name: "TextBlockTemplateBegin",
pattern: MAKE_PATTERN('"""{{LineTerminator}}{{TextBlockCharacter}}*?\\\\\\{'),
push_mode: "textBlockTemplate"
});

createToken(
{
name: "TextBlockTemplateEnd",
pattern: MAKE_PATTERN('\\}{{TextBlockCharacter}}*?"""'),
pop_mode: true
},
["textBlockTemplate"]
);

createToken({
name: "StringLiteral",
pattern: MAKE_PATTERN('"(?:[^\\\\"]|{{StringCharacter}})*"')
pattern: MAKE_PATTERN('"{{StringCharacter}}*?"')
});

createToken({
name: "StringTemplateBegin",
pattern: MAKE_PATTERN('"{{StringCharacter}}*?\\\\\\{'),
push_mode: "stringTemplate"
});

createToken(
{
name: "StringTemplateEnd",
pattern: MAKE_PATTERN('\\}{{StringCharacter}}*?"'),
pop_mode: true
},
["stringTemplate"]
);

createToken(
{
name: "StringTemplateMid",
pattern: MAKE_PATTERN("\\}{{StringCharacter}}*?\\\\\\{")
},
["stringTemplate"]
);

createToken(
{
name: "TextBlockTemplateMid",
pattern: MAKE_PATTERN("\\}{{TextBlockCharacter}}*?\\\\\\{")
},
["textBlockTemplate"]
);

// https://docs.oracle.com/javase/specs/jls/se21/html/jls-3.html#jls-3.9
// TODO: how to handle the special rule (see spec above) for "requires" and "transitive"
const restrictedKeywords = [
Expand Down Expand Up @@ -376,8 +436,16 @@ createToken({ name: "Colon", pattern: ":" });
createToken({ name: "QuestionMark", pattern: "?" });
createToken({ name: "LBrace", pattern: "(", categories: [Separators] });
createToken({ name: "RBrace", pattern: ")", categories: [Separators] });
createToken({ name: "LCurly", pattern: "{", categories: [Separators] });
createToken({ name: "RCurly", pattern: "}", categories: [Separators] });
createToken({
name: "LCurly",
pattern: "{",
categories: [Separators],
push_mode: allTokens.defaultMode
});
createToken(
{ name: "RCurly", pattern: "}", categories: [Separators], pop_mode: true },
[allTokens.defaultMode]
);
createToken({ name: "LSquare", pattern: "[", categories: [Separators] });
createToken({ name: "RSquare", pattern: "]", categories: [Separators] });

Expand Down Expand Up @@ -514,7 +582,7 @@ createToken({

// Identifier must appear AFTER all the keywords to avoid ambiguities.
// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
allTokens.push(Identifier);
allModes.forEach(mode => allTokens.modes[mode].push(Identifier));
tokenDictionary["Identifier"] = Identifier;

function sortDescLength(arr) {
Expand Down
Loading

0 comments on commit 2bad5eb

Please sign in to comment.