Skip to content

cache Antlr parse result #3621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 27, 2020
Merged

cache Antlr parse result #3621

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bfcdb0a
add cache in parser
Danieladu Mar 26, 2020
52beef3
make some renaming
Danieladu Mar 26, 2020
5b0d31f
Merge branch 'master' into hond/parse-perf
Danieladu Mar 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 69 additions & 34 deletions libraries/Microsoft.Bot.Builder.LanguageGeneration/TemplatesParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public static Templates ParseFile(
var fullPath = Path.GetFullPath(filePath.NormalizePath());
var content = File.ReadAllText(fullPath);

return ParseText(content, fullPath, importResolver, expressionParser);
return InnerParseText(content, fullPath, importResolver, expressionParser);
}

/// <summary>
Expand All @@ -61,20 +61,39 @@ public static Templates ParseText(
ImportResolverDelegate importResolver = null,
ExpressionParser expressionParser = null)
{
importResolver = importResolver ?? DefaultFileResolver;
var lg = new Templates(content: content, id: id, importResolver: importResolver, expressionParser: expressionParser);
return InnerParseText(content, id, importResolver, expressionParser);
}

/// <summary>
/// Parser to turn lg content into a <see cref="Templates"/> based on the original LGFile.
/// </summary>
/// <param name="content">Text content contains lg templates.</param>
/// <param name="lg">original LGFile.</param>
/// <returns>new <see cref="Templates"/> entity.</returns>
public static Templates ParseTextWithRef(string content, Templates lg)
{
if (lg == null)
{
throw new ArgumentNullException(nameof(lg));
}

var id = "inline content";
var newLG = new Templates(content: content, id: id, importResolver: lg.ImportResolver, options: lg.Options);
var diagnostics = new List<Diagnostic>();
try
{
var (templates, imports, invalidTemplateErrors, options) = AntlrParse(content, id);
lg.AddRange(templates);
lg.Imports = imports;
lg.Options = options;
newLG.AddRange(templates);
newLG.Imports = imports;
newLG.Options = options;
diagnostics.AddRange(invalidTemplateErrors);

lg.References = GetReferences(lg, importResolver);
var semanticErrors = new StaticChecker(lg).Check();
newLG.References = GetReferences(newLG)
.Union(lg.References)
.Union(new List<Templates> { lg })
.ToList();

var semanticErrors = new StaticChecker(newLG).Check();
diagnostics.AddRange(semanticErrors);
}
catch (TemplateException ex)
Expand All @@ -86,41 +105,47 @@ public static Templates ParseText(
diagnostics.Add(BuildDiagnostic(err.Message, source: id));
}

lg.Diagnostics = diagnostics;
newLG.Diagnostics = diagnostics;

return lg;
return newLG;
}

/// <summary>
/// Parser to turn lg content into a <see cref="Templates"/> based on the original LGFile.
/// Parser to turn lg content into a <see cref="Templates"/>.
/// </summary>
/// <param name="content">Text content contains lg templates.</param>
/// <param name="lg">original LGFile.</param>
/// <param name="id">id is the identifier of content. If importResolver is null, id must be a full path string. </param>
/// <param name="importResolver">resolver to resolve LG import id to template text.</param>
/// <param name="expressionParser">expressionEngine parser engine for parsing expressions.</param>
/// <param name="cachedTemplates">give the file path and templates to avoid parsing and to improve performance.</param>
/// <returns>new <see cref="Templates"/> entity.</returns>
public static Templates ParseTextWithRef(string content, Templates lg)
private static Templates InnerParseText(
string content,
string id = "",
ImportResolverDelegate importResolver = null,
ExpressionParser expressionParser = null,
Dictionary<string, Templates> cachedTemplates = null)
{
if (lg == null)
cachedTemplates = cachedTemplates ?? new Dictionary<string, Templates>();
if (cachedTemplates.ContainsKey(id))
{
throw new ArgumentNullException(nameof(lg));
return cachedTemplates[id];
}

var id = "inline content";
var newLG = new Templates(content: content, id: id, importResolver: lg.ImportResolver, options: lg.Options);
importResolver = importResolver ?? DefaultFileResolver;
var lg = new Templates(content: content, id: id, importResolver: importResolver, expressionParser: expressionParser);

var diagnostics = new List<Diagnostic>();
try
{
var (templates, imports, invalidTemplateErrors, options) = AntlrParse(content, id);
newLG.AddRange(templates);
newLG.Imports = imports;
newLG.Options = options;
lg.AddRange(templates);
lg.Imports = imports;
lg.Options = options;
diagnostics.AddRange(invalidTemplateErrors);

newLG.References = GetReferences(newLG, newLG.ImportResolver)
.Union(lg.References)
.Union(new List<Templates> { lg })
.ToList();

var semanticErrors = new StaticChecker(newLG).Check();
lg.References = GetReferences(lg, cachedTemplates);
var semanticErrors = new StaticChecker(lg).Check();
diagnostics.AddRange(semanticErrors);
}
catch (TemplateException ex)
Expand All @@ -132,9 +157,9 @@ public static Templates ParseTextWithRef(string content, Templates lg)
diagnostics.Add(BuildDiagnostic(err.Message, source: id));
}

newLG.Diagnostics = diagnostics;
lg.Diagnostics = diagnostics;

return newLG;
return lg;
}

/// <summary>
Expand Down Expand Up @@ -280,16 +305,16 @@ private static IList<TemplateImport> ExtractLGImports(LGFileParser.FileContext f
.ToList();
}

private static IList<Templates> GetReferences(Templates file, ImportResolverDelegate importResolver)
private static IList<Templates> GetReferences(Templates file, Dictionary<string, Templates> cachedTemplates = null)
{
var resourcesFound = new HashSet<Templates>();
ResolveImportResources(file, resourcesFound, importResolver);
ResolveImportResources(file, resourcesFound, cachedTemplates ?? new Dictionary<string, Templates>());

resourcesFound.Remove(file);
return resourcesFound.ToList();
}

private static void ResolveImportResources(Templates start, HashSet<Templates> resourcesFound, ImportResolverDelegate importResolver)
private static void ResolveImportResources(Templates start, HashSet<Templates> resourcesFound, Dictionary<string, Templates> cachedTemplates)
{
var resourceIds = start.Imports.Select(lg => lg.Id);
resourcesFound.Add(start);
Expand All @@ -298,11 +323,21 @@ private static void ResolveImportResources(Templates start, HashSet<Templates> r
{
try
{
var (content, path) = importResolver(start.Id, id);
var (content, path) = start.ImportResolver(start.Id, id);
if (resourcesFound.All(u => u.Id != path))
{
var childResource = ParseText(content, path, importResolver, start.ExpressionParser);
ResolveImportResources(childResource, resourcesFound, importResolver);
Templates childResource;
if (cachedTemplates.ContainsKey(path))
{
childResource = cachedTemplates[path];
}
else
{
childResource = InnerParseText(content, path, start.ImportResolver, start.ExpressionParser, cachedTemplates);
cachedTemplates.Add(path, childResource);
}

ResolveImportResources(childResource, resourcesFound, cachedTemplates);
}
}
catch (TemplateException err)
Expand Down