optimize section body extraction and error handling

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -246,7 +246,7 @@ const mergeInterruptionIntent = function (fromUtterances, toResource, intentName
     // construct new content here
     const dedupUtterances = dedupFromUtterances.filter(u => !existingUtterances.includes(u))
     if (dedupUtterances && dedupUtterances.length > 0) {
-      let newFileContent = `> Source: cross training. Please do not edit these directly!${NEWLINE}# ${intentName}${NEWLINE}- `
+      let newFileContent = `${NEWLINE}> Source: cross training. Please do not edit these directly!${NEWLINE}# ${intentName}${NEWLINE}- `
       newFileContent += dedupUtterances.join(`${NEWLINE}- `)
 
       // add section here
@@ -383,7 +383,7 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
 
   // add questions from qna file to corresponding lu file with intent named DeferToRecognizer_QnA_${fileName}
   if (questionsContent && questionsContent !== '' && utterances.length > 0) {
-    const questionsToUtterances = `${crossTrainingComments}${NEWLINE}# DeferToRecognizer_QnA_${fileName}${NEWLINE}${questionsContent}`
+    const questionsToUtterances = `${NEWLINE}${crossTrainingComments}${NEWLINE}# DeferToRecognizer_QnA_${fileName}${NEWLINE}${questionsContent}`
     trainedLuResource = new SectionOperator(trainedLuResource).addSection(questionsToUtterances)
   }
 
@@ -413,7 +413,7 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
   if (qnaContents && qnaContents !== '') {
     const modelInfoSections = qnaResource.Sections.filter(s => s.SectionType === LUSectionTypes.MODELINFOSECTION)
     const modelInforContent = modelInfoSections.map(m => m.ModelInfo).join(NEWLINE)
-    trainedQnaResource = new SectionOperator(new LUResource([], modelInforContent, [])).addSection(qnaContents)
+    trainedQnaResource = new SectionOperator(new LUResource([], modelInforContent, [])).addSection(NEWLINE + qnaContents)
   }
 
   // remove utterances which are duplicated with local qna questions
@@ -424,7 +424,7 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
 
   // add utterances from lu file to corresponding qna file with question set to all utterances
   if (utterancesContent && utterancesContent !== '' && qnaSections.length > 0) {
-    const utterancesToQuestion = `${crossTrainingComments}${NEWLINE}# ? ${utterancesContent}${NEWLINE}${NEWLINE}**Filters:**${NEWLINE}- dialogName=${fileName}${NEWLINE}${NEWLINE}\`\`\`${NEWLINE}intent=DeferToRecognizer_LUIS_${fileName}${NEWLINE}\`\`\``
+    const utterancesToQuestion = `${NEWLINE}${crossTrainingComments}${NEWLINE}# ? ${utterancesContent}${NEWLINE}${NEWLINE}**Filters:**${NEWLINE}- dialogName=${fileName}${NEWLINE}${NEWLINE}\`\`\`${NEWLINE}intent=DeferToRecognizer_LUIS_${fileName}${NEWLINE}\`\`\``
     trainedQnaResource = new SectionOperator(trainedQnaResource).addSection(utterancesToQuestion)
   }
 

packages/lu/src/parser/lufile/LUFileParser.g4

@@ -22,6 +22,10 @@ newline
     : WS* (NEWLINE | EOF)
     ;
 
+errorString
+    : (WS|INVALID_TOKEN_DEFAULT_MODE)+
+    ;
+
 nestedIntentSection
     : nestedIntentNameLine nestedIntentBodyDefinition
     ;
@@ -67,17 +71,13 @@ intentBody
 	;
 
 normalIntentBody
-    : WS* ((normalIntentString newline) | errorIntentString)+
+    : WS* ((normalIntentString newline) | errorString)+
     ;
 
 normalIntentString
 	: WS* DASH (WS|TEXT|EXPRESSION|ESCAPE_CHARACTER)*
 	;
 
-errorIntentString
-    : (WS|INVALID_TOKEN_DEFAULT_MODE)+
-    ;
-
 newEntitySection
     : newEntityDefinition
     ;
@@ -87,11 +87,11 @@ newEntityDefinition
     ;
 
 newEntityListbody
-    : ((normalItemString newline) | errorItemString)+
+    : ((normalItemString newline) | errorString)+
     ;
 
 newEntityLine
-    : WS* AT WS* newEntityType? WS* (newEntityName|newEntityNameWithWS) WS* newEntityRoles? WS* newEntityUsesFeatures? WS* EQUAL? WS* (newCompositeDefinition|newRegexDefinition)? newline
+    : WS* AT WS* newEntityType? WS* (newEntityName|newEntityNameWithWS)? WS* newEntityRoles? WS* newEntityUsesFeatures? WS* EQUAL? WS* (newCompositeDefinition|newRegexDefinition)? newline
     ;
 
 newCompositeDefinition
@@ -135,7 +135,7 @@ entityDefinition
     ;
 
 entityLine
-    : WS* DOLLAR entityName COLON_MARK entityType
+    : WS* DOLLAR (entityName COLON_MARK entityType)?
     ;
 
 entityName
@@ -155,17 +155,13 @@ regexEntityIdentifier
     ;
 
 entityListBody
-    : ((normalItemString newline) | errorItemString)+
+    : ((normalItemString newline) | errorString)+
     ;
 
 normalItemString
     : WS* DASH (WS|TEXT|EXPRESSION|ESCAPE_CHARACTER)*
     ;
 
-errorItemString
-    : (WS|INVALID_TOKEN_DEFAULT_MODE)+
-    ;
-
 importSection
     : importDefinition
     ;

packages/lu/src/parser/lufile/entitySection.js

@@ -11,30 +11,42 @@ class EntitySection {
     constructor(parseTree) {
         this.ParseTree = parseTree;
         this.SectionType = LUSectionTypes.ENTITYSECTION;
+        this.Errors = []
         this.Name = this.ExtractName(parseTree);
         this.Type = this.ExtractType(parseTree);
-        const result = this.ExtractSynonymsOrPhraseList(parseTree);
-        this.SynonymsOrPhraseList = result.synonymsOrPhraseList;
-        this.Errors = result.errors;
+        this.SynonymsOrPhraseList = this.ExtractSynonymsOrPhraseList(parseTree);
         this.Id = `${this.SectionType}_${this.Name}`;
     }
 
     ExtractName(parseTree) {
-        return parseTree.entityDefinition().entityLine().entityName().getText().trim();
+        if (parseTree.entityDefinition().entityLine().entityName()) {
+            return parseTree.entityDefinition().entityLine().entityName().getText().trim();
+        } else {
+            this.Errors.push(BuildDiagnostic({
+                message: "Invalid entity line, did you miss entity name after $",
+                context: parseTree.entityDefinition().entityLine()
+            }))     
+        }
     }
 
     ExtractType(parseTree) {
-        return parseTree.entityDefinition().entityLine().entityType().getText().trim();
+        if (parseTree.entityDefinition().entityLine().entityType()) {
+            return parseTree.entityDefinition().entityLine().entityType().getText().trim();
+        } else {
+            this.Errors.push(BuildDiagnostic({
+                message: "Invalid entity line, did you miss entity type after $",
+                context: parseTree.entityDefinition().entityLine()
+            }))   
+        }
     }
 
     ExtractSynonymsOrPhraseList(parseTree) {
         let synonymsOrPhraseList = [];
-        let errors = [];
 
         if (parseTree.entityDefinition().entityListBody()) {
-            for (const errorItemStr of parseTree.entityDefinition().entityListBody().errorItemString()) {
+            for (const errorItemStr of parseTree.entityDefinition().entityListBody().errorString()) {
                 if (errorItemStr.getText().trim() !== '') {
-                    errors.push(BuildDiagnostic({
+                    this.Errors.push(BuildDiagnostic({
                     message: "Invalid list entity line, did you miss '-' at line begin",
                     context: errorItemStr
                 }))}
@@ -46,18 +58,18 @@ class EntitySection {
             }
         }
 
-        if (this.Type.indexOf('=') > -1 && synonymsOrPhraseList.length === 0) {
+        if (this.Type && this.Type.indexOf('=') > -1 && synonymsOrPhraseList.length === 0) {
             let errorMsg = `no synonyms list found for list entity definition: "${parseTree.entityDefinition().entityLine().getText()}"`;
             let error = BuildDiagnostic({
                 message: errorMsg,
                 context: parseTree.entityDefinition().entityLine(),
                 severity: DiagnosticSeverity.WARN
             })
 
-            errors.push(error);
+            this.Errors.push(error);
         }
 
-        return { synonymsOrPhraseList, errors };
+        return synonymsOrPhraseList;
     }
 }