CRUD optimizations for qna sections

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -439,8 +439,8 @@ const qnaCrossTrainCore = function (luResource, qnaResource, fileName, interrupt
       qnaSectionContent += `> !# @qna.pair.source = ${qnaSection.source}${NEWLINE}${NEWLINE}`
     }
 
-    if (qnaSection.Id) {
-      qnaSectionContent += `<a id = "${qnaSection.Id}"></a>${NEWLINE}${NEWLINE}`
+    if (qnaSection.QAPairId) {
+      qnaSectionContent += `<a id = "${qnaSection.QAPairId}"></a>${NEWLINE}${NEWLINE}`
     }
 
     qnaSectionContent += `# ? ${Array.from(new Set(qnaSection.Questions)).join(NEWLINE + '- ')}${NEWLINE}${NEWLINE}**Filters:**${NEWLINE}- ${qnaSection.FilterPairs.map(f => f.key + '=' + f.value).join(NEWLINE + '- ')}${NEWLINE}${NEWLINE}\`\`\`${NEWLINE}${qnaSection.Answer}${NEWLINE}\`\`\``

packages/lu/src/parser/lufile/luParser.js

@@ -124,7 +124,7 @@ class LUParser {
             }))
         }
 
-        this.extractIntentBody(sections, content)
+        this.extractSectionBody(sections, content)
 
         return new LUResource(sections, content, errors);
     }
@@ -300,11 +300,14 @@ class LUParser {
      * @param {any[]} sections
      * @param {string} content
      */
-    static extractIntentBody(sections, content) {
+    static extractSectionBody(sections, content) {
         sections.sort((a, b) => a.ParseTree.start.line - b.ParseTree.start.line)
         const originList = content.split(/\r?\n/)
+        let qnaSectionIndex = 0
         sections.forEach(function (section, index) {
-            if (section.SectionType === SectionType.SIMPLEINTENTSECTION || section.SectionType === SectionType.NESTEDINTENTSECTION) {
+            if (section.SectionType === SectionType.SIMPLEINTENTSECTION
+                || section.SectionType === SectionType.NESTEDINTENTSECTION
+                || section.SectionType === SectionType.QNASECTION) {
                 const startLine = section.ParseTree.start.line - 1
                 let stopLine
                 if (index + 1 < sections.length) {
@@ -316,13 +319,21 @@ class LUParser {
                     stopLine = originList.length
                 }
 
-                const destList = originList.slice(startLine + 1, stopLine)
+                let destList
+                if (section.SectionType === SectionType.QNASECTION) {
+                    destList = originList.slice(startLine, stopLine)
+                    section.Id = qnaSectionIndex
+                    qnaSectionIndex++
+                } else {
+                    destList = originList.slice(startLine + 1, stopLine)
+                }
+
                 section.Body = destList.join(NEWLINE)
                 section.StartLine = startLine
                 section.StopLine = stopLine - 1
 
                 if (section.SectionType === SectionType.NESTEDINTENTSECTION) {
-                    LUParser.extractIntentBody(section.SimpleIntentSections, originList.slice(0, stopLine).join(NEWLINE))
+                    LUParser.extractSectionBody(section.SimpleIntentSections, originList.slice(0, stopLine).join(NEWLINE))
                 }
             } else {
                 section.StartLine = section.ParseTree.start.line

packages/lu/src/parser/lufile/parseFileContents.js

@@ -1817,8 +1817,8 @@ const parseAndHandleQnaSection = function (parsedContent, luResource) {
     let qnas = luResource.Sections.filter(s => s.SectionType === SectionType.QNASECTION);
     if (qnas && qnas.length > 0) {
         for (const qna of qnas) {
-            if (qna.Id) {
-                qna.Id = parseInt(qna.Id);
+            if (qna.QAPairId) {
+                qna.QAPairId = parseInt(qna.QAPairId);
             } 
             let questions = qna.Questions;
             // detect if any question is a reference
@@ -1844,7 +1844,7 @@ const parseAndHandleQnaSection = function (parsedContent, luResource) {
                     context.prompts.push(new qnaPrompt(prompt.displayText, prompt.linkedQuestion, undefined, contextOnly, idx));
                 })
             }
-            parsedContent.qnaJsonStructure.qnaList.push(new qnaListObj(qna.Id || 0, answer.trim(), qna.source, questions, metadata, context));
+            parsedContent.qnaJsonStructure.qnaList.push(new qnaListObj(qna.QAPairId || 0, answer.trim(), qna.source, questions, metadata, context));
         }
     }
 }

packages/lu/src/parser/lufile/qnaSection.js

@@ -1,7 +1,6 @@
 const QnaSectionContext = require('./generated/LUFileParser').LUFileParser.QnaSectionContext;
 const LUSectionTypes = require('./../utils/enums/lusectiontypes');
 const BuildDiagnostic = require('./diagnostic').BuildDiagnostic;
-const helpers = require('../utils/helpers');
 const QNA_GENERIC_SOURCE = "custom editorial";
 
 class QnaSection {
@@ -24,14 +23,14 @@ class QnaSection {
         this.prompts = result.promptDefinitions;
         this.promptsText = result.promptTextList;
         this.Errors = this.Errors.concat(result.errors);
-        this.Id = this.ExtractAssignedId(parseTree);
+        this.QAPairId = this.ExtractAssignedId(parseTree);
         this.source = this.ExtractSourceInfo(parseTree);
     }
 
     ExtractSourceInfo(parseTree) {
         let srcAssignment = parseTree.qnaDefinition().qnaSourceInfo()
         if (srcAssignment) {
-            let srcRegExp = new RegExp(/^[ ]*\>[ ]*!#[ ]*@qna.pair.source[ ]*=[ ]*(?<sourceInfo>.*?)$/gmi);
+            let srcRegExp = /^[ ]*\>[ ]*!#[ ]*@qna.pair.source[ ]*=[ ]*(?<sourceInfo>.*?)$/gmi;
             let srcParsed = srcRegExp.exec(srcAssignment.getText().trim());
             return srcParsed.groups.sourceInfo || QNA_GENERIC_SOURCE;
         }
@@ -41,7 +40,7 @@ class QnaSection {
     ExtractAssignedId(parseTree) {
         let idAssignment = parseTree.qnaDefinition().qnaIdMark()
         if (idAssignment) {
-            let idTextRegExp = new RegExp(/^\<a[ ]*id[ ]*=[ ]*[\"\'](?<idCaptured>.*?)[\"\'][ ]*>[ ]*\<\/a\>$/gmi);
+            let idTextRegExp = /^\<a[ ]*id[ ]*=[ ]*[\"\'](?<idCaptured>.*?)[\"\'][ ]*>[ ]*\<\/a\>$/gmi;
             let idTextParsed = idTextRegExp.exec(idAssignment.getText().trim());
             return idTextParsed.groups.idCaptured || undefined;
         }
@@ -70,7 +69,7 @@ class QnaSection {
             let filterLineText = promptLine.getText().trim();
             filterLineText = filterLineText.substr(1).trim();
             promptTextList.push(filterLineText);
-            let promptConfigurationRegExp = new RegExp(/^\[(?<displayText>.*?)]\([ ]*\#[ ]*[ ?]*(?<linkedQuestion>.*?)\)[ ]*(?<contextOnly>\`context-only\`)?.*?$/gmi);
+            let promptConfigurationRegExp = /^\[(?<displayText>.*?)]\([ ]*\#[ ]*[ ?]*(?<linkedQuestion>.*?)\)[ ]*(?<contextOnly>\`context-only\`)?.*?$/gmi;
             let splitLine = promptConfigurationRegExp.exec(filterLineText);
             if (!splitLine) {
                 errors.push(BuildDiagnostic({

packages/lu/src/parser/lufile/sectionOperator.js

@@ -49,15 +49,25 @@ class SectionOperator {
     return luParser.parse(newContent);
   }
 
-  replaceRangeContent(originString, startLine, stopLine, replaceString) {
-
-    if (!originString) {
-      throw new Error('replace content with error parameters.');
+  insertSection(id, sectionContent) {
+    sectionContent = helpers.sanitizeNewLines(sectionContent);
+    const section = this.Luresource.Sections.find(u => u.Id === id);
+    if (!section && this.Luresource.Sections.length > 0 ) {
+      return this.Luresource;
     }
 
+    const startLine = section ? section.StartLine : 0;
+    const newContent = this.replaceRangeContent(this.Luresource.Content, startLine, startLine - 1, sectionContent);
+
+    const result = luParser.parse(newContent);
+
+    return result;
+  }
+
+  replaceRangeContent(originString, startLine, stopLine, replaceString) {
     const originList = originString.split(/\r?\n/);
     let destList = [];
-    if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine || originList.length <= stopLine) {
+    if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine + 1 || originList.length <= stopLine) {
       throw new Error("index out of range.");
     }
 

packages/lu/test/parser/lufile/sectionapi.test.js

@@ -293,4 +293,113 @@ describe('Section CRUD tests for error import in utterances', () => {
         assert.equal(luresource.Sections[1].Name, 'Cancel')
         assert.equal(luresource.Sections[1].Body, '- cancel that')
     });
+});
+
+describe('Section CRUD tests for qna', () => {
+    let luresource = undefined;
+
+    let fileContent = 
+`# ? who is CEO of Microsoft
+- Microsoft CEO
+
+\`\`\`
+Satya Nadella
+\`\`\``;
+
+    let addedFileContent = 
+`# ? what about the weather of Seattle
+- how about the weather of Seattle
+
+\`\`\`
+warm and rainy
+\`\`\``
+
+    let updatedFileConent = 
+`# ? who is CEO of Facebook
+- Facebook CEO
+
+\`\`\`
+Mark Zuckerberg
+\`\`\``;
+
+    let insertFileContent =
+`# ? how to greet
+
+\`\`\`
+hello
+\`\`\``
+
+    it('add qna section test', () => {
+        luresource = luparser.parse(fileContent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 1);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), fileContent);
+
+        luresource = new SectionOperator(luresource).addSection(addedFileContent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 2);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g,"\n"), addedFileContent);
+    });
+
+    it('update qna section test', () => {
+        luresource = new SectionOperator(luresource).updateSection(luresource.Sections[0].Id, updatedFileConent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 2);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), updatedFileConent);
+    });
+
+    it('insert qna section at begining test', () => {
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[0].Id, insertFileContent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 3);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), insertFileContent);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g,"\n"), updatedFileConent);
+        assert.equal(luresource.Sections[2].Body.replace(/\r\n/g,"\n"), addedFileContent);
+    });
+
+    it('delete qna section test', () => {
+        luresource = new SectionOperator(luresource).deleteSection(luresource.Sections[0].Id);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 2);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), updatedFileConent);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g,"\n"), addedFileContent);
+    });
+
+    it('insert qna section at middle test', () => {
+        luresource = new SectionOperator(luresource).insertSection(luresource.Sections[1].Id, insertFileContent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 3);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[1].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[2].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), updatedFileConent);
+        assert.equal(luresource.Sections[1].Body.replace(/\r\n/g,"\n"), insertFileContent);
+        assert.equal(luresource.Sections[2].Body.replace(/\r\n/g,"\n"), addedFileContent);
+    });
+
+    it('insert qna section at empty file', () => {
+        luresource = luparser.parse('');
+        luresource = new SectionOperator(luresource).insertSection(0, insertFileContent);
+
+        assert.equal(luresource.Errors.length, 0);
+        assert.equal(luresource.Sections.length, 1);
+        assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION);
+        assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), insertFileContent + '\n');
+    });
 });