Expose import resolver interface for cross-train/luis:build/qnamaker:build

packages/lu/src/parser/cross-train/crossTrainer.js

@@ -28,19 +28,20 @@ module.exports = {
    * @param {any[]} luContents the lu content array whose element includes path and content
    * @param {any[]} qnaContents the qna content array whose element includes path and content
    * @param {any} crossTrainConfig cross train json config
+   * @param {any} importResolver import Resolver when resolving import files
    * @returns {Map<string, LUResource>} map of file id and luResource
    * @throws {exception} throws errors
    */
-  crossTrain: async function (luContents, qnaContents, crossTrainConfig) {
+  crossTrain: async function (luContents, qnaContents, crossTrainConfig, importResolver) {
     try {
       let {luObjectArray, qnaObjectArray} = pretreatment(luContents, qnaContents)
       const {rootIds, triggerRules, intentName, verbose} = crossTrainConfig
 
       // parse lu content to LUResource object
-      let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose)
+      let {fileIdToResourceMap: luFileIdToResourceMap, allEmpty: allLuEmpty} = await parseAndValidateContent(luObjectArray, verbose, importResolver)
 
       // parse qna content to LUResource object
-      let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose)
+      let {fileIdToResourceMap: qnaFileIdToResourceMap, allEmpty: allQnAEmpty} = await parseAndValidateContent(qnaObjectArray, verbose, importResolver)
 
       if (!allLuEmpty) {
         // construct resource tree to build the father-children relationship among lu files
@@ -496,21 +497,22 @@ const qnaAddMetaData = function (qnaResource, fileName) {
  * Parse and validate lu or qna object array to convert to LUResource object dict
  * @param {luObject[]} objectArray the lu or qna object list to be parsed
  * @param {boolean} verbose indicate to enable log messages or not
+ * @param {any} importResolver import Resolver when resolving import files
  * @returns {Map<string, LUResource>} map of file id and luResource
  * @throws {exception} throws errors
  */
-const parseAndValidateContent = async function (objectArray, verbose) {
+const parseAndValidateContent = async function (objectArray, verbose, importResolver) {
   let fileIdToResourceMap = new Map()
   let allEmpty = true
   for (const object of objectArray) {    
     let fileContent = object.content
     if (object.content && object.content !== '') {
       if (object.id.toLowerCase().endsWith(fileExtEnum.LUFile)) {
-        let result = await LuisBuilderVerbose.build([object], verbose)
+        let result = await LuisBuilderVerbose.build([object], verbose, undefined, importResolver)
         let luisObj = new Luis(result)
         fileContent = luisObj.parseToLuContent()
       } else {
-        let result = await qnaBuilderVerbose.build([object], verbose)
+        let result = await qnaBuilderVerbose.build([object], verbose, importResolver)
         fileContent = result.parseToQnAContent()
       }
     }

packages/lu/src/parser/lu/luMerger.js

@@ -30,15 +30,15 @@ module.exports = {
      */
     Build: async function(luObjArray, verbose, luis_culture, luSearchFn){
         let allParsedContent = await buildLuJsonObject(luObjArray, verbose, luis_culture, luSearchFn)
-        let refTree = buildRefTree(allParsedContent)
+        let refTree = await buildRefTree(allParsedContent, luSearchFn)
         resolveTreeRefs(refTree, luObjArray);
         return allParsedContent
     }
 }
 
-const buildRefTree = function(allParsedContent) {
+const buildRefTree = async function(allParsedContent, luSearchFn) {
     let refs = {};
-    allParsedContent.LUISContent.forEach((parserObj, objIdx) => {
+    await Promise.all(allParsedContent.LUISContent.map(async (parserObj, objIdx) => {
         let luObj = {
             obj : parserObj.LUISJsonStructure,
             srcFile : parserObj.srcFile,
@@ -54,23 +54,23 @@ const buildRefTree = function(allParsedContent) {
             }
         }
         parserObj.LUISJsonStructure.uttHash = {};
-        (parserObj.LUISJsonStructure.utterances || []).forEach((utterance, uttIdx) => {
+        (parserObj.LUISJsonStructure.utterances || []).forEach(async (utterance, uttIdx) => {
             parserObj.LUISJsonStructure.uttHash[utterance.text] = '';
             if (helpers.isUtteranceLinkRef(utterance.text)) {
-                let parsedLinkUri = helpers.parseLinkURI(utterance.text);
+                let parsedLinkUri = await helpers.parseLinkURI(utterance.text, parserObj.srcFile, luSearchFn);
                 refs[parserObj.srcFile].luis.refs.push({
-                    refId : parsedLinkUri.fileName,
-                    uttId : uttIdx,
-                    parsedLink  : parsedLinkUri,
-                    uttObj : utterance,
-                    text : utterance.text,
-                    type : 'luis'
+                    refId: parsedLinkUri.fileName,
+                    uttId: uttIdx,
+                    parsedLink: parsedLinkUri,
+                    uttObj: utterance,
+                    text: utterance.text,
+                    type: 'luis'
                 })
             }
         })
-    })
+    }))
 
-    allParsedContent.QnAContent.forEach((parserObj, objIdx) => {
+    await Promise.all(allParsedContent.QnAContent.map(async (parserObj, objIdx) => {
         let qnaObj = {
             obj : parserObj.qnaJsonStructure,
             alt : allParsedContent.QnAAlterations[objIdx].qnaAlterations,
@@ -84,22 +84,23 @@ const buildRefTree = function(allParsedContent) {
                 refs[parserObj.srcFile].qna = qnaObj;
             }
         }
-        (parserObj.qnaJsonStructure.qnaList.forEach(qnaPair => {
-            qnaPair.questions.forEach((question, qIdx) => {
+        (parserObj.qnaJsonStructure.qnaList.forEach(async qnaPair => {
+            qnaPair.questions.forEach(async (question, qIdx) => {
                 if (helpers.isUtteranceLinkRef(question)) {
-                    let parsedLinkUri = helpers.parseLinkURI(question);
+                    let parsedLinkUri = await helpers.parseLinkURI(question)
                     refs[parserObj.srcFile].qna.refs.push({
-                        refId : parsedLinkUri.fileName,
-                        qId : qIdx,
-                        text : question,
-                        qObj : qnaPair,
-                        parsedLink : parsedLinkUri, 
-                        type : 'qna'
+                        refId: parsedLinkUri.fileName,
+                        qId: qIdx,
+                        text: question,
+                        qObj: qnaPair,
+                        parsedLink: parsedLinkUri,
+                        type: 'qna'
                     })
                 }
             })
         }))
-    });
+    }));
+
     return refs;
 }
 

packages/lu/src/parser/lubuild/builder.ts

@@ -34,7 +34,8 @@ export class Builder {
     culture: string,
     suffix: string,
     region: string,
-    schema?: string) {
+    schema?: string,
+    importResolver?: object) {
     let multiRecognizers = new Map<string, MultiLanguageRecognizer>()
     let settings: any
     let recognizers = new Map<string, Recognizer>()
@@ -82,7 +83,7 @@ export class Builder {
       if (luFiles.length <= 0) continue
 
       try {
-        result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
+        result = await LuisBuilderVerbose.build(luFiles, true, fileCulture, importResolver)
         luisObj = new Luis(result)
         fileContent = luisObj.parseToLuContent()
       } catch (err) {

packages/lu/src/parser/lufile/parseFileContents.js

@@ -174,10 +174,10 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
     parseAndHandleEntitySection(parsedContent, luResource, log, locale);
 
     // parse simple intent section
-    parseAndHandleSimpleIntentSection(parsedContent, luResource);
+    await parseAndHandleSimpleIntentSection(parsedContent, luResource);
 
     // parse qna section
-    parseAndHandleQnaSection(parsedContent, luResource);
+    await parseAndHandleQnaSection(parsedContent, luResource);
 
     if (featuresToProcess && featuresToProcess.length > 0) {
         parseFeatureSections(parsedContent, featuresToProcess);
@@ -753,7 +753,7 @@ const parseAndHandleNestedIntentSection = function (luResource, enableMergeInten
  * @param {LUResouce} luResource resources extracted from lu file content
  * @throws {exception} Throws on errors. exception object includes errCode and text.
  */
-const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
+const parseAndHandleSimpleIntentSection = async function (parsedContent, luResource) {
     // handle intent
     let intents = luResource.Sections.filter(s => s.SectionType === SectionType.SIMPLEINTENTSECTION);
     let hashTable = {}
@@ -769,7 +769,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
                 // Fix for BF-CLI #122. 
                 // Ensure only links are detected and passed on to be parsed.
                 if (helpers.isUtteranceLinkRef(utterance || '')) {
-                    let parsedLinkUriInUtterance = helpers.parseLinkURI(utterance);
+                    let parsedLinkUriInUtterance = await helpers.parseLinkURI(utterance);
                     // examine and add these to filestoparse list.
                     parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
                 }
@@ -1818,7 +1818,7 @@ const handleRegExEntity = function(parsedContent, entityName, entityType, entity
  * @param {LUResouce} luResource resources extracted from lu file content
  * @throws {exception} Throws on errors. exception object includes errCode and text.
  */
-const parseAndHandleQnaSection = function (parsedContent, luResource) {
+const parseAndHandleQnaSection = async function (parsedContent, luResource) {
     // handle QNA
     let qnas = luResource.Sections.filter(s => s.SectionType === SectionType.QNASECTION);
     if (qnas && qnas.length > 0) {
@@ -1828,14 +1828,14 @@ const parseAndHandleQnaSection = function (parsedContent, luResource) {
             } 
             let questions = qna.Questions;
             // detect if any question is a reference
-            (questions || []).forEach(question => {
+            await Promise.all((questions || []).map(async question => {
                 // Ensure only links are detected and passed on to be parsed.
                 if (helpers.isUtteranceLinkRef(question || '')) {
-                    let parsedLinkUriInUtterance = helpers.parseLinkURI(question);
+                    let parsedLinkUriInUtterance = await helpers.parseLinkURI(question);
                     // examine and add these to filestoparse list.
                     parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false));
                 }
-            })
+            }))
             let filterPairs = qna.FilterPairs;
             let metadata = [];
             if (filterPairs && filterPairs.length > 0) {

packages/lu/src/parser/qnabuild/builder.ts

@@ -36,7 +36,8 @@ export class Builder {
     suffix: string,
     region: string,
     culture: string,
-    schema?: string) {
+    schema?: string,
+    importResolver?: object) {
     let multiRecognizers = new Map<string, MultiLanguageRecognizer>()
     let settings: any
     let recognizers = new Map<string, Recognizer>()
@@ -133,7 +134,7 @@ export class Builder {
       }
     }
 
-    await this.resolveMergedQnAContentIds(qnaContents, qnaObjects)
+    await this.resolveMergedQnAContentIds(qnaContents, qnaObjects, importResolver)
 
     return {qnaContents: [...qnaContents.values()], recognizers, multiRecognizers, settings, crosstrainedRecognizers}
   }
@@ -583,11 +584,11 @@ export class Builder {
     this.handler(`Publishing finished for kb ${kbName}\n`)
   }
 
-  async resolveMergedQnAContentIds(contents: Map<string, any>, objects: Map<string, any[]>) {
+  async resolveMergedQnAContentIds(contents: Map<string, any>, objects: Map<string, any[]>, importResolver?: object) {
     for (const [name, content] of contents) {
       let qnaObjects = objects.get(name)
       try {
-        let result = await qnaBuilderVerbose.build(qnaObjects, true)
+        let result = await qnaBuilderVerbose.build(qnaObjects, true, importResolver)
         let mergedContent = result.parseToQnAContent()
         content.content = mergedContent
         contents.set(name, content)

packages/lu/src/parser/utils/helpers.js

@@ -67,18 +67,23 @@ const helpers = {
     /**
      * Helper function to parse link URIs in utterances
      * @param {String} utterance
-     * @param {String} srcPath
+     * @param {String} srcId
+     * @param {Object} luSearchFn
      * @returns {Object} Object that contains luFile and ref. ref can be Intent-Name or ? or * or **
      * @throws {exception} Throws on errors. exception object includes errCode and text. 
      */
-    parseLinkURI: function (utterance, srcPath = null) {
+    parseLinkURI: async function (utterance, srcId = null, luSearchFn = null) {
         let linkValueList = utterance.trim().match(new RegExp(/\(.*?\)/g));
         let linkValue = linkValueList[0].replace('(', '').replace(')', '');
         if (linkValue === '') throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}"`));
         // reference can either be #<Intent-Name> or #? or /*#? or /**#? or #*utterance* or #<Intent-Name>*patterns*
         let splitRegExp = new RegExp(/^(?<fileName>.*?)(?<segment>#|\*+)(?<path>.*?)$/gim);
         let splitReference = splitRegExp.exec(linkValue);
         if (!splitReference) throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n Reference needs a qualifier - either a #Intent-Name or #? or *#? or **#? or #*utterances* etc.`));
+        if (splitReference.groups.fileName && srcId && luSearchFn) {
+            let luObjects = await luSearchFn(srcId, [{filePath: splitReference.groups.fileName, includeInCollate: false}])
+            if (luObjects && luObjects.length > 0) splitReference.groups.fileName = luObjects[0].id
+        }
         if (splitReference.groups.segment.includes('*')) {
             if (splitReference.groups.path === '') {
                 throw (new exception(retCode.errorCode.INVALID_LU_FILE_REF, `[ERROR]: Invalid LU File Ref: "${utterance}".\n '*' and '**' can only be used with QnA qualitifier. e.g. *#? and **#?`));

packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/common.en-us.lu

@@ -0,0 +1,9 @@
+[import](welcome.lu)
+
+# help
+- could you help
+- [import help](help.lu#help)
+
+# cancel
+- cancel that
+- [import cancel](terminate.lu#*utterances*)

packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/help.en-us.lu

@@ -0,0 +1,2 @@
+# help
+- can you help me

packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/terminate.en-us.lu

@@ -0,0 +1,5 @@
+# cancel
+- cancel the task
+
+# stop
+- stop that

packages/lu/test/fixtures/testcases/import-resolver/lu-import-resolver/welcome.en-us.lu

@@ -0,0 +1,2 @@
+# welcome
+- welcome here

packages/lu/test/fixtures/testcases/import-resolver/qna-import-resolver/common.en-us.qna

@@ -0,0 +1,8 @@
+[import](welcome.qna)
+[import](./terminate.en-us.qna)
+
+# ? help
+- could you help
+```
+help answer
+``` 

packages/lu/test/fixtures/testcases/import-resolver/qna-import-resolver/terminate.en-us.qna

@@ -0,0 +1,9 @@
+# ? cancel
+```
+cancel the task
+```
+
+# ? stop
+```
+stop that
+```

packages/lu/test/fixtures/testcases/import-resolver/qna-import-resolver/welcome.en-us.qna

@@ -0,0 +1,4 @@
+# ? welcome
+```
+welcome here
+```