Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit c33554d

Browse files
vishwacsenaVishwac Sena KannanVishwac Sena KannanVishwac Sena Kannan
authored and
munozemilio
committed
Various fixes to bf-lu (#813)
* various fixes. * fixes * updates * updates * fix for luis:build issue. * updates for nDepth reference resolver. * Update luis:build to create result using luis() Co-authored-by: Vishwac Sena Kannan <vishwacsenakannan@MININT-GO8OQNV.redmond.corp.microsoft.com> Co-authored-by: Vishwac Sena Kannan <vishwacsenakannan@Vishwacs-MBP.guest.corp.microsoft.com> Co-authored-by: Vishwac Sena Kannan <vishwacsenakannan@za-lyshange1811.africa.corp.microsoft.com>
1 parent 74e9d0e commit c33554d

File tree

10 files changed

+697
-185
lines changed

10 files changed

+697
-185
lines changed

‎packages/lu/src/parser/lubuild/builder.ts‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const retCode = require('./../utils/enums/CLI-errors')
1717
const exception = require('./../utils/exception')
1818
const LuisBuilderVerbose = require('./../luis/luisCollate')
1919
const LuisBuilder = require('./../luis/luisBuilder')
20+
const Luis = require('./../luis/luis')
2021
const LUOptions = require('./../lu/luOptions')
2122
const Content = require('./../lu/lu')
2223
const recognizerType = require('./../utils/enums/recognizertypes')
@@ -43,11 +44,23 @@ export class Builder {
4344
let fileName: string
4445
const luFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.LUFile)
4546

47+
let cultureFromPath = fileHelper.getCultureFromPath(file)
48+
if (cultureFromPath) {
49+
fileCulture = cultureFromPath
50+
let fileNameWithCulture = path.basename(file, path.extname(file))
51+
fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - fileCulture.length - 1)
52+
} else {
53+
fileCulture = culture
54+
fileName = path.basename(file, path.extname(file))
55+
}
56+
4657
let fileContent = ''
4758
let result
59+
let luisObj
4860
try {
49-
result = await LuisBuilderVerbose.build(luFiles, true, culture)
50-
fileContent = result.parseToLuContent()
61+
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
62+
luisObj = new Luis(result)
63+
fileContent = luisObj.parseToLuContent()
5164
} catch (err) {
5265
if (err.source) {
5366
err.text = `Invalid LU file ${err.source}: ${err.text}`
@@ -58,15 +71,6 @@ export class Builder {
5871
}
5972

6073
this.handler(`${file} loaded\n`)
61-
let cultureFromPath = fileHelper.getCultureFromPath(file)
62-
if (cultureFromPath) {
63-
fileCulture = cultureFromPath
64-
let fileNameWithCulture = path.basename(file, path.extname(file))
65-
fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - fileCulture.length - 1)
66-
} else {
67-
fileCulture = result.culture !== 'en-us' ? result.culture : culture
68-
fileName = path.basename(file, path.extname(file))
69-
}
7074

7175
const fileFolder = path.dirname(file)
7276
const multiRecognizerPath = path.join(fileFolder, `${fileName}.lu.dialog`)

‎packages/lu/src/parser/lufile/parseFileContents.js‎

Lines changed: 53 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -179,10 +179,6 @@ const parseLuAndQnaWithAntlr = async function (parsedContent, fileContent, log,
179179

180180
validateNDepthEntities(parsedContent.LUISJsonStructure.entities, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles, parsedContent.LUISJsonStructure.intents);
181181

182-
// This nDepth child might have been labelled, if so, remove the duplicate simple entity.
183-
// If utterances have this child, then all parents must be included in the label
184-
updateModelBasedOnNDepthEntities(parsedContent.LUISJsonStructure.utterances, parsedContent.LUISJsonStructure.entities);
185-
186182
// Update intent and entities with phrase lists if any
187183
updateIntentAndEntityFeatures(parsedContent.LUISJsonStructure);
188184

@@ -229,23 +225,30 @@ const updateModelBasedOnNDepthEntities = function(utterances, entities) {
229225
entityFoundInMaster.push({id: idx, entityRoot: entity, path: entityPath});
230226
}
231227
});
228+
let isParentLabelled = false;
232229
entityFoundInMaster.forEach(entityInMaster => {
233230
let splitPath = entityInMaster.path.split("/").filter(item => item.trim() !== "");
234231
if (entityFoundInMaster.length > 1 && splitPath.length === 0 && (!entityInMaster.entityRoot.children || entityInMaster.entityRoot.children.length === 0)) {
235232
// this child needs to be removed. Note: There can only be at most one more entity due to utterance validation rules.
236233
entities.splice(entityInMaster.id, 1);
237234
} else {
238-
splitPath.reverse().forEach(parent => {
239-
// Ensure each parent is also labelled in this utterance
240-
let parentLabelled = utterance.entities.find(entityUtt => entityUtt.entity == parent);
241-
if (!parentLabelled) {
242-
const errorMsg = `Every child entity labelled in an utterance must have its parent labelled in that utterance. Parent "${parent}" for child "${entityInUtterance.entity}" is not labelled in utterance "${utterance.text}" for intent "${utterance.intent}".`;
243-
const error = BuildDiagnostic({
244-
message: errorMsg
245-
});
246-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
247-
}
248-
})
235+
if (isParentLabelled === false) {
236+
let rSplitPath = splitPath.reverse();
237+
rSplitPath.splice(0, 1);
238+
rSplitPath.forEach(parent => {
239+
// Ensure each parent is also labelled in this utterance
240+
let parentLabelled = utterance.entities.find(entityUtt => entityUtt.entity == parent);
241+
if (!parentLabelled) {
242+
const errorMsg = `Every child entity labelled in an utterance must have its parent labelled in that utterance. Parent "${parent}" for child "${entityInUtterance.entity}" is not labelled in utterance "${utterance.text}" for intent "${utterance.intent}".`;
243+
const error = BuildDiagnostic({
244+
message: errorMsg
245+
});
246+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
247+
} else {
248+
isParentLabelled = true;
249+
}
250+
})
251+
}
249252
}
250253
})
251254
})
@@ -870,37 +873,43 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
870873
if (entity.role) {
871874
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.CLOSEDLISTS, entity.entity, [entity.role.trim()]);
872875
} else {
873-
let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
874-
let error = BuildDiagnostic({
875-
message: errorMsg,
876-
context: utteranceAndEntities.context
877-
});
878-
879-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
876+
if (!isChildEntity(entity, entitiesFound)) {
877+
let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
878+
let error = BuildDiagnostic({
879+
message: errorMsg,
880+
context: utteranceAndEntities.context
881+
});
882+
883+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
884+
}
880885
}
881886
} else if (prebuiltExists !== undefined) {
882887
if (entity.role) {
883888
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entity.entity, [entity.role.trim()]);
884889
} else {
885-
let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
886-
let error = BuildDiagnostic({
887-
message: errorMsg,
888-
context: utteranceAndEntities.context
889-
});
890-
891-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
890+
if (!isChildEntity(entity, entitiesFound)) {
891+
let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
892+
let error = BuildDiagnostic({
893+
message: errorMsg,
894+
context: utteranceAndEntities.context
895+
});
896+
897+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
898+
}
892899
}
893900
} else if (regexExists !== undefined) {
894901
if (entity.role) {
895902
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.REGEX, entity.entity, [entity.role.trim()]);
896903
} else {
897-
let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
898-
let error = BuildDiagnostic({
899-
message: errorMsg,
900-
context: utteranceAndEntities.context
901-
});
902-
903-
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
904+
if (!isChildEntity(entity, entitiesFound)) {
905+
let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
906+
let error = BuildDiagnostic({
907+
message: errorMsg,
908+
context: utteranceAndEntities.context
909+
});
910+
911+
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error]));
912+
}
904913
}
905914
} else if (patternAnyExists !== undefined) {
906915
if (entity.value != '') {
@@ -975,6 +984,11 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) {
975984
}
976985
}
977986

987+
const isChildEntity = function(entity, entitiesFound) {
988+
// return true if the current entity is contained within a parent bounding label
989+
return (entitiesFound || []).find(item => item.entity !== entity.entity && entity.startPos >= item.startPos && entity.endPos <= item.endPos) === undefined ? false : true;
990+
}
991+
978992
/**
979993
* Helper function to get entity type based on a name match
980994
* @param {String} entityName name of entity to look up type for.
@@ -1155,6 +1169,7 @@ const handleNDepthEntity = function(parsedContent, entityName, entityRoles, enti
11551169
const SPACEASTABS = 4;
11561170
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles);
11571171
let rootEntity = parsedContent.LUISJsonStructure.entities.find(item => item.name == entityName);
1172+
rootEntity.explicitlyAdded = true;
11581173
let defLine = line.start.line;
11591174
let baseTabLevel = 0;
11601175
let entityIdxByLevel = [];
@@ -1634,6 +1649,8 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo
16341649
} else if (entityType.toLowerCase() === 'simple') {
16351650
// add this to entities if it doesnt exist
16361651
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles);
1652+
let rootEntity = parsedContent.LUISJsonStructure.entities.find(item => item.name == entityName);
1653+
rootEntity.explicitlyAdded = true;
16371654
} else if (entityType.endsWith('=')) {
16381655
// is this qna maker alterations list?
16391656
if (entityType.includes(PARSERCONSTS.QNAALTERATIONS)) {

‎packages/lu/src/parser/lufile/simpleIntentSection.js‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,24 @@ class SimpleIntentSection {
5050
}
5151

5252
for (const normalIntentStr of parseTree.intentDefinition().intentBody().normalIntentBody().normalIntentString()) {
53-
let utteranceAndEntities = visitor.visitNormalIntentStringContext(normalIntentStr);
54-
utteranceAndEntities.context = normalIntentStr;
55-
utteranceAndEntitiesMap.push(utteranceAndEntities);
56-
utteranceAndEntities.errorMsgs.forEach(errorMsg => errors.push(BuildDiagnostic({
57-
message: errorMsg,
58-
context: normalIntentStr
59-
})))
53+
let utteranceAndEntities;
54+
try {
55+
utteranceAndEntities = visitor.visitNormalIntentStringContext(normalIntentStr);
56+
}
57+
catch (err) {
58+
errors.push(BuildDiagnostic({
59+
message: "Invalid utterance definition found. Did you miss a '{' or '}'?",
60+
context: normalIntentStr
61+
}))
62+
};
63+
if (utteranceAndEntities !== undefined) {
64+
utteranceAndEntities.context = normalIntentStr;
65+
utteranceAndEntitiesMap.push(utteranceAndEntities);
66+
utteranceAndEntities.errorMsgs.forEach(errorMsg => errors.push(BuildDiagnostic({
67+
message: errorMsg,
68+
context: normalIntentStr
69+
})));
70+
}
6071
}
6172
}
6273

‎packages/lu/src/parser/luis/luis.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class Luis {
3030

3131
parseToLuContent(){
3232
helpers.checkAndUpdateVersion(this)
33+
helpers.cleanUpExplicitEntityProperty(this)
3334
return luConverter(this)
3435
}
3536

@@ -54,6 +55,12 @@ const initialize = function(instance, LuisJSON) {
5455
if (Object.keys(LuisJSON).includes(regexEntities)) instance[regexEntities] = LuisJSON[regexEntities];
5556

5657
settingsAndTokenizerCheck(instance, LuisJSON)
58+
59+
initializeEntities(instance)
60+
}
61+
62+
const initializeEntities = function (instance) {
63+
(instance.entities || []).forEach(e => e.explicitlyAdded = true)
5764
}
5865

5966
const settingsAndTokenizerCheck = function(instance, LuisJSON) {

‎packages/lu/src/parser/luis/luisCollate.js‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const Luis = require('./luis')
55
const helpers = require('./../utils/helpers')
66
const mergeLuFiles = require('./../lu/luMerger').Build
77
const exception = require('./../utils/exception')
8+
const retCode = require('../utils/enums/CLI-errors')
89

910
/**
1011
* Builds a Luis instance from a Lu list.
@@ -41,6 +42,7 @@ const collate = function(luisList) {
4142
let blob = luisList[i]
4243
mergeResults(blob, luisObject, LUISObjNameEnum.INTENT);
4344
mergeResults(blob, luisObject, LUISObjNameEnum.ENTITIES);
45+
mergeNDepthEntities(blob.entities, luisObject.entities);
4446
mergeResults_closedlists(blob, luisObject, LUISObjNameEnum.CLOSEDLISTS);
4547
mergeResults(blob, luisObject, LUISObjNameEnum.PATTERNANYENTITY);
4648
mergeResultsWithHash(blob, luisObject, LUISObjNameEnum.UTTERANCE, hashTable);
@@ -52,6 +54,7 @@ const collate = function(luisList) {
5254
buildPatternAny(blob, luisObject)
5355
}
5456
helpers.checkAndUpdateVersion(luisObject)
57+
helpers.cleanUpExplicitEntityProperty(luisObject)
5558
cleanupEntities(luisObject)
5659
return luisObject
5760
}
@@ -102,7 +105,77 @@ const mergeResultsWithHash = function (blob, finalCollection, type, hashTable) {
102105
}
103106
});
104107
}
108+
const mergeNDepthEntities = function (blob, finalCollection) {
109+
let nDepthInBlob = (blob || []).filter(x => x.children !== undefined && Array.isArray(x.children) && x.children.length !== 0);
110+
if (nDepthInBlob === undefined) return;
111+
nDepthInBlob.forEach(item => {
112+
let itemExistsInFinal = (finalCollection || []).find(x => x.name == item.name);
113+
if (itemExistsInFinal === undefined) {
114+
finalCollection.push(item);
115+
} else {
116+
// de-dupe and merge roles
117+
(item.roles || []).forEach(r => {
118+
if (itemExistsInFinal.roles === undefined) {
119+
itemExistsInFinal.roles = [r];
120+
} else {
121+
if (!itemExistsInFinal.roles.includes(r)) {
122+
itemExistsInFinal.roles.push(r);
123+
}
124+
}
125+
})
126+
// de-dupe and merge children
127+
if (item.children !== undefined && Array.isArray(item.children) && item.children.length !== 0) {
128+
recursivelyMergeChildrenAndFeatures(item.children, itemExistsInFinal.children)
129+
}
130+
}
131+
})
132+
}
105133

134+
const recursivelyMergeChildrenAndFeatures = function(srcChildren, tgtChildren) {
135+
if (tgtChildren === undefined || !Array.isArray(tgtChildren) || tgtChildren.length === 0) {
136+
tgtChildren = srcChildren;
137+
return;
138+
}
139+
(srcChildren || []).forEach(item => {
140+
// find child in tgt
141+
let itemExistsInFinal = (tgtChildren || []).find(x => x.name == item.name);
142+
if (itemExistsInFinal === undefined) {
143+
tgtChildren.push(item);
144+
} else {
145+
// merge features
146+
if (item.features !== undefined && item.features.length !== 0) {
147+
// merge and verify type
148+
let typeForFinalItem = (itemExistsInFinal.features || []).find(t => t.isRequired == true);
149+
let typeForItem = (item.features || []).find(t1 => t1.isRequired == true);
150+
if (typeForFinalItem !== undefined) {
151+
if (typeForItem !== undefined) {
152+
if (typeForFinalItem.modelName !== typeForItem.modelName) {
153+
throw new exception(retCode.errorCode.INVALID_REGEX_ENTITY, `Child entity ${item.name} does not have consistent type definition. Please verify all definitions for this entity.`)
154+
}
155+
}
156+
}
157+
item.features.forEach(f => {
158+
let featureInFinal = (itemExistsInFinal.features || []).find(itFea => {
159+
return ((itFea.featureName !== undefined && itFea.featureName == f.featureName) ||
160+
(itFea.modelName !== undefined && itFea.modelName == f.modelName))
161+
});
162+
if (featureInFinal === undefined) {
163+
itemExistsInFinal.features.push(f);
164+
} else {
165+
// throw if isRequired is not the same.
166+
if (featureInFinal.isRequired !== f.isRequired) {
167+
throw new exception(retCode.errorCode.INVALID_REGEX_ENTITY, `Feature ${f.featureName} does not have consistent definition for entity ${item.name}. Please verify all definitions for this feature for this entity.`)
168+
}
169+
}
170+
})
171+
}
172+
// de-dupe and merge children
173+
if (item.children !== undefined && Array.isArray(item.children) && item.children.length !== 0) {
174+
recursivelyMergeChildrenAndFeatures(item.children, itemExistsInFinal.children)
175+
}
176+
}
177+
})
178+
}
106179
/**
107180
* Helper function to merge item if it does not already exist
108181
*

0 commit comments

Comments
 (0)