Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

[Do Not Merge]luis and qnamaker build support to write out empty crosstrained recognizer #870

Closed
feich-ms wants to merge 6 commits into from
Closed

[Do Not Merge]luis and qnamaker build support to write out empty crosstrained recognizer #870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5fda06a
adjust luis build to handle empty lu file
feich-ms Jun 28, 2020
8edb834
support empty crosstrained recognizer in qnamaker:build
feich-ms Jun 28, 2020
1f7dfef
Merge branch 'master' into feich/WriteEmptyCrossTrainedRecognizer
feich-ms Jun 30, 2020
c76b713
fix empty .lu.qna.dialog
feich-ms Jul 6, 2020
30450f1
Merge branch 'master' into feich/WriteEmptyCrossTrainedRecognizer
feich-ms Jul 7, 2020
132f242
Merge branch 'master' into feich/WriteEmptyCrossTrainedRecognizer
feich-ms Jul 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/lu/src/parser/lu/luMerger.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ const parseLuFile = async function(luOb, log, luis_culture) {
let parsedContent = ''
if (!luOb.content) {
let error = BuildDiagnostic({ message: `Cannot parse empty ${luOb.id}. Please add content to the file or remove it.` })
throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, error.toString()));
throw(new exception(retCode.errorCode.EMPTY_CONTENT, error.toString()));
}
try {
parsedContent = await parseFileContents.parseFile(luOb.content, log, luis_culture);
Expand Down
109 changes: 59 additions & 50 deletions packages/lu/src/parser/lubuild/builder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export class Builder {
let settings = new Map<string, Settings>()
let recognizers = new Map<string, Recognizer>()
let luContents: Array<any> = []
let crosstrainedRecognizers = new Map<string, CrossTrainedRecognizer>()

for (const file of files) {
let fileCulture: string
Expand All @@ -55,25 +56,41 @@ export class Builder {
fileName = path.basename(file, path.extname(file))
}

const fileFolder = path.dirname(file)
const crossTrainedFileName = fileName + '.lu.qna.dialog'
const crossTrainedRecognizerPath = path.join(fileFolder, crossTrainedFileName)
if (!crosstrainedRecognizers.has(fileName)) {
let crosstrainedRecognizerContent = []
let crosstrainedRecognizerSchema = schema
if (fs.existsSync(crossTrainedRecognizerPath)) {
let crosstrainedRecognizerObject = JSON.parse(await fileHelper.getContentFromFile(crossTrainedRecognizerPath))
crosstrainedRecognizerContent = crosstrainedRecognizerObject.recognizers
crosstrainedRecognizerSchema = crosstrainedRecognizerSchema || crosstrainedRecognizerObject.$schema
this.handler(`${crossTrainedRecognizerPath} loaded\n`)
}

crosstrainedRecognizers.set(fileName, new CrossTrainedRecognizer(crossTrainedRecognizerPath, crosstrainedRecognizerContent, crosstrainedRecognizerSchema as string))
}

let fileContent = ''
let result
let luisObj
try {
result = await LuisBuilderVerbose.build(luFiles, true, fileCulture)
luisObj = new Luis(result)
fileContent = luisObj.parseToLuContent()
this.handler(`${file} loaded\n`)
} catch (err) {
if (err.errCode === retCode.errorCode.EMPTY_CONTENT) continue

if (err.source) {
err.text = `Invalid LU file ${err.source}: ${err.text}`
} else {
err.text = `Invalid LU file ${file}: ${err.text}`
}
throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, err.text))
throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, err.text))
}

this.handler(`${file} loaded\n`)

const fileFolder = path.dirname(file)
const multiRecognizerPath = path.join(fileFolder, `${fileName}.lu.dialog`)
if (!multiRecognizers.has(fileName)) {
let multiRecognizerContent = {}
Expand Down Expand Up @@ -127,7 +144,7 @@ export class Builder {
throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Files with same name and locale are found.'))
}

return {luContents, recognizers, multiRecognizers, settings}
return {luContents, recognizers, multiRecognizers, settings, crosstrainedRecognizers}
}

async build(
Expand All @@ -141,6 +158,8 @@ export class Builder {
deleteOldVersion: boolean,
multiRecognizers?: Map<string, MultiLanguageRecognizer>,
settings?: Map<string, Settings>,
crosstrainedRecognizers?: Map<string, CrossTrainedRecognizer>,
dialogType?: string,
luisAPITPS?: number,
timeBucketOfRequests?: number,
retryCount?: number,
Expand Down Expand Up @@ -210,6 +229,13 @@ export class Builder {
}
}

if (crosstrainedRecognizers && crosstrainedRecognizers.has(content.id)) {
let crosstrainedRecognizer = crosstrainedRecognizers.get(content.id) as CrossTrainedRecognizer
if (!crosstrainedRecognizer.recognizers.includes(content.id + '.lu')) {
crosstrainedRecognizer.recognizers.push(content.id + '.lu')
}
}

// update settings asset
if (settings && settings.has(path.dirname(content.path))) {
let setting = settings.get(path.dirname(content.path)) as Settings
Expand All @@ -234,12 +260,17 @@ export class Builder {
settingValues = Array.from(settings.values())
}

const dialogContents = luBuildCore.generateDeclarativeAssets(recognizerValues, multiRecognizerValues, settingValues)
let crosstrainedRecognizerValues: CrossTrainedRecognizer[] = []
if (dialogType === recognizerType.CROSSTRAINED && crosstrainedRecognizers) {
crosstrainedRecognizerValues = Array.from(crosstrainedRecognizers.values())
}

const dialogContents = luBuildCore.generateDeclarativeAssets(recognizerValues, multiRecognizerValues, settingValues, crosstrainedRecognizerValues)

return dialogContents
}

async writeDialogAssets(contents: any[], force: boolean, out: string, dialogType: string, luconfig: string, schema: string) {
async writeDialogAssets(contents: any[], force: boolean, out: string, luconfig: string) {
let writeDone = false

let writeContents = contents.filter(c => c.id.endsWith('.dialog'))
Expand All @@ -257,30 +288,31 @@ export class Builder {
writeContents.push(this.mergeSettingsContent(outPath, settingsContents))
}

if (out) {
for (const content of writeContents) {
const outFilePath = path.join(path.resolve(out), path.basename(content.path))
if (force || !fs.existsSync(outFilePath)) {
if (!fs.existsSync(path.dirname(outFilePath))) {
fs.mkdirSync(path.dirname(outFilePath))
}
for (const content of writeContents) {
let outFilePath
if (out) {
outFilePath = path.join(path.resolve(out), path.basename(content.path))
} else {
outFilePath = content.path
}

this.handler(`Writing to ${outFilePath}\n`)
await this.writeDialog(content.content, outFilePath, dialogType, schema)
writeDone = true
}
let fileExists = fs.existsSync(outFilePath)
if (fileExists && outFilePath.endsWith('.lu.qna.dialog')) {
let existingCTRecognizerObject = JSON.parse(await fileHelper.getContentFromFile(outFilePath))
let currentCTRecognizerObject = JSON.parse(content.content)
let ctRecognizerToBeMerged = existingCTRecognizerObject.recognizers.filter((r: string) => !currentCTRecognizerObject.recognizers.includes(r))
currentCTRecognizerObject.recognizers = currentCTRecognizerObject.recognizers.concat(ctRecognizerToBeMerged)
content.content = JSON.stringify(currentCTRecognizerObject, null, 4)
}
} else {
for (const content of writeContents) {
if (force || !fs.existsSync(content.path)) {
if (!fs.existsSync(path.dirname(content.path))) {
fs.mkdirSync(path.dirname(content.path))
}

this.handler(`Writing to ${content.path}\n`)
await this.writeDialog(content.content, content.path, dialogType, schema)
writeDone = true
if (force || !fs.existsSync(outFilePath)) {
if (!fs.existsSync(path.dirname(outFilePath))) {
fs.mkdirSync(path.dirname(outFilePath))
}

this.handler(`Writing to ${outFilePath}\n`)
await fs.writeFile(outFilePath, content.content, 'utf-8')
writeDone = true
}
}

Expand Down Expand Up @@ -428,27 +460,4 @@ export class Builder {
app.intents = filteredIntents
}
}

async writeDialog(content: string, filePath: string, dialogType: string, schema: string) {
await fs.writeFile(filePath, content, 'utf-8')
const contentObj = JSON.parse(content)
if (dialogType === recognizerType.CROSSTRAINED && contentObj.$kind === 'Microsoft.MultiLanguageRecognizer') {
const fileName = path.basename(filePath, '.lu.dialog')
const crossTrainedFileName = fileName + '.lu.qna.dialog'
const crossTrainedFilePath = path.join(path.dirname(filePath), crossTrainedFileName)
if (fs.existsSync(crossTrainedFilePath)) {
const existingCRDialog = JSON.parse(await fileHelper.getContentFromFile(crossTrainedFilePath))
if (!existingCRDialog.recognizers.includes(fileName + '.lu')) {
existingCRDialog.recognizers.push(fileName + '.lu')
}

content = JSON.stringify(existingCRDialog, null, 4)
} else {
const recognizers = [fileName + '.lu']
content = new CrossTrainedRecognizer(crossTrainedFilePath, recognizers, schema).save()
}

await fs.writeFile(crossTrainedFilePath, content, 'utf-8')
}
}
}
8 changes: 7 additions & 1 deletion packages/lu/src/parser/lubuild/core.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import {Recognizer} from './recognizer'
import {MultiLanguageRecognizer} from './multi-language-recognizer'
import {Settings} from './settings'
import {CrossTrainedRecognizer} from './cross-trained-recognizer'
import {CognitiveServicesCredentials} from '@azure/ms-rest-azure-js'
import {LUISAuthoringClient} from '@azure/cognitiveservices-luis-authoring'
import * as path from 'path'
Expand Down Expand Up @@ -372,7 +373,7 @@ export class LuBuildCore {
}
}

public generateDeclarativeAssets(recognizers: Array<Recognizer>, multiRecognizers: Array<MultiLanguageRecognizer>, settings: Array<Settings>)
public generateDeclarativeAssets(recognizers: Array<Recognizer>, multiRecognizers: Array<MultiLanguageRecognizer>, settings: Array<Settings>, crosstrainedRecognizers: Array<CrossTrainedRecognizer>)
: Array<any> {
let contents = new Array<any>()
for (const recognizer of recognizers) {
Expand All @@ -390,6 +391,11 @@ export class LuBuildCore {
contents.push(settingsContent)
}

for (const crosstrainedRecognizer of crosstrainedRecognizers) {
const crosstrainedContent = new Content(crosstrainedRecognizer.save(), new LUOptions(path.basename(crosstrainedRecognizer.getDialogPath()), true, '', crosstrainedRecognizer.getDialogPath()))
contents.push(crosstrainedContent)
}

return contents
}

Expand Down
93 changes: 49 additions & 44 deletions packages/lu/src/parser/qnabuild/builder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,36 @@ export class Builder {
let settings: any
let recognizers = new Map<string, Recognizer>()
let qnaContents = new Map<string, string>()
let crosstrainedRecognizer: any

for (const file of files) {
let fileCulture: string
let fileName: string
let cultureFromPath = fileHelper.getCultureFromPath(file)
if (cultureFromPath) {
fileCulture = cultureFromPath
let fileNameWithCulture = path.basename(file, path.extname(file))
fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - cultureFromPath.length - 1)
} else {
fileCulture = culture
fileName = path.basename(file, path.extname(file))
}

const fileFolder = path.dirname(file)

if (crosstrainedRecognizer === undefined) {
const crossTrainedFileName = fileName + '.lu.qna.dialog'
const crossTrainedRecognizerPath = path.join(fileFolder, crossTrainedFileName)
let crosstrainedRecognizerContent = []
let crosstrainedRecognizerSchema = schema
if (fs.existsSync(crossTrainedRecognizerPath)) {
let crosstrainedRecognizerObject = JSON.parse(await fileHelper.getContentFromFile(crossTrainedRecognizerPath))
crosstrainedRecognizerContent = crosstrainedRecognizerObject.recognizers
crosstrainedRecognizerSchema = crosstrainedRecognizerSchema || crosstrainedRecognizerObject.$schema
this.handler(`${crossTrainedRecognizerPath} loaded\n`)
}

crosstrainedRecognizer = new CrossTrainedRecognizer(crossTrainedRecognizerPath, crosstrainedRecognizerContent, crosstrainedRecognizerSchema as string)
}

let fileContent = ''
Expand All @@ -59,7 +81,10 @@ export class Builder {

// construct qna content without file and url references
fileContent = result.parseToQnAContent()
this.handler(`${file} loaded\n`)
} catch (err) {
if (err.errCode === retCode.errorCode.EMPTY_CONTENT) continue

if (err.source) {
err.text = `Invalid QnA file ${err.source}: ${err.text}`
} else {
Expand All @@ -68,9 +93,6 @@ export class Builder {
throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, err.text))
}

this.handler(`${file} loaded\n`)

const fileFolder = path.dirname(file)
if (multiRecognizer === undefined) {
const multiRecognizerPath = path.join(fileFolder, `${botName}.qna.dialog`)
let multiRecognizerContent = {}
Expand Down Expand Up @@ -121,7 +143,7 @@ export class Builder {
}
}

return {qnaContents: [...qnaContents.values()], recognizers, multiRecognizer, settings}
return {qnaContents: [...qnaContents.values()], recognizers, multiRecognizer, settings, crosstrainedRecognizer}
}

async build(
Expand All @@ -133,7 +155,9 @@ export class Builder {
suffix: string,
fallbackLocale: string,
multiRecognizer?: MultiLanguageRecognizer,
settings?: Settings) {
settings?: Settings,
crosstrainedRecognizer?: CrossTrainedRecognizer,
dialogType?: string) {
// qna api TPS which means concurrent transactions to qna maker api in 1 second
let qnaApiTps = 3

Expand Down Expand Up @@ -207,6 +231,12 @@ export class Builder {
}
}

if (crosstrainedRecognizer) {
if (!crosstrainedRecognizer.recognizers.includes(botName + '.qna')) {
crosstrainedRecognizer.recognizers.push(botName + '.qna')
}
}

// update settings asset
if (settings) {
settings.qna[content.name.split('.').join('_').replace(/-/g, '_')] = recognizer.getKBId()
Expand All @@ -221,7 +251,8 @@ export class Builder {
recognizerValues = Array.from(recognizers.values())
}

const dialogContents = qnaBuildCore.generateDeclarativeAssets(recognizerValues, multiRecognizer as MultiLanguageRecognizer, settings as Settings)
crosstrainedRecognizer = dialogType === recognizerType.CROSSTRAINED ? crosstrainedRecognizer : undefined
const dialogContents = qnaBuildCore.generateDeclarativeAssets(recognizerValues, multiRecognizer as MultiLanguageRecognizer, settings as Settings, crosstrainedRecognizer)

return dialogContents
}
Expand Down Expand Up @@ -302,7 +333,7 @@ export class Builder {
return kbToLuContent
}

async writeDialogAssets(contents: any[], force: boolean, out: string, dialogType: string, files: string[], schema: string) {
async writeDialogAssets(contents: any[], force: boolean, out: string) {
let writeDone = false

for (const content of contents) {
Expand All @@ -313,9 +344,18 @@ export class Builder {
outFilePath = content.path
}

if (force || !fs.existsSync(outFilePath)) {
let fileExists = fs.existsSync(outFilePath)
if (fileExists && outFilePath.endsWith('.lu.qna.dialog')) {
let existingCTRecognizerObject = JSON.parse(await fileHelper.getContentFromFile(outFilePath))
let currentCTRecognizerObject = JSON.parse(content.content)
let ctRecognizerToBeMerged = existingCTRecognizerObject.recognizers.filter((r: string) => !currentCTRecognizerObject.recognizers.includes(r))
currentCTRecognizerObject.recognizers = currentCTRecognizerObject.recognizers.concat(ctRecognizerToBeMerged)
content.content = JSON.stringify(currentCTRecognizerObject, null, 4)
}

if (force || !fileExists) {
this.handler(`Writing to ${outFilePath}\n`)
await this.writeDialog(content.content, outFilePath, dialogType, files, schema)
await fs.writeFile(outFilePath, content.content, 'utf-8')
writeDone = true
}
}
Expand Down Expand Up @@ -480,39 +520,4 @@ export class Builder {
await qnaBuildCore.publishKB(recognizer.getKBId())
this.handler(`Publishing finished for kb ${kbName}\n`)
}

async writeDialog(content: string, filePath: string, dialogType: string, files: string[], schema: string) {
await fs.writeFile(filePath, content, 'utf-8')
const contentObj = JSON.parse(content)
if (dialogType === recognizerType.CROSSTRAINED && contentObj.$kind === 'Microsoft.MultiLanguageRecognizer') {
const fileName = path.basename(filePath, '.dialog')

for (const file of files) {
let qnafileName
let cultureFromPath = fileHelper.getCultureFromPath(file)
if (cultureFromPath) {
let fileNameWithCulture = path.basename(file, path.extname(file))
qnafileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - cultureFromPath.length - 1)
} else {
qnafileName = path.basename(file, path.extname(file))
}

let crossTrainedFileName = `${qnafileName}.lu.qna.dialog`
let crossTrainedFilePath = path.join(path.dirname(filePath), crossTrainedFileName)
if (fs.existsSync(crossTrainedFilePath)) {
let existingCRDialog = JSON.parse(await fileHelper.getContentFromFile(crossTrainedFilePath))
if (!existingCRDialog.recognizers.includes(fileName)) {
existingCRDialog.recognizers.push(fileName)
}

content = JSON.stringify(existingCRDialog, null, 4)
} else {
let recognizers = [fileName]
content = new CrossTrainedRecognizer(crossTrainedFilePath, recognizers, schema).save()
}

await fs.writeFile(crossTrainedFilePath, content, 'utf-8')
}
}
}
}
Loading