Skip to content
This repository was archived by the owner on Jul 9, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ describe('lu operation', () => {
const luFiles = await proj.createLuFile(id, content, dir);
const result = luFiles.find(f => f.id === id);

expect(proj.files.length).toEqual(9);
expect(proj.files.length).toEqual(8);
expect(luFiles.length).toEqual(4);

expect(result).not.toBeUndefined();
Expand All @@ -224,7 +224,7 @@ describe('lu operation', () => {
const luFiles = await proj.updateLuFile(id, content);
const result = luFiles.find(f => f.id === id);

expect(proj.files.length).toEqual(9);
expect(proj.files.length).toEqual(8);
expect(luFiles.length).toEqual(4);

expect(result).not.toBeUndefined();
Expand All @@ -246,7 +246,7 @@ describe('lu operation', () => {
const luFiles = await proj.removeLuFile(id);
const result = luFiles.find(f => f.id === id);

expect(proj.files.length).toEqual(8);
expect(proj.files.length).toEqual(7);
expect(luFiles.length).toEqual(3);

expect(result).toBeUndefined();
Expand Down
132 changes: 37 additions & 95 deletions Composer/packages/server/__tests__/models/bot/luisPublisher.test.ts
Original file line number Diff line number Diff line change
@@ -1,123 +1,65 @@
import mock from 'mock-fs';
import { FileUpdateType } from '../../../src/models/bot/interface';
import { Path } from '../../../src/utility/path';

import { LuPublisher } from './../../../src/models/bot/luPublisher';
import service from './../../../src/services/storage';

jest.mock('azure-storage', () => {
return {};
});

const bot1Dir = '/path/to/fake/bot1';
const bot2Dir = '/path/to/fake/bot2';
const luFile1Path = '/path/to/fake/bot1/a.lu';
const luFile2Path = '/path/to/fake/bot2/a.lu';

const luisConfig = {
authoringKey: '111111111111',
authoringRegion: 'westus',
defaultLanguage: 'en-us',
environment: 'test',
name: 'test',
};

const botDir = Path.join(__dirname, '../../mocks/samplebots/bot1');
const storage = service.getStorageClient('default');

beforeEach(() => {
mock({
'/path/to/fake/bot1': {
'a.lu': ` # Greeting
- hi`,
},
'/path/to/fake/bot2': {
'a.lu': ` # Greeting
- hi`,
generated: {
'a.en-us.lu.dialog': '',
'luis.settings.test.westus.json': `
{
"luis": {
"a_en-us_lu": "e094b572-2127-4403-8052-cd7a49ec5848"
},
"status": {
"a_en-us_lu": {
"version": "0000000005",
"state": "published"
}
}
}
`,
},
},
describe('luis status management', () => {
it('will get luis status', async () => {
const luPublisher = new LuPublisher(botDir, storage);
const status = await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu', 'bot1/Main.lu']);
expect(status['bot1/a.lu'].lastUpdateTime).toBe(1);
expect(status['bot1/a.lu'].lastPublishTime).toBe(0);
});
});

afterEach(mock.restore);
it('can update luis status', async () => {
const luPublisher = new LuPublisher(botDir, storage);

describe.skip('getLuisStatus', () => {
it('will get luis status', async () => {
const luPublisher = new LuPublisher(bot2Dir, storage);
luPublisher.setLuisConfig(luisConfig);
const settings = await luPublisher.getLuisStatus();
expect(settings[0].name).toEqual('a.lu');
expect(settings[0].state).toBe('published');
await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu', 'bot1/Main.lu']);
const oldUpdateTime = luPublisher.status['bot1/a.lu'].lastUpdateTime;

await luPublisher.onFileChange('bot1/a.lu', FileUpdateType.UPDATE);
const newUpdateTime = luPublisher.status['bot1/a.lu'].lastUpdateTime;
// update should increase the update time
expect(newUpdateTime).toBeGreaterThan(oldUpdateTime);
});
});

describe.skip('getUnpublisedFiles', () => {
it('will get unpublished files when no setting.json exist', async () => {
describe('get unpublishedFiles', () => {
it('will get unpublished files', async () => {
const lufiles = [
{
diagnostics: [],
id: 'a.lu',
relativePath: '/path/to/fake/bot1/a.lu',
id: 'a',
relativePath: 'bot1/a.lu',
content: '',
parsedContent: {},
lastUpdateTime: 1,
lastPublishTime: 1,
},
];
const luPublisher = new LuPublisher(bot1Dir, storage);
luPublisher.setLuisConfig(luisConfig);
const files = await luPublisher.getUnpublisedFiles(lufiles);
expect(files.length).toBe(1);
});

it('will get unpublished files when setting.json exist', async () => {
const lufiles = [
{
diagnostics: [],
id: 'a.lu',
relativePath: '/path/to/fake/bot2/a.lu',
id: 'b',
relativePath: 'bot1/b.lu',
content: '',
parsedContent: {},
lastUpdateTime: 1,
lastPublishTime: 1,
},
];
const luPublisher = new LuPublisher(bot2Dir, storage);
luPublisher.setLuisConfig(luisConfig);
const files = await luPublisher.getUnpublisedFiles(lufiles);
expect(files.length).toBe(0);
});
});

describe.skip('update', () => {
it('will not update the statuse if no setting.json', async () => {
const luPublisher = new LuPublisher(bot1Dir, storage);
luPublisher.setLuisConfig(luisConfig);
await luPublisher.update(true, luFile1Path);
const settings = await luPublisher.getLuisStatus();
expect(settings).toEqual([]);
});
const luPublisher = new LuPublisher(botDir, storage);
await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu']); // relative path is key

it('update the statuse if setting.json exist', async () => {
const luPublisher = new LuPublisher(bot2Dir, storage);
luPublisher.setLuisConfig(luisConfig);
await luPublisher.update(false, luFile2Path);
let settings = await luPublisher.getLuisStatus();
expect(settings[0].state).toBe('published');
await luPublisher.update(true, luFile2Path);
settings = await luPublisher.getLuisStatus();
expect(settings[0].state).toBe('unpublished');
let files = await luPublisher.getUnpublisedFiles(lufiles);
expect(files.length).toBe(2);
const curTime = Date.now();
luPublisher.status['bot1/a.lu'].lastPublishTime = curTime; // assumming we publish a.lu
luPublisher.status['bot1/b.lu'].lastPublishTime = curTime; // and b.lu
files = await luPublisher.getUnpublisedFiles(lufiles);
expect(files.length).toBe(0);

await luPublisher.onFileChange('bot1/a.lu', FileUpdateType.UPDATE);
files = await luPublisher.getUnpublisedFiles(lufiles);
expect(files.length).toBe(1);
});
});
109 changes: 48 additions & 61 deletions Composer/packages/server/src/models/bot/botProject.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { copyDir } from '../../utility/storage';
import StorageService from '../../services/storage';

import { IFileStorage } from './../storage/interface';
import { LocationRef, FileInfo, LGFile, Dialog, LUFile, ILuisConfig, ILuisStatusOperation } from './interface';
import { LocationRef, FileInfo, LGFile, Dialog, LUFile, ILuisConfig, LuisStatus, FileUpdateType } from './interface';
import { DialogIndexer } from './indexers/dialogIndexers';
import { LGIndexer } from './indexers/lgIndexer';
import { LUIndexer } from './indexers/luIndexer';
Expand Down Expand Up @@ -41,7 +41,7 @@ export class BotProject {

this.dialogIndexer = new DialogIndexer(this.name);
this.lgIndexer = new LGIndexer();
this.luIndexer = new LUIndexer(this.fileStorage, this.dir);
this.luIndexer = new LUIndexer();
this.luPublisher = new LuPublisher(this.dir, this.fileStorage);
}

Expand All @@ -51,18 +51,32 @@ export class BotProject {
this.lgIndexer.index(this.files);
await this.luIndexer.index(this.files); // ludown parser is async
await this._checkProjectStructure();
await this.luPublisher.loadStatus(this.luIndexer.getLuFiles().map(f => f.relativePath));
};

public getIndexes = () => {
return {
botName: this.name,
dialogs: this.dialogIndexer.getDialogs(),
lgFiles: this.lgIndexer.getLgFiles(),
luFiles: this.luIndexer.getLuFiles(),
luFiles: this.mergeLuStatus(this.luIndexer.getLuFiles(), this.luPublisher.status),
schemas: this.getSchemas(),
};
};

// merge the status managed by luPublisher to the LuFile structure to keep a unified interface
private mergeLuStatus = (luFiles: LUFile[], luStatus: { [key: string]: LuisStatus }) => {
return luFiles.map(x => {
if (!luStatus[x.relativePath]) {
throw new Error(`No luis status for lu file ${x.relativePath}`);
}
return {
...x,
status: luStatus[x.relativePath],
};
});
};

public getSchemas = () => {
let editorSchema = this.defaultEditorSchema;
let sdkSchema = this.defaultSDKSchema;
Expand Down Expand Up @@ -204,20 +218,10 @@ export class BotProject {
const isUpdate = !isEqual(currentLufileParsedContentLUISJsonStructure, preLufileParsedContentLUISJsonStructure);
if (!isUpdate) return this.luIndexer.getLuFiles();

const updateTime = Date.now();
await this._updateFile(luFile.relativePath, content);
await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.UPDATE);

const data: ILuisStatusOperation = {};
data[id] = {
content,
parsedContent: currentLufileParsedContentLUISJsonStructure,
lastUpdateTime: updateTime,
};

this.luIndexer.updateLuInMemoryIfUpdate(this.files, data, content);
await this.luIndexer.flush(this.files, luFile.relativePath);

const luFiles = this.luIndexer.getLuFiles();
return luFiles;
return this.mergeLuStatus(this.luIndexer.getLuFiles(), this.luPublisher.status);
};

public createLuFile = async (id: string, content: string, dir: string = ''): Promise<LUFile[]> => {
Expand All @@ -226,42 +230,34 @@ export class BotProject {
throw new Error(`${id} lu file already exist`);
}
const relativePath = Path.join(dir, `${id.trim()}.lu`);
await this.luIndexer.updateLuInMemoryIfCreate(this.files, content, relativePath, id);
await this.luIndexer.flush(this.files, relativePath);
return this.luIndexer.getLuFiles();

// TODO: validate before save
await this._createFile(relativePath, content);
await this.luPublisher.onFileChange(relativePath, FileUpdateType.CREATE); // let publisher know that some files changed
return this.mergeLuStatus(this.luIndexer.getLuFiles(), this.luPublisher.status); // return a merged LUFile always
};

public removeLuFile = async (id: string): Promise<LUFile[]> => {
const luFile = this.luIndexer.getLuFiles().find(lu => lu.id === id);
if (luFile === undefined) {
throw new Error(`no such lu file ${id}`);
}
this.luIndexer.updateLuInMemoryIfRemove(this.files, luFile.relativePath, id);
await this.luIndexer.flush(this.files, luFile.relativePath);
return this.luIndexer.getLuFiles();
await this._removeFile(luFile.relativePath);
await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.DELETE);
return this.mergeLuStatus(this.luIndexer.getLuFiles(), this.luPublisher.status);
};

public setLuisConfig = async (config: ILuisConfig) => {
this.luPublisher.setLuisConfig(config);
};

public publishLuis = async () => {
//TODO luIndexer.getLuFiles() depends on luIndexer.index() not reliable when http call publish
const toPublish = this.luIndexer.getLuFiles().filter(this.isReferred);
const unpublished = await this.luPublisher.getUnpublisedFiles(toPublish);
if (unpublished.length === 0) {
return this.luIndexer.getLuFiles();
}
const referred = this.luIndexer.getLuFiles().filter(this.isReferred);
const unpublished = await this.luPublisher.getUnpublisedFiles(referred);

const invalidLuFile = unpublished.filter(file => file.diagnostics.length !== 0);
if (invalidLuFile.length !== 0) {
const msg = invalidLuFile.reduce((msg, file) => {
const fileErrorText = file.diagnostics.reduce((text, diagnostic) => {
text += `\n ${diagnostic.text}`;
return text;
}, `In ${file.id}.lu: `);
msg += `\n ${fileErrorText} \n`;
return msg;
}, '');
const msg = this.generateErrorMessage(invalidLuFile);
throw new Error(`The Following LuFile(s) are invalid: \n` + msg);
}
const emptyLuFiles = unpublished.filter(this.isEmpty);
Expand All @@ -270,24 +266,14 @@ export class BotProject {
throw new Error(`You have the following empty LuFile(s): ` + msg);
}

const toUpdateLuId = unpublished.map(file => file.id);
const publishTime = Date.now();
const data: ILuisStatusOperation = {};
toUpdateLuId.forEach(
id =>
(data[id] = {
lastPublishTime: publishTime,
})
);

try {
await this.luPublisher.publish(unpublished);
this.luIndexer.updateLuInMemoryIfPublish(this.files, data);
await this.luIndexer.flush(this.files);
if (unpublished.length > 0) {
await this.luPublisher.publish(unpublished);
}
} catch (error) {
throw new Error(error);
throw error;
}
return this.luIndexer.getLuFiles();
return this.mergeLuStatus(this.luIndexer.getLuFiles(), this.luPublisher.status);
};

public checkLuisPublished = async () => {
Expand Down Expand Up @@ -426,16 +412,6 @@ export class BotProject {
}
}

const luisStatusPath = Path.join(this.dir, 'generated/luis.status.json');
if (await this.fileStorage.exists(luisStatusPath)) {
const content = await this.fileStorage.readFile(luisStatusPath);
fileList.push({
name: Path.basename(luisStatusPath),
content,
path: luisStatusPath,
relativePath: Path.relative(this.dir, luisStatusPath),
});
}
return fileList;
};

Expand Down Expand Up @@ -502,4 +478,15 @@ export class BotProject {
}
return false;
};

private generateErrorMessage = (invalidLuFile: LUFile[]) => {
return invalidLuFile.reduce((msg, file) => {
const fileErrorText = file.diagnostics.reduce((text, diagnostic) => {
text += `\n ${diagnostic.text}`;
return text;
}, `In ${file.id}.lu: `);
msg += `\n ${fileErrorText} \n`;
return msg;
}, '');
};
}
Loading