From a9003e59ea9df7795c2c6d1dcb660ccdb1295775 Mon Sep 17 00:00:00 2001 From: Josh Gummersall <1235378+joshgummersall@users.noreply.github.com> Date: Wed, 24 Mar 2021 17:37:55 -0700 Subject: [PATCH] port: add speach middleware to runtime (#3435) * port: add speak middleware to runtime Fixes https://github.com/microsoft/botbuilder-js/issues/3432 * fix: add telephony to channels, remove console * fix: coerce to strings --- libraries/botbuilder-runtime/src/index.ts | 20 +++++ libraries/botbuilder/package.json | 3 +- libraries/botbuilder/src/index.ts | 5 +- .../botbuilder/src/setSpeakMiddleware.ts | 89 +++++++++++++++++++ .../tests/setSpeakMiddleware.test.js | 84 +++++++++++++++++ libraries/botframework-schema/src/index.ts | 1 + yarn.lock | 40 +++++++++ 7 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 libraries/botbuilder/src/setSpeakMiddleware.ts create mode 100644 libraries/botbuilder/tests/setSpeakMiddleware.test.js diff --git a/libraries/botbuilder-runtime/src/index.ts b/libraries/botbuilder-runtime/src/index.ts index 4e10263c83..b86b95d7ac 100644 --- a/libraries/botbuilder-runtime/src/index.ts +++ b/libraries/botbuilder-runtime/src/index.ts @@ -32,6 +32,7 @@ import { MemoryStorage, MiddlewareSet, NullTelemetryClient, + SetSpeakMiddleware, ShowTypingMiddleware, SkillHandler, SkillHttpClient, @@ -49,6 +50,25 @@ function addFeatures(services: ServiceCollection, configuration: Conf middlewareSet.use(new ShowTypingMiddleware()); } + const setSpeak = await configuration.type( + ['setSpeak'], + t.Record({ + voiceFontName: t.String.Or(t.Undefined), + lang: t.String, + fallbackToTextForSpeechIfEmpty: t.Boolean, + }) + ); + + if (setSpeak) { + middlewareSet.use( + new SetSpeakMiddleware( + setSpeak.voiceFontName ?? null, + setSpeak.lang, + setSpeak.fallbackToTextForSpeechIfEmpty + ) + ); + } + if (await configuration.bool(['traceTranscript'])) { const blobsTranscript = await configuration.type( ['blobTranscript'], diff --git a/libraries/botbuilder/package.json b/libraries/botbuilder/package.json index 096db35041..981062e0a0 100644 --- a/libraries/botbuilder/package.json +++ b/libraries/botbuilder/package.json @@ -33,9 +33,10 @@ "botbuilder-stdlib": "4.1.6", "botframework-connector": "4.1.6", "botframework-streaming": "4.1.6", + "dayjs": "^1.10.3", "filenamify": "^4.1.0", "fs-extra": "^7.0.1", - "dayjs": "^1.10.3", + "htmlparser2": "^6.0.1", "uuid": "^8.3.2" }, "devDependencies": { diff --git a/libraries/botbuilder/src/index.ts b/libraries/botbuilder/src/index.ts index 16ece7ec17..90b0d3f5ca 100644 --- a/libraries/botbuilder/src/index.ts +++ b/libraries/botbuilder/src/index.ts @@ -6,13 +6,16 @@ * Licensed under the MIT License. */ +export * from 'botbuilder-core'; + export * from './fileTranscriptStore'; export * from './inspectionMiddleware'; +export * from './setSpeakMiddleware'; export * from './skills'; export * from './teamsActivityHandler'; export * from './teamsActivityHelpers'; export * from './teamsInfo'; -export * from 'botbuilder-core'; + export { BotFrameworkAdapter, BotFrameworkAdapterSettings } from './botFrameworkAdapter'; export { BotFrameworkHttpClient } from './botFrameworkHttpClient'; export { ChannelServiceHandler } from './channelServiceHandler'; diff --git a/libraries/botbuilder/src/setSpeakMiddleware.ts b/libraries/botbuilder/src/setSpeakMiddleware.ts new file mode 100644 index 0000000000..1f1a570148 --- /dev/null +++ b/libraries/botbuilder/src/setSpeakMiddleware.ts @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ActivityTypes, Channels, Middleware, TurnContext } from 'botbuilder-core'; +import { parseDocument } from 'htmlparser2'; +import { tests } from 'botbuilder-stdlib'; + +const supportedChannels = new Set([Channels.DirectlineSpeech, Channels.Emulator, Channels.Telephony]); + +// Iterate through `obj` and all children in an attempt to locale a key `tag` +function hasTag(tag: string, nodes: unknown[]): boolean { + while (nodes.length) { + const item = nodes.shift(); + + if (tests.isDictionary(item)) { + if (item.tagName === tag) { + return true; + } + + if (tests.isArray(item.children)) { + nodes.push(...item.children); + } + } + } + + return false; +} + +/** + * Support the DirectLine speech and telephony channels to ensure the appropriate SSML tags are set on the + * Activity Speak property. + */ +export class SetSpeakMiddleware implements Middleware { + /** + * Initializes a new instance of the SetSpeakMiddleware class. + * + * @param voiceName The SSML voice name attribute value. + * @param lang The xml:lang value. + * @param fallbackToTextForSpeak true if an empty Activity.Speak is populated with Activity.Text. + */ + constructor( + private readonly voiceName: string | null, + private readonly lang: string, + private readonly fallbackToTextForSpeak: boolean + ) { + if (!lang) throw new TypeError('`lang` must be a non-empty string'); + } + + /** + * Processes an incoming activity. + * + * @param turnContext The context object for this turn. + * @param next The delegate to call to continue the bot middleware pipeline. + * @returns A promise representing the async operation. + */ + onTurn(turnContext: TurnContext, next: () => Promise): Promise { + turnContext.onSendActivities(async (_ctx, activities, next) => { + await Promise.all( + activities.map(async (activity) => { + if (activity.type !== ActivityTypes.Message) { + return; + } + + if (this.fallbackToTextForSpeak && !activity.speak) { + activity.speak = activity.text; + } + + const channelId = turnContext.activity.channelId?.trim().toLowerCase(); + + if (activity.speak && this.voiceName !== null && supportedChannels.has(channelId)) { + const nodes = parseDocument(activity.speak).childNodes; + + if (!hasTag('speak', nodes.slice())) { + if (!hasTag('voice', nodes.slice())) { + activity.speak = `${activity.speak}`; + } + + activity.speak = `${activity.speak}`; + } + } + }) + ); + + return next(); + }); + + return next(); + } +} diff --git a/libraries/botbuilder/tests/setSpeakMiddleware.test.js b/libraries/botbuilder/tests/setSpeakMiddleware.test.js new file mode 100644 index 0000000000..24568a65c6 --- /dev/null +++ b/libraries/botbuilder/tests/setSpeakMiddleware.test.js @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('assert'); +const { SetSpeakMiddleware } = require('..'); +const { MessageFactory, TestAdapter } = require('botbuilder-core'); + +describe('SetSpeakMiddleware', function () { + describe('constructor', function () { + it('works', function () { + new SetSpeakMiddleware('voiceName', 'lang', false); + }); + + it('throws for falsy lang param', function () { + assert.throws(() => new SetSpeakMiddleware('voiceName', '', false)); + }); + }); + + describe('onTurn', function () { + function makeAdapter({ + channelId = 'emulator', + fallback = true, + logic = async (context) => { + await context.sendActivity(MessageFactory.text('OK')); + }, + voice = 'male', + } = {}) { + return new TestAdapter(logic, { channelId }).use(new SetSpeakMiddleware(voice, 'en-us', fallback)); + } + + it('no fallback does nothing to speak', async function () { + const adapter = makeAdapter({ fallback: false }); + + await adapter + .send('foo') + .assertReply((activity) => assert(activity.speak == null)) + .startTest(); + }); + + it('unsupported channel and empty speak yields speak === text', async function () { + const adapter = makeAdapter({ channelId: 'doesnotsupportspeach' }); + + await adapter + .send('foo') + .assertReply((activity) => assert.strictEqual(activity.speak, activity.text)) + .startTest(); + }); + + it('unsupported channel and non-empty value yields untouched speak', async function () { + const adapter = makeAdapter({ + channelId: 'doesnotsupportspeach', + logic: async (context) => { + const activity = MessageFactory.text('OK'); + activity.speak = 'custom speak'; + + await context.sendActivity(activity); + }, + }); + + await adapter + .send('foo') + .assertReply((activity) => assert.strictEqual(activity.speak, 'custom speak')) + .startTest(); + }); + + it('supported channel yields speak with voice added', async function () { + const adapter = makeAdapter(); + + await adapter + .send('foo') + .assertReply((activity) => assert(activity.speak.includes(' assert.strictEqual(activity.speak, 'OK')) + .startTest(); + }); + }); +}); diff --git a/libraries/botframework-schema/src/index.ts b/libraries/botframework-schema/src/index.ts index 752174523f..4e6b513b03 100644 --- a/libraries/botframework-schema/src/index.ts +++ b/libraries/botframework-schema/src/index.ts @@ -2118,6 +2118,7 @@ export enum Channels { Slack = 'slack', Sms = 'sms', Telegram = 'telegram', + Telephony = 'telephony', Test = 'test', Twilio = 'twilio-sms', Webchat = 'webchat', diff --git a/yarn.lock b/yarn.lock index 190c536433..b41ebdba3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4422,6 +4422,15 @@ dom-serializer@0: domelementtype "^2.0.1" entities "^2.0.0" +dom-serializer@^1.0.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.2.0.tgz#3433d9136aeb3c627981daa385fc7f32d27c48f1" + integrity sha512-n6kZFH/KlCrqs/1GHMOd5i2fd/beQHuehKdWvNNffbGHTr/almdhuVvTVFb3V7fglz+nC50fFusu3lY33h12pA== + dependencies: + domelementtype "^2.0.1" + domhandler "^4.0.0" + entities "^2.0.0" + domain-browser@^1.1.1, domain-browser@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda" @@ -4437,6 +4446,11 @@ domelementtype@^2.0.1: resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.0.2.tgz#f3b6e549201e46f588b59463dd77187131fe6971" integrity sha512-wFwTwCVebUrMgGeAwRL/NhZtHAUyT9n9yg4IMDwf10+6iCMxSkVq9MGCVEH+QZWo1nNidy8kNvwmv4zWHDTqvA== +domelementtype@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.1.0.tgz#a851c080a6d1c3d94344aed151d99f669edf585e" + integrity sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w== + domhandler@2.3: version "2.3.0" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.3.0.tgz#2de59a0822d5027fabff6f032c2b25a2a8abe738" @@ -4444,6 +4458,13 @@ domhandler@2.3: dependencies: domelementtype "1" +domhandler@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.0.0.tgz#01ea7821de996d85f69029e81fa873c21833098e" + integrity sha512-KPTbnGQ1JeEMQyO1iYXoagsI6so/C96HZiFyByU3T6iAzpXn8EGEvct6unm1ZGoed8ByO2oirxgwxBmqKF9haA== + dependencies: + domelementtype "^2.1.0" + domutils@1.5: version "1.5.1" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" @@ -4452,6 +4473,15 @@ domutils@1.5: dom-serializer "0" domelementtype "1" +domutils@^2.4.4: + version "2.5.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.5.0.tgz#42f49cffdabb92ad243278b331fd761c1c2d3039" + integrity sha512-Ho16rzNMOFk2fPwChGh3D2D9OEHAfG19HgmRR2l+WLSsIstNsAYBzePH412bL0y5T44ejABIVfTHQ8nqi/tBCg== + dependencies: + dom-serializer "^1.0.1" + domelementtype "^2.0.1" + domhandler "^4.0.0" + dotenv@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-7.0.0.tgz#a2be3cd52736673206e8a85fb5210eea29628e7c" @@ -6529,6 +6559,16 @@ htmlparser2@3.8.x: entities "1.0" readable-stream "1.1" +htmlparser2@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.0.1.tgz#422521231ef6d42e56bd411da8ba40aa36e91446" + integrity sha512-GDKPd+vk4jvSuvCbyuzx/unmXkk090Azec7LovXP8as1Hn8q9p3hbjmDGbUqqhknw0ajwit6LiiWqfiTUPMK7w== + dependencies: + domelementtype "^2.0.1" + domhandler "^4.0.0" + domutils "^2.4.4" + entities "^2.0.0" + http-cache-semantics@3.8.1: version "3.8.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"