Skip to content

Commit dabc4f4

Browse files
garyprettyTom Laird-McConnellcarlosscastrostevengumswagatmishra2007
authored
Fixes for QnA / Recognizer Adaptive telemetry (#3715)
* Fix AdaptiveTesting to have correct preview labels * Proactive auth: trust incoming token url conditionally (#3533) * Add unit test for CancelAllDialogs and fix eventName and eventValue could be null (#3536) * fix lack of a generic typed method for posting activities to skills (#3539) (#3542) * fix lack of a generic typed method for posting activities to skills * remove duplicated code Co-authored-by: Swagat Mishra <swagatm@microsoft.com> Co-authored-by: swagat mishra <swagatmishra2007@gmail.com> Co-authored-by: Swagat Mishra <swagatm@microsoft.com> * EoC should only be handled when coming from parent in RunAsync (#3540) (#3543) * Added check to only cancel dialogs in RunAsync when EoC comes from a parent bot (root or skill). * Applied URI prefix to caller ID as per OBI spec https://github.com/microsoft/botframework-obi/blob/master/protocols/botframework-activity/botframework-activity.md#bot-calling-skill (cherry picked from commit 3730963) * Merge pull request #3564 from microsoft/tedlee/fix-oauth-credentials (#3571) [OAuth]do not set signInLink to empty when OAuthAppCredentials is set * Address QnAMaker / recognizer telemetry across dialogs / adaptive * Fixes following merge from master * Added comments to the TrackRecognizerResult method on the Recognizer class. * More comments. Co-authored-by: Tom Laird-McConnell <tomlm@microsoft.com> Co-authored-by: Carlos Castro <carlosscastro@users.noreply.github.com> Co-authored-by: Steven Gum <14935595+stevengum@users.noreply.github.com> Co-authored-by: swagat mishra <swagatmishra2007@gmail.com> Co-authored-by: Swagat Mishra <swagatm@microsoft.com> Co-authored-by: Gabo Gilabert <gabog@users.noreply.github.com>
1 parent b524ea5 commit dabc4f4

File tree

11 files changed

+117
-34
lines changed

11 files changed

+117
-34
lines changed

‎libraries/Microsoft.Bot.Builder.AI.QnA/Dialogs/QnAMakerDialog.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ public QnAMakerDialog([CallerFilePath] string sourceFilePath = "", [CallerLineNu
164164
[JsonIgnore]
165165
public HttpClient HttpClient { get; set; }
166166

167+
/// <summary>
168+
/// Gets or sets a value indicating whether to log personal information that came from the user to telemetry.
169+
/// </summary>
170+
/// <value>If true, personal information is logged to Telemetry; otherwise the properties will be filtered.</value>
171+
public bool LogPersonalInformation { get; set; } = false;
172+
167173
/// <summary>
168174
/// Called when the dialog is started and pushed onto the dialog stack.
169175
/// </summary>
@@ -229,7 +235,7 @@ protected async virtual Task<IQnAMakerClient> GetQnAMakerClientAsync(DialogConte
229235
KnowledgeBaseId = this.knowledgeBaseId
230236
};
231237
var options = await GetQnAMakerOptionsAsync(dc).ConfigureAwait(false);
232-
return new QnAMaker(endpoint, options, HttpClient);
238+
return new QnAMaker(endpoint, options, HttpClient, this.TelemetryClient, this.LogPersonalInformation);
233239
}
234240

235241
/// <summary>

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/LUIS/LuisAdaptiveRecognizer.cs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,13 @@ public LuisAdaptiveRecognizer()
8181
public HttpClientHandler HttpClient { get; set; }
8282

8383
/// <summary>
84-
/// Gets or sets a value indicating whether to log personal information that came from the user to telemetry.
84+
/// Gets or sets the flag to determine if personal information should be logged in telemetry.
8585
/// </summary>
86-
/// <value>If true, personal information is logged to Telemetry; otherwise the properties will be filtered.</value>
87-
public bool LogPersonalInformation { get; set; } = false;
86+
/// <value>
87+
/// The flag to indicate in personal information should be logged in telemetry.
88+
/// </value>
89+
[JsonProperty("logPersonalInformation")]
90+
public BoolExpression LogPersonalInformation { get; set; } = "=settings.telemetry.logPersonalInformation";
8891

8992
/// <inheritdoc/>
9093
public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialogContext, Activity activity, CancellationToken cancellationToken = default, Dictionary<string, string> telemetryProperties = null, Dictionary<string, double> telemetryMetrics = null)
@@ -100,7 +103,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
100103

101104
var result = await wrapper.RecognizeAsync(tempContext, cancellationToken).ConfigureAwait(false);
102105

103-
this.TelemetryClient.TrackEvent("LuisResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties, dialogContext.Context), telemetryMetrics);
106+
this.TrackRecognizerResult(dialogContext, "LuisResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties, dialogContext), telemetryMetrics);
104107

105108
return result;
106109
}
@@ -134,19 +137,22 @@ public LuisRecognizerOptionsV3 RecognizerOptions(DialogContext dialogContext)
134137
};
135138
}
136139

137-
protected override Dictionary<string, string> FillRecognizerResultTelemetryProperties(RecognizerResult recognizerResult, Dictionary<string, string> telemetryProperties, ITurnContext turnContext)
140+
protected override Dictionary<string, string> FillRecognizerResultTelemetryProperties(RecognizerResult recognizerResult, Dictionary<string, string> telemetryProperties, DialogContext dc)
138141
{
142+
var (logPersonalInfo, error) = this.LogPersonalInformation.TryGetValue(dc.State);
143+
var (applicationId, error2) = this.ApplicationId.TryGetValue(dc.State);
144+
139145
var topTwoIntents = (recognizerResult.Intents.Count > 0) ? recognizerResult.Intents.OrderByDescending(x => x.Value.Score).Take(2).ToArray() : null;
140146

141147
// Add the intent score and conversation id properties
142148
var properties = new Dictionary<string, string>()
143149
{
144-
{ LuisTelemetryConstants.ApplicationIdProperty, ApplicationId.ExpressionText },
150+
{ LuisTelemetryConstants.ApplicationIdProperty, applicationId },
145151
{ LuisTelemetryConstants.IntentProperty, topTwoIntents?[0].Key ?? string.Empty },
146152
{ LuisTelemetryConstants.IntentScoreProperty, topTwoIntents?[0].Value.Score?.ToString("N2") ?? "0.00" },
147153
{ LuisTelemetryConstants.Intent2Property, (topTwoIntents?.Count() > 1) ? topTwoIntents?[1].Key ?? string.Empty : string.Empty },
148154
{ LuisTelemetryConstants.IntentScore2Property, (topTwoIntents?.Count() > 1) ? topTwoIntents?[1].Value.Score?.ToString("N2") ?? "0.00" : "0.00" },
149-
{ LuisTelemetryConstants.FromIdProperty, turnContext.Activity.From.Id },
155+
{ LuisTelemetryConstants.FromIdProperty, dc.Context.Activity.From.Id },
150156
};
151157

152158
if (recognizerResult.Properties.TryGetValue("sentiment", out var sentiment) && sentiment is JObject)
@@ -166,9 +172,9 @@ protected override Dictionary<string, string> FillRecognizerResultTelemetryPrope
166172
properties.Add(LuisTelemetryConstants.EntitiesProperty, entities);
167173

168174
// Use the LogPersonalInformation flag to toggle logging PII data, text is a common example
169-
if (LogPersonalInformation && !string.IsNullOrEmpty(turnContext.Activity.Text))
175+
if (logPersonalInfo && !string.IsNullOrEmpty(dc.Context.Activity.Text))
170176
{
171-
properties.Add(LuisTelemetryConstants.QuestionProperty, turnContext.Activity.Text);
177+
properties.Add(LuisTelemetryConstants.QuestionProperty, dc.Context.Activity.Text);
172178
}
173179

174180
// Additional Properties can override "stock" properties.

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/QnAMaker/QnAMakerDialog2.cs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,16 @@ public QnAMakerDialog2([CallerFilePath] string sourceFilePath = "", [CallerLineN
136136
public ArrayExpression<Metadata> StrictFilters { get; set; }
137137

138138
/// <summary>
139-
/// Gets or sets a value indicating whether to call test or prod environment of knowledge base to be called.
139+
/// Gets or sets the flag to determine if personal information should be logged in telemetry.
140+
/// </summary>
141+
/// <value>
142+
/// The flag to indicate in personal information should be logged in telemetry.
143+
/// </value>
144+
[JsonProperty("logPersonalInformation")]
145+
public new BoolExpression LogPersonalInformation { get; set; } = "=settings.telemetry.logPersonalInformation";
146+
147+
/// <summary>
148+
/// Gets or sets a value indicating whether gets or sets environment of knowledgebase to be called.
140149
/// </summary>
141150
/// <value>
142151
/// A value indicating whether to call test or prod environment of knowledge base.
@@ -172,6 +181,7 @@ protected async override Task<IQnAMakerClient> GetQnAMakerClientAsync(DialogCont
172181
var (epKey, _) = this.EndpointKey.TryGetValue(dc.State);
173182
var (hn, _) = this.HostName.TryGetValue(dc.State);
174183
var (kbId, _) = this.KnowledgeBaseId.TryGetValue(dc.State);
184+
var (logPersonalInformation, _) = this.LogPersonalInformation.TryGetValue(dc.State);
175185

176186
var endpoint = new QnAMakerEndpoint
177187
{
@@ -180,7 +190,8 @@ protected async override Task<IQnAMakerClient> GetQnAMakerClientAsync(DialogCont
180190
KnowledgeBaseId = kbId
181191
};
182192
var options = await GetQnAMakerOptionsAsync(dc).ConfigureAwait(false);
183-
return new QnAMaker(endpoint, options, this.HttpClient);
193+
194+
return new QnAMaker(endpoint, options, this.HttpClient, this.TelemetryClient, (bool)logPersonalInformation);
184195
}
185196

186197
/// <summary>

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/QnAMaker/QnAMakerRecognizer.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,15 @@ public QnAMakerRecognizer()
131131
[JsonIgnore]
132132
public HttpClient HttpClient { get; set; }
133133

134+
/// <summary>
135+
/// Gets or sets the flag to determine if personal information should be logged in telemetry.
136+
/// </summary>
137+
/// <value>
138+
/// The flag to indicate in personal information should be logged in telemetry.
139+
/// </value>
140+
[JsonProperty("logPersonalInformation")]
141+
public BoolExpression LogPersonalInformation { get; set; } = "=settings.telemetry.logPersonalInformation";
142+
134143
public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialogContext, Activity activity, CancellationToken cancellationToken, Dictionary<string, string> telemetryProperties = null, Dictionary<string, double> telemetryMetrics = null)
135144
{
136145
// Identify matched intents
@@ -210,7 +219,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
210219
recognizerResult.Intents.Add("None", new IntentScore() { Score = 1.0f });
211220
}
212221

213-
this.TelemetryClient.TrackEvent("QnAMakerRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognizerResult, telemetryProperties), telemetryMetrics);
222+
this.TrackRecognizerResult(dialogContext, "QnAMakerRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognizerResult, telemetryProperties), telemetryMetrics);
214223

215224
return recognizerResult;
216225
}
@@ -227,6 +236,7 @@ protected virtual Task<IQnAMakerClient> GetQnAMakerClientAsync(DialogContext dc)
227236
var (epKey, error) = this.EndpointKey.TryGetValue(dc.State);
228237
var (hn, error2) = this.HostName.TryGetValue(dc.State);
229238
var (kbId, error3) = this.KnowledgeBaseId.TryGetValue(dc.State);
239+
var (logPersonalInfo, error4) = this.LogPersonalInformation.TryGetValue(dc.State);
230240

231241
var endpoint = new QnAMakerEndpoint
232242
{
@@ -235,7 +245,7 @@ protected virtual Task<IQnAMakerClient> GetQnAMakerClientAsync(DialogContext dc)
235245
KnowledgeBaseId = (string)kbId ?? throw new ArgumentNullException(nameof(KnowledgeBaseId), error3)
236246
};
237247

238-
return Task.FromResult<IQnAMakerClient>(new QnAMaker(endpoint, httpClient: this.HttpClient));
248+
return Task.FromResult<IQnAMakerClient>(new QnAMaker(endpoint, new QnAMakerOptions(), this.HttpClient, this.TelemetryClient, (bool)logPersonalInfo));
239249
}
240250
}
241251
}

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Recognizers/CrossTrainedRecognizerSet.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
8989

9090
var result = ProcessResults(results);
9191

92-
this.TelemetryClient.TrackEvent("CrossTrainedRecognizerSetResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
92+
this.TrackRecognizerResult(dialogContext, "CrossTrainedRecognizerSetResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
9393

9494
return result;
9595
}

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Recognizers/MultiLanguageRecognizer.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
6060
if (this.Recognizers.TryGetValue(option, out var recognizer))
6161
{
6262
var result = await recognizer.RecognizeAsync(dialogContext, activity, cancellationToken, telemetryProperties, telemetryMetrics).ConfigureAwait(false);
63-
this.TelemetryClient.TrackEvent("MultiLanguagesRecognizerResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
63+
this.TrackRecognizerResult(dialogContext, "MultiLanguagesRecognizerResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
6464
return result;
6565
}
6666
}
6767

68-
this.TelemetryClient.TrackEvent("MultiLanguagesRecognizerResult", this.FillRecognizerResultTelemetryProperties(new RecognizerResult() { }, telemetryProperties), telemetryMetrics);
68+
this.TrackRecognizerResult(dialogContext, "MultiLanguagesRecognizerResult", this.FillRecognizerResultTelemetryProperties(new RecognizerResult() { }, telemetryProperties), telemetryMetrics);
6969

7070
// nothing recognized
7171
return new RecognizerResult() { };

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Recognizers/RecognizerSet.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
5959
// merge intents
6060
var result = MergeResults(results);
6161

62-
this.TelemetryClient.TrackEvent("RecognizerSetResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
62+
this.TrackRecognizerResult(dialogContext, "RecognizerSetResult", this.FillRecognizerResultTelemetryProperties(result, telemetryProperties), telemetryMetrics);
6363

6464
return result;
6565
}

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Recognizers/RegexRecognizer.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
5151
// Identify matched intents
5252
var text = activity.Text ?? string.Empty;
5353
var locale = activity.Locale ?? "en-us";
54-
54+
5555
var recognizerResult = new RecognizerResult()
5656
{
5757
Text = text,
@@ -171,7 +171,7 @@ public override async Task<RecognizerResult> RecognizeAsync(DialogContext dialog
171171

172172
await dialogContext.Context.TraceActivityAsync(nameof(RegexRecognizer), JObject.FromObject(recognizerResult), "RecognizerResult", "Regex RecognizerResult", cancellationToken).ConfigureAwait(false);
173173

174-
this.TelemetryClient.TrackEvent("RegexRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognizerResult, telemetryProperties), telemetryMetrics);
174+
this.TrackRecognizerResult(dialogContext, "RegexRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognizerResult, telemetryProperties), telemetryMetrics);
175175

176176
return recognizerResult;
177177
}

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/Recognizers/ValueRecognizer.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ public override Task<RecognizerResult> RecognizeAsync(DialogContext dialogContex
7676
}
7777
}
7878

79-
this.TelemetryClient.TrackEvent("ValueRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognized, telemetryProperties), telemetryMetrics);
79+
this.TrackRecognizerResult(dialogContext, "ValueRecognizerResult", this.FillRecognizerResultTelemetryProperties(recognized, telemetryProperties), telemetryMetrics);
8080

8181
return Task.FromResult(recognized);
8282
}

‎libraries/Microsoft.Bot.Builder.Dialogs.Adaptive/TelemetryExtensions.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
namespace Microsoft.Bot.Builder.Dialogs.Adaptive
5+
{
6+
public static class TelemetryExtensions
7+
{
8+
/// <summary>
9+
/// Register IBotTelemetryClient as default langugage generator.
10+
/// </summary>
11+
/// <param name="dialogManager">botAdapter to add services to.</param>
12+
/// <param name="telemetryClient">IBotTelemetryClient to use.</param>
13+
/// <returns>botAdapter.</returns>
14+
public static DialogManager UseTelemetry(this DialogManager dialogManager, IBotTelemetryClient telemetryClient)
15+
{
16+
dialogManager.TurnState.Set<IBotTelemetryClient>(telemetryClient);
17+
dialogManager.Dialogs.TelemetryClient = telemetryClient;
18+
return dialogManager;
19+
}
20+
}
21+
}

0 commit comments

Comments
 (0)