Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Number Series Copilot] Adding Telemetry #2254

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+244 −4
Open

[Number Series Copilot] Adding Telemetry #2254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesCopilotImpl.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ codeunit 324 "No. Series Copilot Impl."
InherentEntitlements = X;

var
NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry";
IncorrectCompletionErr: Label 'Incorrect completion. The property %1 is empty', Comment = '%1 = property name';
EmptyCompletionErr: Label 'Incorrect completion. The completion is empty.';
IncorrectCompletionNumberOfGeneratedNoSeriesErr: Label 'Incorrect completion. The number of generated number series is incorrect. Expected %1, but got %2', Comment = '%1 = Expected Number, %2 = Actual Number';
Expand All @@ -33,15 +34,14 @@ codeunit 324 "No. Series Copilot Impl."

procedure GetNoSeriesSuggestions()
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotRegister: Codeunit "No. Series Copilot Register";
AzureOpenAI: Codeunit "Azure OpenAI";
begin
NoSeriesCopilotRegister.RegisterCapability();
if not AzureOpenAI.IsEnabled(Enum::"Copilot Capability"::"No. Series Copilot") then
exit;

FeatureTelemetry.LogUptake('0000LF4', FeatureName(), Enum::"Feature Uptake Status"::Discovered);
NoSeriesCopilotTelemetry.LogFeatureDiscovery();

Page.Run(Page::"No. Series Generation");
end;
Expand Down Expand Up @@ -184,14 +184,19 @@ codeunit 324 "No. Series Copilot Impl."
AOAIChatMessages.AddTool(ChangeNoSeriesIntent);
AOAIChatMessages.AddTool(NextYearNoSeriesIntent);

NoSeriesCopilotTelemetry.ResetDurationTracking();
NoSeriesCopilotTelemetry.StartDurationTracking();
AzureOpenAI.GenerateChatCompletion(AOAIChatMessages, AOAIChatCompletionParams, AOAIOperationResponse);
NoSeriesCopilotTelemetry.StopDurationTracking();
if not AOAIOperationResponse.IsSuccess() then
Error(AOAIOperationResponse.GetError());

CompletionAnswerTxt := AOAIChatMessages.GetLastMessage(); // the model can answer to rephrase the question, if the user input is not clear

if AOAIOperationResponse.IsFunctionCall() then
CompletionAnswerTxt := GenerateNoSeriesUsingToolResult(AzureOpenAI, InputText, AOAIOperationResponse, AddNoSeriesIntent.GetExistingNoSeries());
CompletionAnswerTxt := GenerateNoSeriesUsingToolResult(AzureOpenAI, InputText, AOAIOperationResponse, AddNoSeriesIntent.GetExistingNoSeries())
else
NoSeriesCopilotTelemetry.LogToolNotInvoked(AOAIOperationResponse);

exit(CompletionAnswerTxt);
end;
Expand Down Expand Up @@ -260,7 +265,10 @@ codeunit 324 "No. Series Copilot Impl."
begin
MaxAttempts := 3;
for Attempt := 1 to MaxAttempts do begin
NoSeriesCopilotTelemetry.ResetDurationTracking();
NoSeriesCopilotTelemetry.StartDurationTracking();
AzureOpenAI.GenerateChatCompletion(AOAIChatMessages, AOAIChatCompletionParams, AOAIOperationResponse);
NoSeriesCopilotTelemetry.StopDurationTracking();
if not AOAIOperationResponse.IsSuccess() then
Error(AOAIOperationResponse.GetError());

Expand All @@ -272,8 +280,10 @@ codeunit 324 "No. Series Copilot Impl."
Error(AOAIFunctionResponse.GetError());

GeneratedNoSeriesArrayText := AOAIFunctionResponse.GetResult();
if CheckIfValidResult(GeneratedNoSeriesArrayText, AOAIFunctionResponse.GetFunctionName(), ExpectedNoSeriesCount) then
if CheckIfValidResult(GeneratedNoSeriesArrayText, AOAIFunctionResponse.GetFunctionName(), ExpectedNoSeriesCount) then begin
NoSeriesCopilotTelemetry.LogGenerationCompletion(ReadGeneratedNumberSeriesJArray(GeneratedNoSeriesArrayText).Count, ExpectedNoSeriesCount, Attempt);
exit(true);
end;

AOAIChatMessages.DeleteMessage(AOAIChatMessages.GetHistory().Count); // remove the last message with wrong assistant response, as we need to regenerate the completion
Sleep(500);
Expand Down
7 changes: 7 additions & 0 deletions src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesGeneration.Page.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ page 332 "No. Series Generation"
}

var
NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry";
InputText: Text;
PageCaptionLbl: text;
IsGenerationDetailsVisible: Boolean;
Expand All @@ -191,14 +192,19 @@ page 332 "No. Series Generation"
begin
if CloseAction = CloseAction::OK then
ApplyGeneratedNoSeries();

NoSeriesCopilotTelemetry.LogFeatureUsage();
end;

local procedure GenerateNoSeries()
var
GeneratedNoSeries: Record "No. Series Generation Detail";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
begin
NoSeriesCopilotTelemetry.StartDurationTracking();
NoSeriesCopilotImpl.Generate(Rec, GeneratedNoSeries, InputText);
NoSeriesCopilotTelemetry.StopDurationTracking();
NoSeriesCopilotTelemetry.SaveTotalSuggestedLines(GeneratedNoSeries.Count());
CurrPage.GenerationDetails.Page.Load(GeneratedNoSeries);
IsGenerationDetailsVisible := not GeneratedNoSeries.IsEmpty;
end;
Expand All @@ -210,5 +216,6 @@ page 332 "No. Series Generation"
begin
CurrPage.GenerationDetails.Page.GetTempRecord(Rec."No.", GeneratedNoSeries);
NoSeriesCopilotImpl.ApplyGeneratedNoSeries(GeneratedNoSeries);
NoSeriesCopilotTelemetry.LogApply(GeneratedNoSeries);
end;
}
219 changes: 219 additions & 0 deletions ...Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
// ------------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
// ------------------------------------------------------------------------------------------------

namespace Microsoft.Foundation.NoSeries;
using System.Telemetry;
using System.AI;

codeunit 389 "No. Series Copilot Telemetry"
{
Access = Internal;
InherentPermissions = X;
InherentEntitlements = X;

var
StartDateTime: DateTime;
Durations: List of [Duration]; // Generate action can be triggered multiple times
TotalSuggestedLines: List of [Integer]; // Generate action can be triggered multiple times
TotalAppliedLines: Integer;

procedure LogFeatureDiscovery()
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
begin
FeatureTelemetry.LogUptake('0000LF4', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Discovered);
FeatureTelemetry.LogUptake('0000XXX', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::"Set up"); // TODO: Replace with actual telemetry event
end;

procedure LogApply(GeneratedNoSeries: Record "No. Series Generation Detail")
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
begin
TotalAppliedLines := GeneratedNoSeries.Count();
if TotalAppliedLines = 0 then
exit;

FeatureTelemetry.LogUptake('0000XXX', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Used, GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries)); // TODO: Replace with actual telemetry event
end;

procedure LogCreateNewNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer)
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
NoSeriesCopAddIntent: Codeunit "No. Series Cop. Add Intent";
begin
FeatureTelemetry.LogUsage('0000XXX', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopAddIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); // TODO: Replace with actual telemetry event
end;

procedure LogModifyExistingNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer; UpdateForNextYear: Boolean)
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
NoSeriesCopChangeIntent: Codeunit "No. Series Cop. Change Intent";
NoSeriesCopNxtYrIntent: Codeunit "No. Series Cop. Nxt Yr. Intent";
begin
if UpdateForNextYear then
FeatureTelemetry.LogUsage('0000XXX', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopNxtYrIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)) // TODO: Replace with actual telemetry event
else
FeatureTelemetry.LogUsage('0000XXX', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopChangeIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); // TODO: Replace with actual telemetry event
end;

local procedure GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer) CustomDimension: Dictionary of [Text, Text]
begin
CustomDimension.Add('TotalUserSpecifiedEntities', Format(TotalUserSpecifiedEntities));
CustomDimension.Add('CustomPatternsUsed', Format(CustomPatternsUsed));
CustomDimension.Add('TotalBatches', Format(TotalBatches));
CustomDimension.Add('TotalFoundTables', Format(TotalFoundTables));
end;

procedure LogFeatureUsage()
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
begin
// TotalAppliedLines will be zero in case none of the lines were inserted.
// We don't want to log telemetry in case the user did not generate any suggestions.
if Durations.Count() = 0 then
exit;

FeatureTelemetry.LogUsage('0000XXX', NoSeriesCopilotImpl.FeatureName(), 'Statistics', GetFeatureTelemetryCustomDimensions()); // TODO: Replace with actual telemetry event
end;

procedure LogGenerationCompletion(TotalGeneratedLines: Integer; TotalExpectedLines: Integer; Attempt: Integer)
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
TelemetryCD: Dictionary of [Text, Text];
begin
TelemetryCD.Add('TotalGeneratedLines', Format(TotalGeneratedLines));
TelemetryCD.Add('TotalExpectedLines', Format(TotalExpectedLines));
TelemetryCD.Add('Attempt', Format(Attempt));
TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations));

FeatureTelemetry.LogUsage('0000XXX', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', TelemetryCD); // TODO: Replace with actual telemetry event
end;


local procedure GetFeatureTelemetryCustomDimensions() CustomDimension: Dictionary of [Text, Text]
begin
CustomDimension.Add('Durations', ConvertListOfDurationToString(Durations));
CustomDimension.Add('TotalSuggestedLines', ConvertListOfIntegerToString(TotalSuggestedLines));
CustomDimension.Add('TotalAppliedLines', Format(TotalAppliedLines));
end;

local procedure GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries: Record "No. Series Generation Detail") CustomDimension: Dictionary of [Text, Text]
var
AppliedValues: Text;
TotalApplied: Integer;
begin
GetAppliedAreas(GeneratedNoSeries, AppliedValues, TotalApplied);
CustomDimension.Add('AppliedAreas', AppliedValues);
CustomDimension.Add('TotalAppliedAreas', Format(TotalApplied));

GetAppliedEntities(GeneratedNoSeries, AppliedValues, TotalApplied);
CustomDimension.Add('AppliedEntities', AppliedValues);
CustomDimension.Add('TotalAppliedEntities', Format(TotalApplied));
end;

local procedure GetAppliedAreas(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedAreas: Text; var TotalAppliedAreas: Integer)
var
AppliedArea: List of [Text];
begin
Clear(AppliedArea);
Clear(TotalAppliedAreas);
if GeneratedNoSeries.FindSet() then
repeat
if not AppliedArea.Contains(GeneratedNoSeries."Setup Table Name") then
AppliedArea.Add(GeneratedNoSeries."Setup Table Name");
until GeneratedNoSeries.Next() = 0;

AppliedAreas := ConvertListOfTextToString(AppliedArea);
TotalAppliedAreas := AppliedArea.Count();
end;

local procedure GetAppliedEntities(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedEntities: Text; var TotalAppliedEntities: Integer)
var
AppliedEntity: List of [Text];
begin
Clear(AppliedEntity);
Clear(TotalAppliedEntities);
if GeneratedNoSeries.FindSet() then
repeat
if not AppliedEntity.Contains(GeneratedNoSeries."Setup Field Name") then
AppliedEntity.Add(GeneratedNoSeries."Setup Field Name");
until GeneratedNoSeries.Next() = 0;

AppliedEntities := ConvertListOfTextToString(AppliedEntity);
TotalAppliedEntities := AppliedEntity.Count();
end;

local procedure ConvertListOfDurationToString(ListOfDuration: List of [Duration]) Result: Text
var
Dur: Duration;
DurationAsBigInt: BigInteger;
begin
foreach Dur in ListOfDuration do begin
DurationAsBigInt := Dur;
Result += Format(DurationAsBigInt) + ', ';
end;
Result := Result.TrimEnd(', ');
end;

local procedure ConvertListOfIntegerToString(ListOfInteger: List of [Integer]) Result: Text
var
Int: Integer;
begin
foreach Int in ListOfInteger do
Result += Format(Int) + ', ';
Result := Result.TrimEnd(', ');
end;

local procedure ConvertListOfTextToString(ListOfText: List of [Text]) Result: Text
var
Text: Text;
begin
foreach Text in ListOfText do
Result += Text + ', ';
Result := Result.TrimEnd(', ');
end;

procedure LogToolNotInvoked(AOAIOperationResponse: Codeunit "AOAI Operation Response")
var
FeatureTelemetry: Codeunit "Feature Telemetry";
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
TelemetryCD: Dictionary of [Text, Text];
begin
if Durations.Count() <> 0 then
TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations));

if AOAIOperationResponse.GetResult() = '' then
FeatureTelemetry.LogError('0000XXX', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', 'Completion answer is empty', '', TelemetryCD) // TODO: Replace with actual telemetry event
else
FeatureTelemetry.LogError('0000XXX', NoSeriesCopilotImpl.FeatureName(), 'Process function_call', 'function_call not found in the completion answer'); // TODO: Replace with actual telemetry event
end;

procedure ResetDurationTracking()
begin
Clear(Durations);
end;

procedure StartDurationTracking()
begin
StartDateTime := CurrentDateTime();
end;

procedure StopDurationTracking() Duration: Duration
begin
Durations.Add(CurrentDateTime() - StartDateTime);
end;

procedure SaveTotalSuggestedLines(Total: Integer)
begin
TotalSuggestedLines.Add(Total);
end;

}
2 changes: 2 additions & 0 deletions ...usiness Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopAddIntent.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ codeunit 331 "No. Series Cop. Add Intent" implements "AOAI Function"
var
TempNoSeriesField: Record "Field" temporary;
TempSetupTable: Record "Table Metadata" temporary;
NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry";
NewNoSeriesPrompt, CustomPatternsPromptList, TablesYamlList, EmptyList : List of [Text];
NumberOfToolResponses, i, ActualTablesChunkSize : Integer;
NumberOfAddedTables: Integer;
Expand All @@ -85,6 +86,7 @@ codeunit 331 "No. Series Cop. Add Intent" implements "AOAI Function"
ToolResults.Add(ToolsImpl.ConvertListToText(NewNoSeriesPrompt), ActualTablesChunkSize);
end;
Progress.Close();
NoSeriesCopilotTelemetry.LogCreateNewNumberSeriesToolUsage(ToolsImpl.GetEntities(Arguments).Count, CustomPatternsPromptList.Count > 0, NumberOfToolResponses, NumberOfAddedTables);
end;

local procedure GetTablesRequireNoSeries(var Arguments: JsonObject; var TempSetupTable: Record "Table Metadata" temporary; var TempNoSeriesField: Record "Field" temporary)
Expand Down
2 changes: 2 additions & 0 deletions ...ness Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopChangeIntent.Codeunit.al
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ codeunit 334 "No. Series Cop. Change Intent" implements "AOAI Function"
TempSetupTable: Record "Table Metadata" temporary;
TempNoSeriesField: Record "Field" temporary;
NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl.";
NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry";
ChangeNoSeriesPrompt, CustomPatternsPromptList, TablesYamlList, ExistingNoSeriesToChangeList : List of [Text];
NumberOfToolResponses, i, ActualTablesChunkSize : Integer;
NumberOfChangedTables: Integer;
Expand Down Expand Up @@ -102,6 +103,7 @@ codeunit 334 "No. Series Cop. Change Intent" implements "AOAI Function"
ToolResults.Add(ToolsImpl.ConvertListToText(ChangeNoSeriesPrompt), ActualTablesChunkSize);
end;
Progress.Close();
NoSeriesCopilotTelemetry.LogModifyExistingNumberSeriesToolUsage(ToolsImpl.GetEntities(Arguments).Count, CustomPatternsPromptList.Count > 0, NumberOfToolResponses, NumberOfChangedTables, UpdateForNextYear);
end;

[TryFunction]
Expand Down
Loading