Add Microsoft.ML.GenAI.Phi, test package and sample project.

Microsoft.ML.sln

@@ -176,7 +176,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TorchSharp.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TensorFlow.Tests", "test\Microsoft.ML.TensorFlow.Tests\Microsoft.ML.TensorFlow.Tests.csproj", "{763FF013-8309-4680-A769-B54E7BB99612}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core", "src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj", "{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core", "src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj", "{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi", "src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj", "{694BF884-B2E4-4E1C-9342-0564BAAC4575}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Tests", "test\Microsoft.ML.GenAI.Phi.Tests\Microsoft.ML.GenAI.Phi.Tests.csproj", "{867FFC34-DFA7-400F-B9BB-85158326CE08}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -838,6 +846,38 @@ Global
 		{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|Any CPU.Build.0 = Release|Any CPU
 		{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|x64.ActiveCfg = Release|Any CPU
 		{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|x64.Build.0 = Release|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|x64.Build.0 = Debug|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|Any CPU.Build.0 = Release|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|x64.ActiveCfg = Release|Any CPU
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|x64.Build.0 = Release|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|x64.Build.0 = Debug|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|Any CPU.Build.0 = Release|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|x64.ActiveCfg = Release|Any CPU
+		{867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|x64.Build.0 = Release|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|x64.Build.0 = Debug|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|x64.ActiveCfg = Release|Any CPU
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|x64.Build.0 = Release|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Debug|x64.Build.0 = Debug|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
+		{14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -925,6 +965,10 @@ Global
 		{AB8D68F1-6C3E-41FD-B0EC-A093E009341D} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{763FF013-8309-4680-A769-B54E7BB99612} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{DB2CA055-8ABD-4E3E-8089-5B64C3415E85} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+		{694BF884-B2E4-4E1C-9342-0564BAAC4575} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+		{867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
+		{14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="TorchSharp-cuda-windows" Version="0.102.5" Condition="$([MSBuild]::IsOSPlatform('Windows'))" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="$(SemanticKernelVersion)" />
+  </ItemGroup>
+
+</Project>

docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using AutoGen.Core;
+using Microsoft.ML.GenAI.Phi;
+using static TorchSharp.torch;
+using TorchSharp;
+using Microsoft.ML.GenAI.Core;
+using Microsoft.ML.GenAI.Core.Extension;
+
+namespace Microsoft.ML.GenAI.Samples.Phi3Mini;
+
+public class AutoGenSample
+{
+    public static async Task RunAsync()
+    {
+        var device = "cuda";
+        if (device == "cuda")
+        {
+            torch.InitializeDeviceType(DeviceType.CUDA);
+        }
+
+        var defaultType = ScalarType.Float16;
+        torch.manual_seed(1);
+        torch.set_default_dtype(defaultType);
+        var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
+        var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
+
+        // agent
+        var agent = new Phi3Agent(pipeline, "assistant")
+            .RegisterPrintMessage();
+        var question = @"write a C# program to calculate the factorial of a number";
+
+        // chat with the assistant
+        await agent.SendAsync(question);
+    }
+}

docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/SemanticKernelSample.cs

@@ -0,0 +1,62 @@
+using Microsoft.ML.GenAI.Phi.Extension;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using TorchSharp;
+using static TorchSharp.torch;
+
+namespace Microsoft.ML.GenAI.Samples.Phi3Mini;
+
+public class SemanticKernelSample
+{
+    public static async Task RunChatCompletionSample()
+    {
+        var device = "cuda";
+        if (device == "cuda")
+        {
+            torch.InitializeDeviceType(DeviceType.CUDA);
+        }
+
+        var defaultType = ScalarType.Float16;
+        torch.manual_seed(1);
+        torch.set_default_dtype(defaultType);
+        var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
+        var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
+
+
+        var kernel = Kernel.CreateBuilder()
+            .AddPhi3AsChatCompletion(pipeline)
+            .Build();
+        var chatService = kernel.GetRequiredService<IChatCompletionService>();
+        var chatHistory = new ChatHistory();
+        chatHistory.AddSystemMessage("you are a helpful assistant");
+        chatHistory.AddUserMessage("write a C# program to calculate the factorial of a number");
+
+        await foreach (var response in chatService.GetStreamingChatMessageContentsAsync(chatHistory))
+        {
+            Console.Write(response);
+        }
+    }
+
+    public static async Task RunTextGenerationSample()
+    {
+        var device = "cuda";
+        if (device == "cuda")
+        {
+            torch.InitializeDeviceType(DeviceType.CUDA);
+        }
+
+        var defaultType = ScalarType.Float16;
+        torch.manual_seed(1);
+        torch.set_default_dtype(defaultType);
+        var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
+        var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device);
+
+
+        var kernel = Kernel.CreateBuilder()
+            .AddPhi3AsTextGeneration(pipeline)
+            .Build();
+
+        var response = await kernel.InvokePromptAsync("Tell a joke");
+        Console.WriteLine(response);
+    }
+}

docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.ML.GenAI.Core;
+using Microsoft.ML.GenAI.Phi;
+using Tensorboard;
+using static TorchSharp.torch;
+using TorchSharp;
+using Microsoft.ML.GenAI.Core.Extension;
+using System.Text.Json;
+using Microsoft.ML.Tokenizers;
+
+namespace Microsoft.ML.GenAI.Samples.Phi3Mini;
+
+internal static class Utils
+{
+    public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFolder(
+        string weightFolder,
+        string configName = "config.json",
+        string device = "cuda",
+        int modelSizeOnCudaInGB = 16,
+        int modelSizeOnMemoryInGB = 64,
+        int modelSizeOnDiskInGB = 200,
+        bool quantizeToInt8 = false,
+        bool quantizeToInt4 = false)
+    {
+        Console.WriteLine("Loading Phi3 from huggingface model weight folder");
+        torch.set_default_device("meta");
+        var configPath = System.IO.Path.Combine(weightFolder, configName);
+        var config = JsonSerializer.Deserialize<Phi3Config>(System.IO.File.ReadAllText(configPath)) ?? throw new ArgumentNullException(nameof(configPath));
+        var timer = System.Diagnostics.Stopwatch.StartNew();
+        var model = new Phi3ForCasualLM(config);
+        var tokenzierPath = System.IO.Path.Combine(weightFolder, "tokenizer.model");
+        var tokenizer = Phi3TokenizerHelper.FromPretrained(tokenzierPath);
+
+        if (quantizeToInt8)
+        {
+            model.ToInt8QuantizeModule();
+        }
+        else if (quantizeToInt4)
+        {
+            model.ToInt4QuantizeModule();
+        }
+
+        var deviceSizeMap = new Dictionary<string, long>
+        {
+            ["cuda"] = modelSizeOnCudaInGB * 1L * 1024 * 1024 * 1024,
+            ["cpu"] = modelSizeOnMemoryInGB * 1L * 1024 * 1024 * 1024,
+            ["disk"] = modelSizeOnDiskInGB * 1L * 1024 * 1024 * 1024,
+        };
+
+        var deviceMap = model.InferDeviceMapForEachLayer(
+            devices: ["cuda", "cpu", "disk"],
+            deviceSizeMapInByte: deviceSizeMap);
+
+        var deviceMapJson = JsonSerializer.Serialize(deviceMap, new JsonSerializerOptions { WriteIndented = true });
+        Console.WriteLine($"Device map:");
+        Console.WriteLine(deviceMapJson);
+
+        // load weight
+        torch.set_default_device("cpu");
+
+        Console.WriteLine("Start loading");
+        timer = System.Diagnostics.Stopwatch.StartNew();
+        model = new Phi3ForCasualLM(config);
+        timer.Stop();
+        Console.WriteLine($"Phi3 model created in {timer.ElapsedMilliseconds / 1000} s");
+
+        timer = System.Diagnostics.Stopwatch.StartNew();
+        model.LoadSafeTensors(weightFolder);
+        timer.Stop();
+        Console.WriteLine($"Phi3 weight loaded in {timer.ElapsedMilliseconds / 1000} s");
+
+        if (quantizeToInt8 || quantizeToInt4)
+        {
+            timer = System.Diagnostics.Stopwatch.StartNew();
+            Console.WriteLine("Start quantizing if needed");
+            if (quantizeToInt8)
+            {
+                model.ToInt8QuantizeModule();
+            }
+            else if (quantizeToInt4)
+            {
+                model.ToInt4QuantizeModule();
+            }
+            Console.WriteLine("Quantizing done");
+            timer.Stop();
+            Console.WriteLine($"Quantizing done in {timer.ElapsedMilliseconds / 1000} s");
+        }
+
+        timer = System.Diagnostics.Stopwatch.StartNew();
+        Console.WriteLine($"Start loading to device: {device}");
+        model = model.ToDynamicLoadingModel(deviceMap, "cuda");
+        timer.Stop();
+        Console.WriteLine($"Phi3 loaded to device: {device} in {timer.ElapsedMilliseconds / 1000} s");
+        var pipeline = new CausalLMPipeline<Tokenizer, Phi3ForCasualLM>(tokenizer, model, device);
+        torch.set_default_device(device);
+
+        return pipeline;
+    }
+}

docs/samples/Microsoft.ML.GenAI.Samples/Program.cs

@@ -0,0 +1,4 @@
+// See https://aka.ms/new-console-template for more information
+using Microsoft.ML.GenAI.Samples.Phi3Mini;
+
+await SemanticKernelSample.RunChatCompletionSample();

eng/Versions.props

@@ -63,6 +63,9 @@
     <TensorflowDotNETVersion>0.20.1</TensorflowDotNETVersion>
     <TensorFlowMajorVersion>2</TensorFlowMajorVersion>
     <TensorFlowVersion>2.3.1</TensorFlowVersion>
+    <TorchSharpPyBridgeVersion>1.4.1</TorchSharpPyBridgeVersion>
+    <AutoGenVersion>0.0.15</AutoGenVersion>
+    <SemanticKernelVersion>1.15.0</SemanticKernelVersion>
     <TorchSharpVersion>0.102.7</TorchSharpVersion>
     <LibTorchVersion>2.2.1.1</LibTorchVersion>
     <!-- Build/infrastructure Dependencies -->
@@ -75,6 +78,7 @@
     <SystemCompositionVersion>1.2.0</SystemCompositionVersion>
     <!-- Test-only Dependencies -->
     <ApprovalTestsVersion>5.4.7</ApprovalTestsVersion>
+    <MoqVersion>4.20.70</MoqVersion>
     <BenchmarkDotNetVersion>0.13.1</BenchmarkDotNetVersion>
     <DotNetRuntime60Version>6.0.26</DotNetRuntime60Version>
     <DotNetRuntime80Version>8.0.1</DotNetRuntime80Version>