Added a common augmentation interface & default implementation

README.md

@@ -20,7 +20,7 @@ Things we are currently working on:
   - [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb)
   - [ ] App: Implement the continuous process of vectorizing data
   - [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~
-  - [ ] App: Define a common augmentation interface for the integration of RAG processes in chats
+  - [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288))~~
   - [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~
 
 

app/MindWork AI Studio.sln.DotSettings

@@ -7,6 +7,7 @@
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MSG/@EntryIndexedValue">MSG</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RAG/@EntryIndexedValue">RAG</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UI/@EntryIndexedValue">UI</s:String>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=agentic/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=groq/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=ollama/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=tauri_0027s/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

app/MindWork AI Studio/Chat/ChatRole.cs

@@ -12,6 +12,7 @@ public enum ChatRole
     USER,
     AI,
     AGENT,
+    RAG,
 }
 
 /// <summary>

app/MindWork AI Studio/Chat/ContentImage.cs

@@ -7,7 +7,7 @@ namespace AIStudio.Chat;
 /// <summary>
 /// Represents an image inside the chat.
 /// </summary>
-public sealed class ContentImage : IContent
+public sealed class ContentImage : IContent, IImageSource
 {
     #region Implementation of IContent
 
@@ -47,62 +47,4 @@ public Task CreateFromProviderAsync(IProvider provider, Model chatModel, IConten
     /// The image source.
     /// </summary>
     public required string Source { get; set; }
-
-    /// <summary>
-    /// Read the image content as a base64 string.
-    /// </summary>
-    /// <remarks>
-    /// The images are directly converted to base64 strings. The maximum
-    /// size of the image is around 10 MB. If the image is larger, the method
-    /// returns an empty string.
-    ///
-    /// As of now, this method does no sort of image processing. LLMs usually
-    /// do not work with arbitrary image sizes. In the future, we might have
-    /// to resize the images before sending them to the model.
-    /// </remarks>
-    /// <param name="token">The cancellation token.</param>
-    /// <returns>The image content as a base64 string; might be empty.</returns>
-    public async Task<string> AsBase64(CancellationToken token = default)
-    {
-        switch (this.SourceType)
-        {
-            case ContentImageSource.BASE64:
-                return this.Source;
-
-            case ContentImageSource.URL:
-            {
-                using var httpClient = new HttpClient();
-                using var response = await httpClient.GetAsync(this.Source, HttpCompletionOption.ResponseHeadersRead, token);
-                if(response.IsSuccessStatusCode)
-                {
-                    // Read the length of the content:
-                    var lengthBytes = response.Content.Headers.ContentLength;
-                    if(lengthBytes > 10_000_000)
-                        return string.Empty;
-
-                    var bytes = await response.Content.ReadAsByteArrayAsync(token);
-                    return Convert.ToBase64String(bytes);
-                }
-
-                return string.Empty;
-            }
-
-            case ContentImageSource.LOCAL_PATH:
-                if(File.Exists(this.Source))
-                {
-                    // Read the content length:
-                    var length = new FileInfo(this.Source).Length;
-                    if(length > 10_000_000)
-                        return string.Empty;
-
-                    var bytes = await File.ReadAllBytesAsync(this.Source, token);
-                    return Convert.ToBase64String(bytes);
-                }
-
-                return string.Empty;
-
-            default:
-                return string.Empty;
-        }
-    }
 }

app/MindWork AI Studio/Chat/IImageSource.cs

@@ -0,0 +1,17 @@
+namespace AIStudio.Chat;
+
+public interface IImageSource
+{
+    /// <summary>
+    /// The type of the image source.
+    /// </summary>
+    /// <remarks>
+    /// Is the image source a URL, a local file path, a base64 string, etc.?
+    /// </remarks>
+    public ContentImageSource SourceType { get; init; }
+
+    /// <summary>
+    /// The image source.
+    /// </summary>
+    public string Source { get; set; }
+}

app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

@@ -0,0 +1,63 @@
+namespace AIStudio.Chat;
+
+public static class IImageSourceExtensions
+{
+    /// <summary>
+    /// Read the image content as a base64 string.
+    /// </summary>
+    /// <remarks>
+    /// The images are directly converted to base64 strings. The maximum
+    /// size of the image is around 10 MB. If the image is larger, the method
+    /// returns an empty string.
+    /// 
+    /// As of now, this method does no sort of image processing. LLMs usually
+    /// do not work with arbitrary image sizes. In the future, we might have
+    /// to resize the images before sending them to the model.
+    /// </remarks>
+    /// <param name="image">The image source.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The image content as a base64 string; might be empty.</returns>
+    public static async Task<string> AsBase64(this IImageSource image, CancellationToken token = default)
+    {
+        switch (image.SourceType)
+        {
+            case ContentImageSource.BASE64:
+                return image.Source;
+
+            case ContentImageSource.URL:
+            {
+                using var httpClient = new HttpClient();
+                using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, token);
+                if(response.IsSuccessStatusCode)
+                {
+                    // Read the length of the content:
+                    var lengthBytes = response.Content.Headers.ContentLength;
+                    if(lengthBytes > 10_000_000)
+                        return string.Empty;
+
+                    var bytes = await response.Content.ReadAsByteArrayAsync(token);
+                    return Convert.ToBase64String(bytes);
+                }
+
+                return string.Empty;
+            }
+
+            case ContentImageSource.LOCAL_PATH:
+                if(File.Exists(image.Source))
+                {
+                    // Read the content length:
+                    var length = new FileInfo(image.Source).Length;
+                    if(length > 10_000_000)
+                        return string.Empty;
+
+                    var bytes = await File.ReadAllBytesAsync(image.Source, token);
+                    return Convert.ToBase64String(bytes);
+                }
+
+                return string.Empty;
+
+            default:
+                return string.Empty;
+        }
+    }
+}

app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

@@ -37,6 +37,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs

@@ -49,6 +49,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs

@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs

@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs

@@ -48,6 +48,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -76,6 +76,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
                     ChatRole.USER => "user",
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
+                    ChatRole.RAG => "assistant",
                     ChatRole.SYSTEM => systemPromptRole,
 
                     _ => "user",

app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs

@@ -46,6 +46,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Provider/X/ProviderX.cs

@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
                     ChatRole.AI => "assistant",
                     ChatRole.AGENT => "assistant",
                     ChatRole.SYSTEM => "system",
+                    ChatRole.RAG => "assistant",
 
                     _ => "user",
                 },

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -0,0 +1,119 @@
+using System.Text;
+
+using AIStudio.Chat;
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.RAG.AugmentationProcesses;
+
+public sealed class AugmentationOne : IAugmentationProcess
+{
+    #region Implementation of IAugmentationProcess
+
+    /// <inheritdoc />
+    public string TechnicalName => "AugmentationOne";
+
+    /// <inheritdoc />
+    public string UIName => "Standard augmentation process";
+
+    /// <inheritdoc />
+    public string Description => "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.";
+
+    /// <inheritdoc />
+    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
+    {
+        var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AugmentationOne>>()!;
+        if(retrievalContexts.Count == 0)
+        {
+            logger.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
+            return chatThread;
+        }
+
+        var numTotalRetrievalContexts = retrievalContexts.Count;
+        logger.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
+
+        //
+        // We build a huge prompt from all retrieval contexts:
+        //
+        var sb = new StringBuilder();
+        sb.AppendLine("The following useful information will help you in processing the user prompt:");
+        sb.AppendLine();
+
+        var index = 0;
+        foreach(var retrievalContext in retrievalContexts)
+        {
+            index++;
+            sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
+            sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
+            sb.AppendLine($"Content category: {retrievalContext.Category}");
+            sb.AppendLine($"Content type: {retrievalContext.Type}");
+            sb.AppendLine($"Content path: {retrievalContext.Path}");
+
+            if(retrievalContext.Links.Count > 0)
+            {
+                sb.AppendLine("Additional links:");
+                foreach(var link in retrievalContext.Links)
+                    sb.AppendLine($"- {link}");
+            }
+
+            switch(retrievalContext)
+            {
+                case RetrievalTextContext textContext:
+                    sb.AppendLine();
+                    sb.AppendLine("Matched text content:");
+                    sb.AppendLine("````");
+                    sb.AppendLine(textContext.MatchedText);
+                    sb.AppendLine("````");
+
+                    if(textContext.SurroundingContent.Count > 0)
+                    {
+                        sb.AppendLine();
+                        sb.AppendLine("Surrounding text content:");
+                        foreach(var surrounding in textContext.SurroundingContent)
+                        {
+                            sb.AppendLine();
+                            sb.AppendLine("````");
+                            sb.AppendLine(surrounding);
+                            sb.AppendLine("````");
+                        }
+                    }
+
+
+                    break;
+
+                case RetrievalImageContext imageContext:
+                    sb.AppendLine();
+                    sb.AppendLine("Matched image content as base64-encoded data:");
+                    sb.AppendLine("````");
+                    sb.AppendLine(await imageContext.AsBase64(token));
+                    sb.AppendLine("````");
+                    break;
+
+                default:
+                    logger.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
+                    break;
+            }
+
+            sb.AppendLine();
+        }
+
+        //
+        // Append the entire augmentation to the chat thread,
+        // just before the user prompt:
+        //
+        chatThread.Blocks.Insert(chatThread.Blocks.Count - 1, new()
+        {
+            Role = ChatRole.RAG,
+            Time = DateTimeOffset.UtcNow,
+            ContentType = ContentType.TEXT,
+            HideFromUser = true,
+            Content = new ContentText
+            {
+                Text = sb.ToString(),
+            }
+        });
+
+        return chatThread;
+    }
+
+    #endregion
+}