audio media support

Author: jamesmoore

ChatGPTExport/Assets/AssetLocator.cs

@@ -12,10 +12,10 @@ ExistingAssetLocator existingAssetLocator
     {
         private List<string>? cachedSourceList = null;
 
-        public string? GetMarkdownImage(AssetRequest assetRequest)
+        public string? GetMarkdownMediaAsset(AssetRequest assetRequest)
         {
             return FindAssetInSourceDirectory(assetRequest) ??
-                existingAssetLocator.GetMarkdownImage(assetRequest);
+                existingAssetLocator.GetMarkdownMediaAsset(assetRequest);
         }
 
         private IEnumerable<string> GetCachedSourceFiles(string searchPattern)

ChatGPTExport/Assets/ExistingAssetLocator.cs

@@ -23,7 +23,7 @@ public void Add(string newFile)
             GetCache().Add(newFile);
         }
 
-        public string? GetMarkdownImage(AssetRequest assetRequest)
+        public string? GetMarkdownMediaAsset(AssetRequest assetRequest)
         {
             // it may already exist in the destination directory from a previous export 
             var destinationMatches = GetCachedDestinationFiles(assetRequest.SearchPattern).ToList();

ChatGPTExport/Assets/IAssetLocator.cs

@@ -3,6 +3,6 @@ namespace ChatGPTExport.Assets
 {
     public interface IAssetLocator
     {
-        string? GetMarkdownImage(AssetRequest assetRequest);
+        string? GetMarkdownMediaAsset(AssetRequest assetRequest);
     }
 }

ChatGPTExport/Exporters/Html/HtmlExporter.cs

@@ -107,7 +107,7 @@ private MarkdownPipeline GetPipeline()
                 .UseFootnotes()
                 //.UseGridTables()
                 .UseMathematics()
-                //.UseMediaLinks()
+                .UseMediaLinks()
                 .UsePipeTables()
                 .UseListExtras()
                 .UseTaskLists()

ChatGPTExport/Exporters/Markdown/MarkdownContentVisitor.cs

@@ -147,40 +147,75 @@ public MarkdownContentResult Visit(ContentMultimodalText content, ContentVisitor
             {
                 if (part.IsObject)
                 {
-                    var obj = part.ObjectValue;
-                    if (obj.content_type == "image_asset_pointer" && string.IsNullOrWhiteSpace(obj.asset_pointer) == false)
+                    markdownContent.AddRange(GetMarkdownMediaAsset(context, part.ObjectValue));
+                }
+                else if (part.IsString)
+                {
+                    markdownContent.Add(part.StringValue);
+                }
+            }
+            return new MarkdownContentResult(markdownContent);
+        }
+
+        private IEnumerable<string> GetMarkdownMediaAsset(ContentVisitorContext context, ContentMultimodalText.ContentMultimodalTextParts obj)
+        {
+            switch (obj.content_type)
+            {
+                case "image_asset_pointer" when string.IsNullOrWhiteSpace(obj.asset_pointer) == false:
                     {
                         var searchPattern = obj.asset_pointer.Replace("sediment://", string.Empty).Replace("file-service://", string.Empty);
-                        var markdownImage = assetLocator.GetMarkdownImage(new AssetRequest(
-                            searchPattern,
-                            context.Role,
-                            context.CreatedDate,
-                            context.UpdatedDate)
-                            );
+                        var markdownImage = GetMediaAsset(context, searchPattern);
 
                         if (string.IsNullOrWhiteSpace(markdownImage) == false)
                         {
-                            markdownContent.Add(markdownImage);
+                            yield return markdownImage;
                         }
                         else
                         {
                             Console.Error.WriteLine("\tUnable to find asset " + obj.asset_pointer);
-                            markdownContent.Add($"> ⚠️ **Warning:** Could not find asset: {obj.asset_pointer}.");
+                            yield return $"> ⚠️ **Warning:** Could not find asset: {obj.asset_pointer}.";
                         }
 
                         if (context.MessageMetadata.image_gen_title != null)
                         {
-                            markdownContent.Add($"*{context.MessageMetadata.image_gen_title}*  ");
+                            yield return $"*{context.MessageMetadata.image_gen_title}*  ";
                         }
-                        markdownContent.Add($"**Size:** {obj.size_bytes} **Dims:** {obj.width}x{obj.height}  ");
+                        yield return $"**Size:** {obj.size_bytes} **Dims:** {obj.width}x{obj.height}  ";
+                        break;
                     }
-                }
-                else if (part.IsString)
-                {
-                    markdownContent.Add(part.StringValue);
-                }
+
+                case "real_time_user_audio_video_asset_pointer" when string.IsNullOrWhiteSpace(obj.audio_asset_pointer?.asset_pointer) == false:
+                    {
+                        var searchPattern = obj.audio_asset_pointer.asset_pointer.Replace("sediment://", string.Empty).Replace("file-service://", string.Empty);
+                        var markdownAsset = GetMediaAsset(context, searchPattern);
+
+                        yield return $"{markdownAsset}  ";
+                        break;
+                    }
+
+                case "audio_asset_pointer" when string.IsNullOrWhiteSpace(obj.asset_pointer) == false:
+                    {
+                        var searchPattern = obj.asset_pointer.Replace("sediment://", string.Empty).Replace("file-service://", string.Empty);
+                        var markdownAsset = GetMediaAsset(context, searchPattern);
+
+                        yield return $"{markdownAsset}  ";
+                        break;
+                    }
+
+                case "audio_transcription" when string.IsNullOrWhiteSpace(obj.text) == false:
+                    yield return $"*{obj.text}*  ";
+                    break;
             }
-            return new MarkdownContentResult(markdownContent);
+        }
+
+        private string? GetMediaAsset(ContentVisitorContext context, string searchPattern)
+        {
+            return assetLocator.GetMarkdownMediaAsset(new AssetRequest(
+                searchPattern,
+                context.Role,
+                context.CreatedDate,
+                context.UpdatedDate)
+                );
         }
 
         public MarkdownContentResult Visit(ContentCode content, ContentVisitorContext context)

ChatGPTExport/Models/Conversations.cs

@@ -406,6 +406,8 @@ public class ContentMultimodalTextParts
             public int? height { get; set; }
             public object fovea { get; set; }
             public Metadata metadata { get; set; }
+            public AudioAssetPointer audio_asset_pointer { get; set; }
+            public string text { get; set; } // for audio transcription
 
             public class Metadata
             {
@@ -447,6 +449,14 @@ public class Generation
 
             [JsonExtensionData]
             public Dictionary<string, JsonElement> ExtraData { get; set; }
+
+            public class AudioAssetPointer
+            {
+                public string asset_pointer { get; set; }
+
+                [JsonExtensionData]
+                public Dictionary<string, JsonElement> ExtraData { get; set; }
+            }
         }
 
         public override T Accept<T>(IContentVisitor<T> visitor, ContentVisitorContext context)