Skip to content

Commit 3adcc07

Browse files
gunpal5gunpal5
authored
Merge pull request #73 from ericstj/IImageGeneratorSupport
Add image support to GenerativeAI.Microsoft
2 parents b6137d1 + 235ec42 commit 3adcc07

File tree

5 files changed

+898
-0
lines changed

5 files changed

+898
-0
lines changed

‎src/GenerativeAI.Microsoft/GenerativeAIImageGenerator.cs‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#pragma warning disable MEAI001
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Drawing;
5+
using System.Linq;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using GenerativeAI;
9+
using GenerativeAI.Core;
10+
using GenerativeAI.Types;
11+
using GenerativeAI.Microsoft.Extensions;
12+
using Microsoft.Extensions.AI;
13+
14+
namespace GenerativeAI.Microsoft;
15+
16+
/// <summary>
17+
/// Implements Microsoft.Extensions.AI.IImageGenerator using the Google_GenerativeAI SDK by
18+
/// creating a GenerateContentRequest that requests image modality and forwarding it to
19+
/// <see cref="GenerativeModel.GenerateContentAsync(GenerateContentRequest, CancellationToken)"/>.
20+
/// </summary>
21+
public sealed class GenerativeAIImageGenerator : IImageGenerator
22+
{
23+
/// <summary>
24+
/// Underlying GenerativeModel instance.
25+
/// </summary>
26+
public GenerativeModel model { get; }
27+
28+
/// <summary>
29+
/// Creates a new instance using an API key and optional model name.
30+
/// </summary>
31+
public GenerativeAIImageGenerator(string apiKey, string modelName = GoogleAIModels.Gemini2FlashPreviewImageGeneration)
32+
{
33+
model = new GenerativeModel(apiKey, modelName);
34+
}
35+
36+
/// <summary>
37+
/// Creates a new instance using a platform adapter and optional model name.
38+
/// </summary>
39+
public GenerativeAIImageGenerator(IPlatformAdapter adapter, string modelName = GoogleAIModels.Gemini2FlashPreviewImageGeneration)
40+
{
41+
model = new GenerativeModel(adapter, modelName);
42+
}
43+
44+
/// <inheritdoc/>
45+
public void Dispose()
46+
{
47+
}
48+
49+
/// <inheritdoc/>
50+
public object? GetService(Type serviceType, object? serviceKey = null)
51+
{
52+
if (serviceKey == null && serviceType?.IsInstanceOfType(this) == true)
53+
return this;
54+
return null;
55+
}
56+
57+
/// <inheritdoc/>
58+
public async Task<ImageGenerationResponse> GenerateAsync(ImageGenerationRequest request,
59+
ImageGenerationOptions? options = null, CancellationToken cancellationToken = default)
60+
{
61+
#if NET6_0_OR_GREATER
62+
ArgumentNullException.ThrowIfNull(request);
63+
#else
64+
if (request == null) throw new ArgumentNullException(nameof(request));
65+
#endif
66+
67+
var genRequest = ToGenerateContentRequest(request, options);
68+
var resp = await model.GenerateContentAsync(genRequest, cancellationToken).ConfigureAwait(false);
69+
return ToImageGenerationResponse(resp);
70+
}
71+
72+
// Convert the Microsoft request/options into a model-specific GenerateContentRequest
73+
private GenerateContentRequest ToGenerateContentRequest(ImageGenerationRequest request, ImageGenerationOptions? options)
74+
{
75+
List<Part> parts = [];
76+
// Add prompt text (if any)
77+
if (!string.IsNullOrEmpty(request.Prompt))
78+
{
79+
parts.Add(new(request.Prompt!));
80+
}
81+
82+
// If original images provided (image edit scenario), add them as parts
83+
if (request.OriginalImages != null)
84+
{
85+
foreach (var aiContent in request.OriginalImages)
86+
{
87+
parts.Add(aiContent.ToPart()!);
88+
}
89+
}
90+
91+
// Configure generation to request images
92+
GenerationConfig generationConfig = options?.RawRepresentationFactory?.Invoke(this) as GenerationConfig ?? new();
93+
generationConfig.CandidateCount = options?.Count ?? 1;
94+
95+
// We must request both text and image modalities to get images back
96+
generationConfig.ResponseModalities = new List<Modality> { Modality.TEXT, Modality.IMAGE };
97+
98+
if (options != null)
99+
{
100+
if (!string.IsNullOrEmpty(options.MediaType))
101+
generationConfig.ResponseMimeType = options.MediaType;
102+
103+
// Map requested image size (basic heuristic)
104+
if (options.ImageSize.HasValue)
105+
{
106+
var sz = options.ImageSize.Value;
107+
if (sz.Width >= 1024 || sz.Height >= 1024)
108+
generationConfig.MediaResolution = MediaResolution.MEDIA_RESOLUTION_HIGH;
109+
else if (sz.Width >= 512 || sz.Height >= 512)
110+
generationConfig.MediaResolution = MediaResolution.MEDIA_RESOLUTION_MEDIUM;
111+
else
112+
generationConfig.MediaResolution = MediaResolution.MEDIA_RESOLUTION_LOW;
113+
}
114+
}
115+
116+
return new GenerateContentRequest()
117+
{
118+
GenerationConfig = generationConfig,
119+
Contents = [new() { Parts = parts }]
120+
};
121+
}
122+
123+
124+
// Convert the model response to ImageGenerationResponse
125+
private static ImageGenerationResponse ToImageGenerationResponse(GenerateContentResponse? resp)
126+
{
127+
var aiContents = resp?.Candidates?.FirstOrDefault()?.Content?.Parts.ToAiContents();
128+
return new ImageGenerationResponse(aiContents) { RawRepresentation = resp };
129+
}
130+
}
131+
#pragma warning restore MEAI001

‎src/GenerativeAI.Microsoft/GenerativeAIImagenGenerator.cs‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
#pragma warning disable MEAI001
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Drawing;
5+
using System.Linq;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using GenerativeAI;
9+
using GenerativeAI.Core;
10+
using GenerativeAI.Types;
11+
using GenerativeAI.Clients;
12+
using GenerativeAI.Microsoft.Extensions;
13+
using Microsoft.Extensions.AI;
14+
15+
namespace GenerativeAI.Microsoft;
16+
17+
/// <summary>
18+
/// Implements Microsoft.Extensions.AI.IImageGenerator by creating an ImagenModel via GenAI.CreateImageModel
19+
/// and calling <see cref="GenerativeAI.Clients.ImagenModel.GenerateImagesAsync(GenerateImageRequest, CancellationToken)"/>.
20+
/// </summary>
21+
public sealed class GenerativeAIImagenGenerator : IImageGenerator
22+
{
23+
/// <summary>
24+
/// Underlying ImagenModel instance created from the provided GenAI factory.
25+
/// </summary>
26+
public ImagenModel model { get; }
27+
28+
/// <summary>
29+
/// Creates a new instance using an API key and optional model name.
30+
/// </summary>
31+
public GenerativeAIImagenGenerator(string apiKey, string modelName = GoogleAIModels.Imagen.Imagen3Generate002):
32+
this(new GoogleAi(apiKey), modelName)
33+
{ }
34+
35+
36+
/// <summary>
37+
/// Creates a new instance using an existing <see cref="GenAI"/> factory and optional model name.
38+
/// </summary>
39+
public GenerativeAIImagenGenerator(GenAI genai, string modelName = GoogleAIModels.Imagen.Imagen3Generate002)
40+
{
41+
#if NET6_0_OR_GREATER
42+
ArgumentNullException.ThrowIfNull(genai);
43+
#else
44+
if (genai == null) throw new ArgumentNullException(nameof(genai));
45+
#endif
46+
model = genai.CreateImageModel(modelName);
47+
}
48+
49+
/// <inheritdoc/>
50+
public void Dispose()
51+
{ }
52+
53+
/// <inheritdoc/>
54+
public object? GetService(Type serviceType, object? serviceKey = null)
55+
{
56+
if (serviceKey == null && serviceType?.IsInstanceOfType(this) == true)
57+
return this;
58+
return null;
59+
}
60+
61+
/// <inheritdoc/>
62+
public async Task<ImageGenerationResponse> GenerateAsync(ImageGenerationRequest request,
63+
ImageGenerationOptions? options = null, CancellationToken cancellationToken = default)
64+
{
65+
#if NET6_0_OR_GREATER
66+
ArgumentNullException.ThrowIfNull(request);
67+
#else
68+
if (request == null) throw new ArgumentNullException(nameof(request));
69+
#endif
70+
71+
var imgRequest = ToGenerateImageRequest(request, options);
72+
var resp = await model.GenerateImagesAsync(imgRequest, cancellationToken).ConfigureAwait(false);
73+
return ToImageGenerationResponse(resp);
74+
}
75+
76+
// Convert Microsoft ImageGenerationRequest + options to a GenerateImageRequest
77+
private GenerateImageRequest ToGenerateImageRequest(ImageGenerationRequest request, ImageGenerationOptions? options)
78+
{
79+
var imgRequest = new GenerateImageRequest();
80+
var instances = new List<ImageGenerationInstance>();
81+
82+
if (request.OriginalImages != null && request.OriginalImages.Any())
83+
{
84+
instances.AddRange(request.OriginalImages.Select(content => new ImageGenerationInstance
85+
{
86+
Prompt = request.Prompt,
87+
Image = ConvertAiContentToImageSource(content)
88+
}));
89+
}
90+
else
91+
{
92+
instances.Add(new ImageGenerationInstance { Prompt = request.Prompt });
93+
}
94+
95+
ImageGenerationParameters parameters = options?.RawRepresentationFactory?.Invoke(this) as ImageGenerationParameters ?? new();
96+
parameters.SampleCount = options?.Count ?? 1;
97+
98+
if (options != null)
99+
{
100+
if (!string.IsNullOrEmpty(options.MediaType))
101+
{
102+
parameters.OutputOptions = new OutputOptions { MimeType = options.MediaType };
103+
}
104+
105+
if (options.ImageSize.HasValue)
106+
{
107+
var sz = options.ImageSize.Value;
108+
parameters.AspectRatio = $"{sz.Width}:{sz.Height}";
109+
110+
}
111+
}
112+
113+
return new GenerateImageRequest
114+
{
115+
Instances = instances,
116+
Parameters = parameters
117+
};
118+
}
119+
120+
// Convert model response to Microsoft ImageGenerationResponse
121+
private static ImageGenerationResponse ToImageGenerationResponse(GenerateImageResponse? resp)
122+
{
123+
var contents = new List<AIContent>();
124+
if (resp?.Predictions != null)
125+
{
126+
foreach (var pred in resp.Predictions)
127+
{
128+
if (!string.IsNullOrEmpty(pred.BytesBase64Encoded))
129+
{
130+
var data = Convert.FromBase64String(pred.BytesBase64Encoded);
131+
contents.Add(new DataContent(data, pred.MimeType ?? "image/png"));
132+
}
133+
}
134+
}
135+
136+
return new ImageGenerationResponse(contents) { RawRepresentation = resp };
137+
}
138+
139+
private static ImageSource? ConvertAiContentToImageSource(AIContent content)
140+
{
141+
if (content == null) return null;
142+
143+
if (content is DataContent dc)
144+
{
145+
return new ImageSource { BytesBase64Encoded = Convert.ToBase64String(dc.Data.ToArray()) };
146+
}
147+
148+
if (content is UriContent uc)
149+
{
150+
var uriVal = uc.Uri?.ToString();
151+
152+
// Only treat known GCS URIs as storage references for Imagen API.
153+
if (uriVal?.StartsWith("gs://", StringComparison.OrdinalIgnoreCase) == true ||
154+
uriVal?.IndexOf("storage.googleapis.com", StringComparison.OrdinalIgnoreCase) >= 0)
155+
{
156+
return new ImageSource { GcsUri = uriVal };
157+
}
158+
}
159+
160+
return null;
161+
}
162+
}
163+
#pragma warning restore MEAI001

‎src/GenerativeAI.Microsoft/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,66 @@ public class MyChatService
8585
}
8686
```
8787

88+
### 4. Using IImageClient
89+
90+
Image client can also be used from a service as above. Here's a sample that shows it's capabilities.
91+
92+
```C#
93+
using System.Diagnostics;
94+
using GenerativeAI.Microsoft;
95+
using Microsoft.Extensions.AI;
96+
97+
#pragma warning disable MEAI001
98+
// ImageGen creates high quality initial images
99+
IImageGenerator imageGenerator = new GenerativeAIImagenGenerator(
100+
Environment.GetEnvironmentVariable("GOOGLE_API_KEY"),
101+
"imagen-4.0-fast-generate-001");
102+
103+
var response = await imageGenerator.GenerateImagesAsync("A clown fish with orange and black-bordered white stripes.");
104+
var img1 = GetImageContent(response);
105+
SaveImage(img1, "i1.png");
106+
ShowImage("i1.png");
107+
108+
response = await imageGenerator.GenerateImagesAsync("A blue tang fish, blue and black with yellow tipped fin and tail.");
109+
var img2 = GetImageContent(response);
110+
SaveImage(img2, "i2.png");
111+
ShowImage("i2.png");
112+
113+
// Imagen cannot edit, but we can use the gemini model for that.
114+
IImageGenerator imageGeneratorEdit = new GenerativeAIImageGenerator(
115+
Environment.GetEnvironmentVariable("GOOGLE_API_KEY"),
116+
"gemini-2.5-flash-image-preview");
117+
var request = new ImageGenerationRequest()
118+
{
119+
Prompt = "Combine the two images into a single scene.",
120+
OriginalImages = new[] { img1, img2 }
121+
};
122+
response = await imageGeneratorEdit.GenerateAsync(request);
123+
var scene = GetImageContent(response);
124+
SaveImage(scene, "scene.png");
125+
ShowImage("scene.png");
126+
127+
response = await imageGeneratorEdit.EditImageAsync(scene, "Change the setting to a fish tank.");
128+
var edit = GetImageContent(response);
129+
SaveImage(edit, "edit.png");
130+
ShowImage("edit.png");
131+
132+
DataContent GetImageContent(ImageGenerationResponse response) =>
133+
response.Contents.OfType<DataContent>().Single();
134+
135+
void SaveImage(DataContent content, string fileName) =>
136+
File.WriteAllBytes(fileName, content.Data.Span);
137+
138+
void ShowImage(string fileName)
139+
{
140+
Process.Start(new ProcessStartInfo
141+
{
142+
FileName = fileName,
143+
UseShellExecute = true
144+
});
145+
}
146+
```
147+
88148
## Dependencies
89149

90150
- [Google_GenerativeAI](https://github.com/Google_GenerativeAI) (Unofficial C# Google Generative AI SDK)

0 commit comments

Comments
 (0)