Skip to content

Commit a71bd1f

Browse files
davidmotsondavidmotson
authored
Merge e710498 into 8b33282 
2 parents 8b33282 + e710498 commit a71bd1f

File tree

7 files changed

+268
-9
lines changed

7 files changed

+268
-9
lines changed

‎firebase-ai/api.txt

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ package com.google.firebase.ai {
6565
}
6666

6767
@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenModel {
68+
method public suspend Object? editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
6869
method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
6970
}
7071

@@ -104,6 +105,7 @@ package com.google.firebase.ai.java {
104105
}
105106

106107
@com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenModelFutures {
108+
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>> editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config);
107109
method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model);
108110
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>> generateImages(String prompt);
109111
method public abstract com.google.firebase.ai.ImagenModel getImageModel();
@@ -484,6 +486,47 @@ package com.google.firebase.ai.type {
484486
public static final class ImagenAspectRatio.Companion {
485487
}
486488

489+
public final class ImagenEditMode {
490+
field public static final com.google.firebase.ai.type.ImagenEditMode.Companion Companion;
491+
}
492+
493+
public static final class ImagenEditMode.Companion {
494+
method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_INSERTION();
495+
method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_REMOVAL();
496+
method public com.google.firebase.ai.type.ImagenEditMode getOUTPAINT();
497+
property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_INSERTION;
498+
property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_REMOVAL;
499+
property public final com.google.firebase.ai.type.ImagenEditMode OUTPAINT;
500+
}
501+
502+
@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenEditingConfig {
503+
ctor public ImagenEditingConfig(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenEditMode editMode, com.google.firebase.ai.type.ImagenInlineImage? mask = null, Double? maskDilation = null, Integer? editSteps = null);
504+
field public static final com.google.firebase.ai.type.ImagenEditingConfig.Companion Companion;
505+
}
506+
507+
public static final class ImagenEditingConfig.Builder {
508+
ctor public ImagenEditingConfig.Builder();
509+
method public com.google.firebase.ai.type.ImagenEditingConfig build();
510+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditMode(com.google.firebase.ai.type.ImagenEditMode editMode);
511+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditSteps(int editSteps);
512+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setImage(com.google.firebase.ai.type.ImagenInlineImage image);
513+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMask(com.google.firebase.ai.type.ImagenInlineImage mask);
514+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMaskDilation(double maskDilation);
515+
field public com.google.firebase.ai.type.ImagenEditMode? editMode;
516+
field public Integer? editSteps;
517+
field public com.google.firebase.ai.type.ImagenInlineImage? image;
518+
field public com.google.firebase.ai.type.ImagenInlineImage? mask;
519+
field public Double? maskDilation;
520+
}
521+
522+
public static final class ImagenEditingConfig.Companion {
523+
method public com.google.firebase.ai.type.ImagenEditingConfig.Builder builder();
524+
}
525+
526+
public final class ImagenEditingConfigKt {
527+
method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenEditingConfig imagenEditingConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.ImagenEditingConfig.Builder,kotlin.Unit> init);
528+
}
529+
487530
@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
488531
ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.ai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.ai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
489532
method public Boolean? getAddWatermark();
@@ -552,6 +595,10 @@ package com.google.firebase.ai.type {
552595
property public final String mimeType;
553596
}
554597

598+
public final class ImagenInlineImageKt {
599+
method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenInlineImage toImagenInlineImage(android.graphics.Bitmap);
600+
}
601+
555602
@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
556603
field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ADULT;
557604
field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ALL;

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import com.google.firebase.ai.common.AppCheckHeaderProvider
2222
import com.google.firebase.ai.common.ContentBlockedException
2323
import com.google.firebase.ai.common.GenerateImageRequest
2424
import com.google.firebase.ai.type.FirebaseAIException
25+
import com.google.firebase.ai.type.ImagenEditingConfig
2526
import com.google.firebase.ai.type.ImagenGenerationConfig
2627
import com.google.firebase.ai.type.ImagenGenerationResponse
2728
import com.google.firebase.ai.type.ImagenInlineImage
@@ -75,30 +76,92 @@ internal constructor(
7576
public suspend fun generateImages(prompt: String): ImagenGenerationResponse<ImagenInlineImage> =
7677
try {
7778
controller
78-
.generateImage(constructRequest(prompt, null, generationConfig))
79+
.generateImage(constructGenerateImageRequest(prompt, generationConfig))
7980
.validate()
8081
.toPublicInline()
8182
} catch (e: Throwable) {
8283
throw FirebaseAIException.from(e)
8384
}
8485

85-
private fun constructRequest(
86+
public suspend fun editImage(
8687
prompt: String,
87-
gcsUri: String?,
88-
config: ImagenGenerationConfig?,
88+
config: ImagenEditingConfig
89+
): ImagenGenerationResponse<ImagenInlineImage> =
90+
try {
91+
controller.generateImage(constructEditRequest(prompt, config)).validate().toPublicInline()
92+
} catch (e: Throwable) {
93+
throw FirebaseAIException.from(e)
94+
}
95+
96+
private fun constructGenerateImageRequest(
97+
prompt: String,
98+
generationConfig: ImagenGenerationConfig? = null,
8999
): GenerateImageRequest {
90100
return GenerateImageRequest(
91101
listOf(GenerateImageRequest.ImagenPrompt(prompt)),
92102
GenerateImageRequest.ImagenParameters(
93-
sampleCount = config?.numberOfImages ?: 1,
103+
sampleCount = generationConfig?.numberOfImages ?: 1,
104+
includeRaiReason = true,
105+
addWatermark = generationConfig?.addWatermark,
106+
personGeneration = safetySettings?.personFilterLevel?.internalVal,
107+
negativePrompt = generationConfig?.negativePrompt,
108+
safetySetting = safetySettings?.safetyFilterLevel?.internalVal,
109+
storageUri = null,
110+
aspectRatio = generationConfig?.aspectRatio?.internalVal,
111+
imageOutputOptions = generationConfig?.imageFormat?.toInternal(),
112+
editMode = null,
113+
editConfig = null
114+
),
115+
)
116+
}
117+
118+
private fun constructEditRequest(
119+
prompt: String,
120+
editConfig: ImagenEditingConfig,
121+
): GenerateImageRequest {
122+
return GenerateImageRequest(
123+
listOf(
124+
GenerateImageRequest.ImagenPrompt(
125+
prompt = prompt,
126+
referenceImages =
127+
buildList {
128+
add(
129+
GenerateImageRequest.ReferenceImage(
130+
referenceType = GenerateImageRequest.ReferenceType.RAW,
131+
referenceId = 1,
132+
referenceImage = editConfig.image.toInternal(),
133+
maskImageConfig = null
134+
)
135+
)
136+
if (editConfig.mask != null) {
137+
add(
138+
GenerateImageRequest.ReferenceImage(
139+
referenceType = GenerateImageRequest.ReferenceType.MASK,
140+
referenceId = 2,
141+
referenceImage = editConfig.mask.toInternal(),
142+
maskImageConfig =
143+
GenerateImageRequest.MaskImageConfig(
144+
maskMode = GenerateImageRequest.MaskMode.USER_PROVIDED,
145+
dilation = editConfig.maskDilation
146+
)
147+
)
148+
)
149+
}
150+
}
151+
)
152+
),
153+
GenerateImageRequest.ImagenParameters(
154+
sampleCount = generationConfig?.numberOfImages ?: 1,
94155
includeRaiReason = true,
95156
addWatermark = generationConfig?.addWatermark,
96157
personGeneration = safetySettings?.personFilterLevel?.internalVal,
97-
negativePrompt = config?.negativePrompt,
158+
negativePrompt = generationConfig?.negativePrompt,
98159
safetySetting = safetySettings?.safetyFilterLevel?.internalVal,
99-
storageUri = gcsUri,
100-
aspectRatio = config?.aspectRatio?.internalVal,
160+
storageUri = null,
161+
aspectRatio = generationConfig?.aspectRatio?.internalVal,
101162
imageOutputOptions = generationConfig?.imageFormat?.toInternal(),
163+
editMode = editConfig.editMode.value,
164+
editConfig = editConfig.toInternal()
102165
),
103166
)
104167
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import com.google.firebase.ai.common.util.fullModelName
2121
import com.google.firebase.ai.common.util.trimmedModelName
2222
import com.google.firebase.ai.type.Content
2323
import com.google.firebase.ai.type.GenerationConfig
24+
import com.google.firebase.ai.type.ImagenEditingConfig
2425
import com.google.firebase.ai.type.ImagenImageFormat
26+
import com.google.firebase.ai.type.ImagenInlineImage
2527
import com.google.firebase.ai.type.PublicPreviewAPI
2628
import com.google.firebase.ai.type.SafetySetting
2729
import com.google.firebase.ai.type.Tool
@@ -75,11 +77,17 @@ internal data class CountTokensRequest(
7577
}
7678

7779
@Serializable
80+
@PublicPreviewAPI
7881
internal data class GenerateImageRequest(
7982
val instances: List<ImagenPrompt>,
8083
val parameters: ImagenParameters,
8184
) : Request {
82-
@Serializable internal data class ImagenPrompt(val prompt: String)
85+
@Serializable
86+
internal data class ImagenPrompt(
87+
val prompt: String? = null,
88+
val image: ImagenInlineImage.Internal? = null,
89+
val referenceImages: List<ReferenceImage>? = null
90+
)
8391

8492
@OptIn(PublicPreviewAPI::class)
8593
@Serializable
@@ -93,5 +101,38 @@ internal data class GenerateImageRequest(
93101
val personGeneration: String?,
94102
val addWatermark: Boolean?,
95103
val imageOutputOptions: ImagenImageFormat.Internal?,
104+
val editMode: String?,
105+
val editConfig: ImagenEditingConfig.Internal?,
106+
)
107+
108+
@Serializable
109+
internal enum class ReferenceType {
110+
@SerialName("REFERENCE_TYPE_UNSPECIFIED") UNSPECIFIED,
111+
@SerialName("REFERENCE_TYPE_RAW") RAW,
112+
@SerialName("REFERENCE_TYPE_MASK") MASK,
113+
@SerialName("REFERENCE_TYPE_CONTROL") CONTROL,
114+
@SerialName("REFERENCE_TYPE_STYLE") STYLE,
115+
@SerialName("REFERENCE_TYPE_SUBJECT") SUBJECT,
116+
@SerialName("REFERENCE_TYPE_MASKED_SUBJECT") MASKED_SUBJECT,
117+
@SerialName("REFERENCE_TYPE_PRODUCT") PRODUCT
118+
}
119+
120+
@Serializable
121+
internal enum class MaskMode {
122+
@SerialName("MASK_MODE_DEFAULT") DEFAULT,
123+
@SerialName("MASK_MODE_USER_PROVIDED") USER_PROVIDED,
124+
@SerialName("MASK_MODE_BACKGROUND") BACKGROUND,
125+
@SerialName("MASK_MODE_FOREGROUND") FOREGROUND,
126+
@SerialName("MASK_MODE_SEMANTIC") SEMANTIC
127+
}
128+
129+
@Serializable internal data class MaskImageConfig(val maskMode: MaskMode, val dilation: Double?)
130+
131+
@Serializable
132+
internal data class ReferenceImage(
133+
val referenceType: ReferenceType,
134+
val referenceId: Int,
135+
val referenceImage: ImagenInlineImage.Internal,
136+
val maskImageConfig: MaskImageConfig?
96137
)
97138
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package com.google.firebase.ai.java
1919
import androidx.concurrent.futures.SuspendToFutureAdapter
2020
import com.google.common.util.concurrent.ListenableFuture
2121
import com.google.firebase.ai.ImagenModel
22+
import com.google.firebase.ai.type.ImagenEditingConfig
2223
import com.google.firebase.ai.type.ImagenGenerationResponse
2324
import com.google.firebase.ai.type.ImagenInlineImage
2425
import com.google.firebase.ai.type.PublicPreviewAPI
@@ -39,6 +40,11 @@ public abstract class ImagenModelFutures internal constructor() {
3940
prompt: String,
4041
): ListenableFuture<ImagenGenerationResponse<ImagenInlineImage>>
4142

43+
public abstract fun editImage(
44+
prompt: String,
45+
config: ImagenEditingConfig
46+
): ListenableFuture<ImagenGenerationResponse<ImagenInlineImage>>
47+
4248
/** Returns the [ImagenModel] object wrapped by this object. */
4349
public abstract fun getImageModel(): ImagenModel
4450

@@ -48,6 +54,12 @@ public abstract class ImagenModelFutures internal constructor() {
4854
): ListenableFuture<ImagenGenerationResponse<ImagenInlineImage>> =
4955
SuspendToFutureAdapter.launchFuture { model.generateImages(prompt) }
5056

57+
override fun editImage(
58+
prompt: String,
59+
config: ImagenEditingConfig
60+
): ListenableFuture<ImagenGenerationResponse<ImagenInlineImage>> =
61+
SuspendToFutureAdapter.launchFuture { model.editImage(prompt, config) }
62+
5163
override fun getImageModel(): ImagenModel = model
5264
}
5365

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.google.firebase.ai.type
2+
3+
public class ImagenEditMode private constructor(internal val value: String) {
4+
5+
public companion object {
6+
public val INPAINT_INSERTION: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_INSERTION")
7+
public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL")
8+
public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT")
9+
}
10+
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package com.google.firebase.ai.type
2+
3+
import kotlinx.serialization.Serializable
4+
5+
@PublicPreviewAPI
6+
public class ImagenEditingConfig(
7+
internal val image: ImagenInlineImage,
8+
internal val editMode: ImagenEditMode,
9+
internal val mask: ImagenInlineImage? = null,
10+
internal val maskDilation: Double? = null,
11+
internal val editSteps: Int? = null,
12+
) {
13+
public companion object {
14+
public fun builder(): Builder = Builder()
15+
}
16+
17+
public class Builder {
18+
@JvmField public var image: ImagenInlineImage? = null
19+
@JvmField public var editMode: ImagenEditMode? = null
20+
@JvmField public var mask: ImagenInlineImage? = null
21+
@JvmField public var maskDilation: Double? = null
22+
@JvmField public var editSteps: Int? = null
23+
24+
public fun setImage(image: ImagenInlineImage): Builder = apply { this.image = image }
25+
26+
public fun setEditMode(editMode: ImagenEditMode): Builder = apply { this.editMode = editMode }
27+
28+
public fun setMask(mask: ImagenInlineImage): Builder = apply { this.mask = mask }
29+
30+
public fun setMaskDilation(maskDilation: Double): Builder = apply {
31+
this.maskDilation = maskDilation
32+
}
33+
34+
public fun setEditSteps(editSteps: Int): Builder = apply { this.editSteps = editSteps }
35+
36+
public fun build(): ImagenEditingConfig {
37+
if (image == null) {
38+
throw IllegalStateException("ImagenEditingConfig must contain an image")
39+
}
40+
if (editMode == null) {
41+
throw IllegalStateException("ImagenEditingConfig must contain an editMode")
42+
}
43+
return ImagenEditingConfig(
44+
image = image!!,
45+
editMode = editMode!!,
46+
mask = mask,
47+
maskDilation = maskDilation,
48+
editSteps = editSteps,
49+
)
50+
}
51+
}
52+
53+
internal fun toInternal(): Internal {
54+
return Internal(baseSteps = editSteps)
55+
}
56+
57+
@Serializable
58+
internal data class Internal(
59+
val baseSteps: Int?,
60+
)
61+
}
62+
63+
@PublicPreviewAPI
64+
public fun imagenEditingConfig(init: ImagenEditingConfig.Builder.() -> Unit): ImagenEditingConfig {
65+
val builder = ImagenEditingConfig.builder()
66+
builder.init()
67+
return builder.build()
68+
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package com.google.firebase.ai.type
1818

1919
import android.graphics.Bitmap
2020
import android.graphics.BitmapFactory
21+
import android.util.Base64
22+
import java.io.ByteArrayOutputStream
23+
import kotlinx.serialization.Serializable
2124

2225
/**
2326
* Represents an Imagen-generated image that is returned as inline data.
@@ -36,4 +39,19 @@ internal constructor(public val data: ByteArray, public val mimeType: String) {
3639
public fun asBitmap(): Bitmap {
3740
return BitmapFactory.decodeByteArray(data, 0, data.size)
3841
}
42+
43+
@Serializable internal data class Internal(val bytesBase64Encoded: String)
44+
45+
internal fun toInternal(): Internal {
46+
val base64 = Base64.encodeToString(data, Base64.NO_WRAP)
47+
return Internal(base64)
48+
}
49+
}
50+
51+
@PublicPreviewAPI
52+
public fun Bitmap.toImagenInlineImage(): ImagenInlineImage {
53+
val byteArrayOutputStream = ByteArrayOutputStream()
54+
this.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream)
55+
val byteArray = byteArrayOutputStream.toByteArray()
56+
return ImagenInlineImage(data = byteArray, mimeType = "image/png")
3957
}

0 commit comments

Comments
 (0)