Skip to content

Commit d1715d3

Browse files
bburdiliakbburdiliak
committed
MediaContent instead of ImageContent / DocumentContent
1 parent 54093be commit d1715d3

File tree

7 files changed

+124
-106
lines changed

7 files changed

+124
-106
lines changed

‎anthropic-client/src/main/scala/io/cequence/openaiscala/anthropic/JsonFormats.scala

Lines changed: 42 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
11
package io.cequence.openaiscala.anthropic
22

33
import io.cequence.openaiscala.anthropic.domain.CacheControl.Ephemeral
4-
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{
5-
DocumentBlock,
6-
ImageBlock,
7-
TextBlock
8-
}
4+
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{MediaBlock, TextBlock}
95
import io.cequence.openaiscala.anthropic.domain.Content.{
10-
ContentBlock,
116
ContentBlockBase,
127
ContentBlocks,
138
SingleString
@@ -39,15 +34,18 @@ trait JsonFormats {
3934
JsonUtil.enumFormat[ChatRole](ChatRole.allValues: _*)
4035
implicit lazy val usageInfoFormat: Format[UsageInfo] = Json.format[UsageInfo]
4136

37+
def writeJsObject(cacheControl: CacheControl): JsObject = cacheControl match {
38+
case CacheControl.Ephemeral =>
39+
Json.obj("cache_control" -> Json.obj("type" -> "ephemeral"))
40+
}
41+
4242
implicit lazy val cacheControlFormat: Format[CacheControl] = new Format[CacheControl] {
4343
def reads(json: JsValue): JsResult[CacheControl] = json match {
4444
case JsObject(Seq(("type", JsString("ephemeral")))) => JsSuccess(CacheControl.Ephemeral)
4545
case _ => JsError("Invalid cache control")
4646
}
4747

48-
def writes(cacheControl: CacheControl): JsValue = cacheControl match {
49-
case CacheControl.Ephemeral => Json.obj("type" -> "ephemeral")
50-
}
48+
def writes(cacheControl: CacheControl): JsValue = writeJsObject(cacheControl)
5149
}
5250

5351
implicit lazy val cacheControlOptionFormat: Format[Option[CacheControl]] =
@@ -87,22 +85,34 @@ trait JsonFormats {
8785
implicit val config: JsonConfiguration = JsonConfiguration(SnakeCase)
8886
Json.writes[TextBlock]
8987
}
90-
implicit lazy val imageBlockWrites: Writes[ImageBlock] =
91-
(block: ImageBlock) =>
92-
Json.obj(
93-
"type" -> "image",
94-
"source" -> Json.obj(
95-
"type" -> block.`type`,
96-
"media_type" -> block.mediaType,
97-
"data" -> block.data
98-
)
99-
)
100-
implicit lazy val documentBlockWrites: Writes[DocumentBlock] =
101-
(block: DocumentBlock) =>
88+
// implicit lazy val imageBlockWrites: Writes[ImageBlock] =
89+
// (block: ImageBlock) =>
90+
// Json.obj(
91+
// "type" -> "image",
92+
// "source" -> Json.obj(
93+
// "type" -> block.`type`,
94+
// "media_type" -> block.mediaType,
95+
// "data" -> block.data
96+
// )
97+
// )
98+
//
99+
// implicit lazy val documentBlockWrites: Writes[DocumentBlock] =
100+
// (block: DocumentBlock) =>
101+
// Json.obj(
102+
// "type" -> "document",
103+
// "source" -> Json.obj(
104+
// "type" -> block.`type`,
105+
// "media_type" -> block.mediaType,
106+
// "data" -> block.data
107+
// )
108+
// )
109+
110+
implicit lazy val mediaBlockWrites: Writes[MediaBlock] =
111+
(block: MediaBlock) =>
102112
Json.obj(
103-
"type" -> "document",
113+
"type" -> block.`type`,
104114
"source" -> Json.obj(
105-
"type" -> block.`type`,
115+
"type" -> block.encoding,
106116
"media_type" -> block.mediaType,
107117
"data" -> block.data
108118
)
@@ -116,15 +126,10 @@ trait JsonFormats {
116126
Json.obj("type" -> "text") ++
117127
Json.toJson(textBlock)(textBlockWrites).as[JsObject] ++
118128
cacheControlToJsObject(cacheControl)
119-
case ContentBlockBase(imageBlock @ ImageBlock(_, _, _), maybeCacheControl) =>
120-
Json.toJson(imageBlock)(imageBlockWrites).as[JsObject] ++
121-
cacheControlToJsObject(maybeCacheControl)
122-
case ContentBlockBase(documentBlock @ DocumentBlock(_, _, _), maybeCacheControl) =>
123-
Json.toJson(documentBlock)(documentBlockWrites).as[JsObject] ++
124-
cacheControlToJsObject(maybeCacheControl) ++
125-
maybeCacheControl
126-
.map(cc => Json.toJson(cc)(cacheControlFormat.writes))
127-
.getOrElse(Json.obj())
129+
case ContentBlockBase(media @ MediaBlock(_, _, _, _), maybeCacheControl) =>
130+
Json.toJson(media)(mediaBlockWrites).as[JsObject] ++
131+
// cacheControlToJsObject(maybeCacheControl)
132+
maybeCacheControl.map(cc => writeJsObject(cc)).getOrElse(Json.obj())
128133

129134
}
130135

@@ -139,23 +144,17 @@ trait JsonFormats {
139144
case _ => JsError("Invalid text block")
140145
}
141146

142-
case "image" =>
143-
for {
144-
source <- (json \ "source").validate[JsObject]
145-
`type` <- (source \ "type").validate[String]
146-
mediaType <- (source \ "media_type").validate[String]
147-
data <- (source \ "data").validate[String]
148-
cacheControl <- (json \ "cache_control").validateOpt[CacheControl]
149-
} yield ContentBlockBase(ImageBlock(`type`, mediaType, data), cacheControl)
150-
151-
case "document" =>
147+
case imageOrDocument @ ("image" | "document") =>
152148
for {
153149
source <- (json \ "source").validate[JsObject]
154150
`type` <- (source \ "type").validate[String]
155151
mediaType <- (source \ "media_type").validate[String]
156152
data <- (source \ "data").validate[String]
157153
cacheControl <- (json \ "cache_control").validateOpt[CacheControl]
158-
} yield ContentBlockBase(DocumentBlock(`type`, mediaType, data), cacheControl)
154+
} yield ContentBlockBase(
155+
MediaBlock(imageOrDocument, `type`, mediaType, data),
156+
cacheControl
157+
)
159158

160159
case _ => JsError("Unsupported or invalid content block")
161160
}

‎anthropic-client/src/main/scala/io/cequence/openaiscala/anthropic/domain/Content.scala

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -77,52 +77,5 @@ object Content {
7777
): ContentBlockBase = image("image/webp")(data, cacheControl)
7878
}
7979

80-
case class ImageBlock(
81-
`type`: String,
82-
mediaType: String,
83-
data: String
84-
) extends ContentBlock
85-
86-
case class DocumentBlock(
87-
`type`: String,
88-
mediaType: String,
89-
data: String
90-
) extends ContentBlock
91-
92-
object DocumentBlock {
93-
def pdf(
94-
data: String,
95-
cacheControl: Option[CacheControl]
96-
): ContentBlockBase =
97-
ContentBlockBase(DocumentBlock("base64", "application/pdf", data), cacheControl)
98-
}
99-
100-
object ImageBlock {
101-
def jpeg(
102-
data: String,
103-
cacheControl: Option[CacheControl]
104-
): ContentBlockBase =
105-
ContentBlockBase(ImageBlock("base64", "image/jpeg", data), cacheControl)
106-
107-
def png(
108-
data: String,
109-
cacheControl: Option[CacheControl]
110-
): ContentBlockBase =
111-
ContentBlockBase(ImageBlock("base64", "image/png", data), cacheControl)
112-
113-
def gif(
114-
data: String,
115-
cacheControl: Option[CacheControl]
116-
): ContentBlockBase =
117-
ContentBlockBase(ImageBlock("base64", "image/gif", data), cacheControl)
118-
119-
def webp(
120-
data: String,
121-
cacheControl: Option[CacheControl]
122-
): ContentBlockBase =
123-
ContentBlockBase(ImageBlock("base64", "image/webp", data), cacheControl)
124-
}
125-
126-
// TODO: check PDF
12780
}
12881
}

‎anthropic-client/src/main/scala/io/cequence/openaiscala/anthropic/service/AnthropicServiceFactory.scala

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,21 @@ object AnthropicServiceFactory extends AnthropicServiceConsts {
7373
new AnthropicServiceClassImpl(defaultCoreUrl, authHeaders, timeouts)
7474
}
7575

76+
def withPdf(
77+
apiKey: String = getAPIKeyFromEnv(),
78+
timeouts: Option[Timeouts] = None
79+
)(
80+
implicit ec: ExecutionContext,
81+
materializer: Materializer
82+
): AnthropicService = {
83+
val authHeaders = Seq(
84+
("x-api-key", s"$apiKey"),
85+
("anthropic-version", apiVersion),
86+
("anthropic-beta", "pdfs-2024-09-25")
87+
)
88+
new AnthropicServiceClassImpl(defaultCoreUrl, authHeaders, timeouts)
89+
}
90+
7691
private def getAPIKeyFromEnv(): String =
7792
Option(System.getenv(envAPIKey)).getOrElse(
7893
throw new IllegalStateException(

‎anthropic-client/src/main/scala/io/cequence/openaiscala/anthropic/service/impl/package.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ package object impl extends AnthropicServiceConsts {
106106
val encoding = mediaType.takeWhile(_ != ',')
107107
val data = encodingAndData.drop(encoding.length + 1)
108108
ContentBlockBase(
109-
Content.ContentBlock.ImageBlock(encoding, mediaType, data),
109+
Content.ContentBlock.MediaBlock("image", encoding, mediaType, data),
110110
cacheControl
111111
) -> newCacheControlCount
112112
} else {

‎anthropic-client/src/test/scala/io/cequence/openaiscala/anthropic/JsonFormatsSpec.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,7 @@ package io.cequence.openaiscala.anthropic
33
import io.cequence.openaiscala.anthropic.JsonFormatsSpec.JsonPrintMode
44
import io.cequence.openaiscala.anthropic.JsonFormatsSpec.JsonPrintMode.{Compact, Pretty}
55
import io.cequence.openaiscala.anthropic.domain.CacheControl.Ephemeral
6-
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{
7-
ImageBlock,
8-
MediaBlock,
9-
TextBlock
10-
}
6+
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{MediaBlock, TextBlock}
117
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlockBase
128
import io.cequence.openaiscala.anthropic.domain.Message
139
import io.cequence.openaiscala.anthropic.domain.Message.{
@@ -124,7 +120,7 @@ class JsonFormatsSpec extends AnyWordSpecLike with Matchers with JsonFormats {
124120
val userMessage =
125121
UserMessageContent(
126122
Seq(
127-
ImageBlock.jpeg("Hello, world!", Some(Ephemeral)),
123+
MediaBlock.jpeg("Hello, world!", Some(Ephemeral)),
128124
ContentBlockBase(TextBlock("How are you?"))
129125
)
130126
)

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/AnthropicCreateMessageWithImage.scala

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package io.cequence.openaiscala.examples.nonopenai
22

3-
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{ImageBlock, TextBlock}
3+
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{MediaBlock, TextBlock}
44
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlockBase
55
import io.cequence.openaiscala.anthropic.domain.Message
66
import io.cequence.openaiscala.anthropic.domain.Message.UserMessageContent
@@ -29,14 +29,8 @@ object AnthropicCreateMessageWithImage extends ExampleBase[AnthropicService] {
2929
private val messages: Seq[Message] = Seq(
3030
UserMessageContent(
3131
Seq(
32-
ContentBlockBase(TextBlock("Describe me what is in the picture!")),
33-
ContentBlockBase(
34-
ImageBlock(
35-
`type` = "base64",
36-
mediaType = "image/jpeg",
37-
data = imageBase64Source
38-
)
39-
)
32+
ContentBlockBase(TextBlock("Describe to me what is in the picture!")),
33+
MediaBlock.jpeg(data = imageBase64Source)
4034
)
4135
)
4236
)

‎openai-examples/src/main/scala/io/cequence/openaiscala/examples/nonopenai/AnthropicCreateMessageWithPdf.scala

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package io.cequence.openaiscala.examples.nonopenai
2+
3+
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlock.{MediaBlock, TextBlock}
4+
import io.cequence.openaiscala.anthropic.domain.Content.ContentBlockBase
5+
import io.cequence.openaiscala.anthropic.domain.Message
6+
import io.cequence.openaiscala.anthropic.domain.Message.UserMessageContent
7+
import io.cequence.openaiscala.anthropic.domain.response.CreateMessageResponse
8+
import io.cequence.openaiscala.anthropic.domain.settings.AnthropicCreateMessageSettings
9+
import io.cequence.openaiscala.anthropic.service.{AnthropicService, AnthropicServiceFactory}
10+
import io.cequence.openaiscala.domain.NonOpenAIModelId
11+
import io.cequence.openaiscala.examples.ExampleBase
12+
13+
import java.awt.image.RenderedImage
14+
import java.io.{ByteArrayOutputStream, File}
15+
import java.nio.file.Files
16+
import java.util.Base64
17+
import javax.imageio.ImageIO
18+
import scala.concurrent.Future
19+
20+
// requires `openai-scala-anthropic-client` as a dependency
21+
object AnthropicCreateMessageWithPdf extends ExampleBase[AnthropicService] {
22+
23+
private val localImagePath = sys.env("EXAMPLE_PDF_PATH")
24+
private val pdfBase64Source =
25+
Base64.getEncoder.encodeToString(readPdfToBytes(localImagePath))
26+
27+
override protected val service: AnthropicService = AnthropicServiceFactory.withPdf()
28+
29+
private val messages: Seq[Message] = Seq(
30+
UserMessageContent(
31+
Seq(
32+
ContentBlockBase(TextBlock("Describe to me what is this PDF about!")),
33+
MediaBlock.pdf(data = pdfBase64Source)
34+
)
35+
)
36+
)
37+
38+
override protected def run: Future[_] =
39+
service
40+
.createMessage(
41+
messages,
42+
settings = AnthropicCreateMessageSettings(
43+
model =
44+
NonOpenAIModelId.claude_3_5_sonnet_20241022, // claude-3-5-sonnet-20241022 supports PDF (beta)
45+
max_tokens = 8192
46+
)
47+
)
48+
.map(printMessageContent)
49+
50+
def readPdfToBytes(filePath: String): Array[Byte] = {
51+
val pdfFile = new File(filePath)
52+
Files.readAllBytes(pdfFile.toPath)
53+
}
54+
55+
private def printMessageContent(response: CreateMessageResponse) = {
56+
val text =
57+
response.content.blocks.collect { case ContentBlockBase(TextBlock(text), _) => text }
58+
.mkString(" ")
59+
println(text)
60+
}
61+
}

0 commit comments

Comments
 (0)