fix (ai/core): prefer auto-detected image mimetype (#4172)

vercel · Dec 20, 2024 · a8669a2 · a8669a2
1 parent 6fb3e91
commit a8669a2
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 10 deletions.
diff --git a/.changeset/breezy-tools-hunt.md b/.changeset/breezy-tools-hunt.md
@@ -0,0 +1,5 @@
+---
+'ai': patch
+---
+
+fix (ai/core): prefer auto-detected image mimetype
diff --git a/packages/ai/core/prompt/convert-to-language-model-prompt.test.ts b/packages/ai/core/prompt/convert-to-language-model-prompt.test.ts
@@ -560,7 +560,7 @@ describe('convertToLanguageModelMessage', () => {
             content: [
               {
                 type: 'image',
-                image: 'data:image/jpg;base64,dGVzdA==',
+                image: 'data:image/jpg;base64,/9j/3Q==',
               },
             ],
           },
@@ -572,8 +572,35 @@ describe('convertToLanguageModelMessage', () => {
           content: [
             {
               type: 'image',
-              image: new Uint8Array([116, 101, 115, 116]),
-              mimeType: 'image/jpg',
+              image: new Uint8Array([255, 216, 255, 221]),
+              mimeType: 'image/jpeg',
+            },
+          ],
+        });
+      });
+
+      it('should prefer detected mimetype', async () => {
+        const result = convertToLanguageModelMessage(
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'image',
+                // incorrect mimetype:
+                image: 'data:image/png;base64,/9j/3Q==',
+              },
+            ],
+          },
+          {},
+        );
+
+        expect(result).toEqual({
+          role: 'user',
+          content: [
+            {
+              type: 'image',
+              image: new Uint8Array([255, 216, 255, 221]),
+              mimeType: 'image/jpeg',
             },
           ],
         });

diff --git a/packages/ai/core/prompt/convert-to-language-model-prompt.ts b/packages/ai/core/prompt/convert-to-language-model-prompt.ts
@@ -284,20 +284,23 @@ function convertPartToLanguageModelPart(
   // Now that we have the normalized data either as a URL or a Uint8Array,
   // we can create the LanguageModelV1Part.
   switch (type) {
-    case 'image':
-      // We give a best effort to detect the mime type if it is not provided.
-      // otherwise, we use the provided mime type.
-      if (mimeType == null && normalizedData instanceof Uint8Array) {
-        mimeType = detectImageMimeType(normalizedData);
-      }
+    case 'image': {
+      // When possible, try to detect the mimetype automatically
+      // to deal with incorrect mimetype inputs.
+      // When detection fails, use provided mimetype.
 
+      if (normalizedData instanceof Uint8Array) {
+        mimeType = detectImageMimeType(normalizedData) ?? mimeType;
+      }
       return {
         type: 'image',
         image: normalizedData,
         mimeType,
         providerMetadata: part.experimental_providerMetadata,
       };
-    case 'file':
+    }
+
+    case 'file': {
       // We should have a mimeType at this point, if not, throw an error.
       if (mimeType == null) {
         throw new Error(`Mime type is missing for file part`);
@@ -312,5 +315,6 @@ function convertPartToLanguageModelPart(
         mimeType,
         providerMetadata: part.experimental_providerMetadata,
       };
+    }
   }
 }