Add the ability for ML Kit to create image from bytes

packages/firebase_ml_vision/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1
+
+* Add capability to create image from bytes.
+
 ## 0.2.0+2
 
 * Fix bug with empty text object.

packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java

@@ -2,6 +2,7 @@
 
 import android.net.Uri;
 import com.google.firebase.ml.vision.common.FirebaseVisionImage;
+import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata;
 import io.flutter.plugin.common.MethodCall;
 import io.flutter.plugin.common.MethodChannel;
 import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
@@ -29,55 +30,74 @@ public static void registerWith(Registrar registrar) {
   @Override
   public void onMethodCall(MethodCall call, Result result) {
     Map<String, Object> options = call.argument("options");
+
     FirebaseVisionImage image;
+    Map<String, Object> imageData = call.arguments();
+    try {
+      image = dataToVisionImage(imageData);
+    } catch (IOException exception) {
+      result.error("MLVisionDetectorIOError", exception.getLocalizedMessage(), null);
+      return;
+    }
+
     switch (call.method) {
       case "BarcodeDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          BarcodeDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        BarcodeDetector.instance.handleDetection(image, options, result);
         break;
       case "FaceDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          FaceDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("faceDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        FaceDetector.instance.handleDetection(image, options, result);
         break;
       case "LabelDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          LabelDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("labelDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        LabelDetector.instance.handleDetection(image, options, result);
         break;
       case "CloudLabelDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          CloudLabelDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("cloudLabelDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        CloudLabelDetector.instance.handleDetection(image, options, result);
         break;
       case "TextRecognizer#processImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          TextRecognizer.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("textRecognizerIOError", e.getLocalizedMessage(), null);
-        }
+        TextRecognizer.instance.handleDetection(image, options, result);
         break;
       default:
         result.notImplemented();
     }
   }
 
-  private FirebaseVisionImage filePathToVisionImage(String path) throws IOException {
-    File file = new File(path);
-    return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
+  private FirebaseVisionImage dataToVisionImage(Map<String, Object> imageData) throws IOException {
+    String imageType = (String) imageData.get("type");
+
+    switch (imageType) {
+      case "file":
+        File file = new File((String) imageData.get("path"));
+        return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
+      case "bytes":
+        @SuppressWarnings("unchecked")
+        Map<String, Object> metadataData = (Map<String, Object>) imageData.get("metadata");
+
+        FirebaseVisionImageMetadata metadata =
+            new FirebaseVisionImageMetadata.Builder()
+                .setWidth((int) (double) metadataData.get("width"))
+                .setHeight((int) (double) metadataData.get("height"))
+                .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21)
+                .setRotation(getRotation((int) metadataData.get("rotation")))
+                .build();
+
+        return FirebaseVisionImage.fromByteArray((byte[]) imageData.get("bytes"), metadata);
+      default:
+        throw new IllegalArgumentException(String.format("No image type for: %s", imageType));
+    }
+  }
+
+  private int getRotation(int rotation) {
+    switch (rotation) {
+      case 0:
+        return FirebaseVisionImageMetadata.ROTATION_0;
+      case 90:
+        return FirebaseVisionImageMetadata.ROTATION_90;
+      case 180:
+        return FirebaseVisionImageMetadata.ROTATION_180;
+      case 270:
+        return FirebaseVisionImageMetadata.ROTATION_270;
+      default:
+        throw new IllegalArgumentException(String.format("No rotation for: %d", rotation));
+    }
   }
 }

packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m

@@ -36,7 +36,7 @@ - (instancetype)init {
 }
 
 - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result {
-  FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]];
+  FIRVisionImage *image = [self dataToVisionImage:call.arguments];
   NSDictionary *options = call.arguments[@"options"];
   if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) {
     [BarcodeDetector handleDetection:image options:options result:result];
@@ -53,8 +53,74 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result
   }
 }
 
-- (FIRVisionImage *)filePathToVisionImage:(NSString *)path {
-  UIImage *image = [UIImage imageWithContentsOfFile:path];
-  return [[FIRVisionImage alloc] initWithImage:image];
+- (FIRVisionImage *)dataToVisionImage:(NSDictionary *)imageData {
+  NSString *imageType = imageData[@"type"];
+
+  if ([@"file" isEqualToString:imageType]) {
+    UIImage *image = [UIImage imageWithContentsOfFile:imageData[@"path"]];
+    return [[FIRVisionImage alloc] initWithImage:image];
+  } else if ([@"bytes" isEqualToString:imageType]) {
+    FlutterStandardTypedData *byteData = imageData[@"bytes"];
+    NSData *imageBytes = byteData.data;
+
+    NSDictionary *metadata = imageData[@"metadata"];
+    NSArray *planeData = metadata[@"planeData"];
+    size_t planeCount = planeData.count;
+
+    size_t widths[planeCount];
+    size_t heights[planeCount];
+    size_t bytesPerRows[planeCount];
+
+    void *baseAddresses[planeCount];
+    baseAddresses[0] = (void *)imageBytes.bytes;
+
+    size_t lastAddressIndex = 0;  // Used to get base address for each plane
+    for (int i = 0; i < planeCount; i++) {
+      NSDictionary *plane = planeData[i];
+
+      NSNumber *width = plane[@"width"];
+      NSNumber *height = plane[@"height"];
+      NSNumber *bytesPerRow = plane[@"bytesPerRow"];
+
+      widths[i] = width.unsignedLongValue;
+      heights[i] = height.unsignedLongValue;
+      bytesPerRows[i] = bytesPerRow.unsignedLongValue;
+
+      if (i > 0) {
+        size_t addressIndex = lastAddressIndex + heights[i - 1] * bytesPerRows[i - 1];
+        baseAddresses[i] = (void *)imageBytes.bytes + addressIndex;
+        lastAddressIndex = addressIndex;
+      }
+    }
+
+    NSNumber *width = metadata[@"width"];
+    NSNumber *height = metadata[@"height"];
+
+    NSNumber *rawFormat = metadata[@"rawFormat"];
+    FourCharCode format = FOUR_CHAR_CODE(rawFormat.unsignedIntValue);
+
+    CVPixelBufferRef pxbuffer = NULL;
+    CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault, width.unsignedLongValue,
+                                       height.unsignedLongValue, format, NULL, imageBytes.length, 2,
+                                       baseAddresses, widths, heights, bytesPerRows, NULL, NULL,
+                                       NULL, &pxbuffer);
+
+    CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pxbuffer];
+
+    CIContext *temporaryContext = [CIContext contextWithOptions:nil];
+    CGImageRef videoImage =
+        [temporaryContext createCGImage:ciImage
+                               fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pxbuffer),
+                                                   CVPixelBufferGetHeight(pxbuffer))];
+
+    UIImage *uiImage = [UIImage imageWithCGImage:videoImage];
+    CGImageRelease(videoImage);
+    return [[FIRVisionImage alloc] initWithImage:uiImage];
+  } else {
+    NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType];
+    @throw [NSException exceptionWithName:NSInvalidArgumentException
+                                   reason:errorReason
+                                 userInfo:nil];
+  }
 }
 @end

packages/firebase_ml_vision/lib/firebase_ml_vision.dart

@@ -7,6 +7,8 @@ library firebase_ml_vision;
 import 'dart:async';
 import 'dart:io';
 import 'dart:math';
+import 'dart:typed_data';
+import 'dart:ui';
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';

packages/firebase_ml_vision/lib/src/barcode_detector.dart

@@ -189,11 +189,10 @@ class BarcodeDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'BarcodeDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{
           'barcodeFormats': options.barcodeFormats.value,
         },
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Barcode> barcodes = <Barcode>[];

packages/firebase_ml_vision/lib/src/cloud_detector_options.dart

@@ -29,7 +29,7 @@ class CloudDetectorOptions {
   /// The type of model to use for the detection.
   final CloudModelType modelType;
 
-  Map<String, dynamic> _toMap() => <String, dynamic>{
+  Map<String, dynamic> _serialize() => <String, dynamic>{
         'maxResults': maxResults,
         'modelType': _enumToString(modelType),
       };

packages/firebase_ml_vision/lib/src/face_detector.dart

@@ -44,15 +44,14 @@ class FaceDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'FaceDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{
           'enableClassification': options.enableClassification,
           'enableLandmarks': options.enableLandmarks,
           'enableTracking': options.enableTracking,
           'minFaceSize': options.minFaceSize,
           'mode': _enumToString(options.mode),
         },
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Face> faces = <Face>[];