Add the ability for ML Kit to create image from bytes

packages/firebase_ml_vision/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1
+
+* Add capability to create image from bytes.
+
 ## 0.2.0+2
 
 * Fix bug with empty text object.

packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java

@@ -2,6 +2,7 @@
 
 import android.net.Uri;
 import com.google.firebase.ml.vision.common.FirebaseVisionImage;
+import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata;
 import io.flutter.plugin.common.MethodCall;
 import io.flutter.plugin.common.MethodChannel;
 import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
@@ -29,55 +30,74 @@ public static void registerWith(Registrar registrar) {
   @Override
   public void onMethodCall(MethodCall call, Result result) {
     Map<String, Object> options = call.argument("options");
+
     FirebaseVisionImage image;
+    Map<String, Object> imageData = call.arguments();
+    try {
+      image = dataToVisionImage(imageData);
+    } catch (IOException exception) {
+      result.error("MLVisionDetectorIOError", exception.getLocalizedMessage(), null);
+      return;
+    }
+
     switch (call.method) {
       case "BarcodeDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          BarcodeDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        BarcodeDetector.instance.handleDetection(image, options, result);
         break;
       case "FaceDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          FaceDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("faceDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        FaceDetector.instance.handleDetection(image, options, result);
         break;
       case "LabelDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          LabelDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("labelDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        LabelDetector.instance.handleDetection(image, options, result);
         break;
       case "CloudLabelDetector#detectInImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          CloudLabelDetector.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("cloudLabelDetectorIOError", e.getLocalizedMessage(), null);
-        }
+        CloudLabelDetector.instance.handleDetection(image, options, result);
         break;
       case "TextRecognizer#processImage":
-        try {
-          image = filePathToVisionImage((String) call.argument("path"));
-          TextRecognizer.instance.handleDetection(image, options, result);
-        } catch (IOException e) {
-          result.error("textRecognizerIOError", e.getLocalizedMessage(), null);
-        }
+        TextRecognizer.instance.handleDetection(image, options, result);
         break;
       default:
         result.notImplemented();
     }
   }
 
-  private FirebaseVisionImage filePathToVisionImage(String path) throws IOException {
-    File file = new File(path);
-    return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
+  private FirebaseVisionImage dataToVisionImage(Map<String, Object> imageData) throws IOException {
+    String imageType = (String) imageData.get("type");
+
+    switch (imageType) {
+      case "file":
+        File file = new File((String) imageData.get("path"));
+        return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
+      case "bytes":
+        @SuppressWarnings("unchecked")
+        Map<String, Object> metadataData = (Map<String, Object>) imageData.get("metadata");
+
+        FirebaseVisionImageMetadata metadata =
+            new FirebaseVisionImageMetadata.Builder()
+                .setWidth((int) (double) metadataData.get("width"))
+                .setHeight((int) (double) metadataData.get("height"))
+                .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21)
+                .setRotation(getRotation((int) metadataData.get("rotation")))
+                .build();
+
+        return FirebaseVisionImage.fromByteArray((byte[]) imageData.get("bytes"), metadata);
+      default:
+        throw new IllegalArgumentException(String.format("No image type for: %s", imageType));
+    }
+  }
+
+  private int getRotation(int rotation) {
+    switch (rotation) {
+      case 0:
+        return FirebaseVisionImageMetadata.ROTATION_0;
+      case 90:
+        return FirebaseVisionImageMetadata.ROTATION_90;
+      case 180:
+        return FirebaseVisionImageMetadata.ROTATION_180;
+      case 270:
+        return FirebaseVisionImageMetadata.ROTATION_270;
+      default:
+        throw new IllegalArgumentException(String.format("No rotation for: %d", rotation));
+    }
   }
 }

packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m

@@ -36,7 +36,7 @@ - (instancetype)init {
 }
 
 - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result {
-  FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]];
+  FIRVisionImage *image = [self dataToVisionImage:call.arguments];
   NSDictionary *options = call.arguments[@"options"];
   if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) {
     [BarcodeDetector handleDetection:image options:options result:result];
@@ -53,8 +53,22 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result
   }
 }
 
-- (FIRVisionImage *)filePathToVisionImage:(NSString *)path {
-  UIImage *image = [UIImage imageWithContentsOfFile:path];
-  return [[FIRVisionImage alloc] initWithImage:image];
+- (FIRVisionImage *)dataToVisionImage:(NSDictionary *)imageData {
+  NSString *imageType = imageData[@"type"];
+
+  if ([@"file" isEqualToString:imageType]) {
+    UIImage *image = [UIImage imageWithContentsOfFile:imageData[@"path"]];
+    return [[FIRVisionImage alloc] initWithImage:image];
+  } else if ([@"bytes" isEqualToString:imageType]) {
+    FlutterStandardTypedData *byteData = imageData[@"bytes"];
+    NSData *imageBytes = byteData.data;
+    UIImage *image = [[UIImage alloc] initWithData:imageBytes];
+    // TODO(bmparr): Rotate image from imageData[@"rotation"].
+    return [[FIRVisionImage alloc] initWithImage:image];
+  } else {
+    NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType];
+    @throw
+        [NSException exceptionWithName:NSInvalidArgumentException reason:errorReason userInfo:nil];
+  }
 }
 @end

packages/firebase_ml_vision/lib/firebase_ml_vision.dart

@@ -7,6 +7,8 @@ library firebase_ml_vision;
 import 'dart:async';
 import 'dart:io';
 import 'dart:math';
+import 'dart:typed_data';
+import 'dart:ui';
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';

packages/firebase_ml_vision/lib/src/barcode_detector.dart

@@ -189,11 +189,10 @@ class BarcodeDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'BarcodeDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{
           'barcodeFormats': options.barcodeFormats.value,
         },
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Barcode> barcodes = <Barcode>[];

packages/firebase_ml_vision/lib/src/cloud_detector_options.dart

@@ -29,7 +29,7 @@ class CloudDetectorOptions {
   /// The type of model to use for the detection.
   final CloudModelType modelType;
 
-  Map<String, dynamic> _toMap() => <String, dynamic>{
+  Map<String, dynamic> _serialize() => <String, dynamic>{
         'maxResults': maxResults,
         'modelType': _enumToString(modelType),
       };

packages/firebase_ml_vision/lib/src/face_detector.dart

@@ -44,15 +44,14 @@ class FaceDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'FaceDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{
           'enableClassification': options.enableClassification,
           'enableLandmarks': options.enableLandmarks,
           'enableTracking': options.enableTracking,
           'minFaceSize': options.minFaceSize,
           'mode': _enumToString(options.mode),
         },
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Face> faces = <Face>[];

packages/firebase_ml_vision/lib/src/firebase_vision.dart

@@ -4,6 +4,13 @@
 
 part of firebase_ml_vision;
 
+enum _ImageType { file, bytes }
+
+/// Indicates the image rotation.
+///
+/// Rotation is counter-clockwise.
+enum ImageRotation { rotation_0, rotation_90, rotation_180, rotation_270 }
+
 /// The Firebase machine learning vision API.
 ///
 /// You can get an instance by calling [FirebaseVision.instance] and then get
@@ -56,22 +63,83 @@ class FirebaseVision {
 ///
 /// Create an instance by calling one of the factory constructors.
 class FirebaseVisionImage {
-  FirebaseVisionImage._(this.imageFile);
+  const FirebaseVisionImage._({
+    @required _ImageType type,
+    FirebaseVisionImageMetadata metadata,
+    File imageFile,
+    Uint8List bytes,
+  })  : _imageFile = imageFile,
+        _metadata = metadata,
+        _bytes = bytes,
+        _type = type;
 
+  // TODO(bmparr): Add [ImageOrientation] when passing file.
   /// Construct a [FirebaseVisionImage] from a file.
   factory FirebaseVisionImage.fromFile(File imageFile) {
     assert(imageFile != null);
-    return FirebaseVisionImage._(imageFile);
+    return FirebaseVisionImage._(
+      type: _ImageType.file,
+      imageFile: imageFile,
+    );
   }
 
   /// Construct a [FirebaseVisionImage] from a file path.
   factory FirebaseVisionImage.fromFilePath(String imagePath) {
     assert(imagePath != null);
-    return FirebaseVisionImage._(File(imagePath));
+    return FirebaseVisionImage._(
+      type: _ImageType.file,
+      imageFile: File(imagePath),
+    );
+  }
+
+  /// Construct a [FirebaseVisionImage] from a list of bytes.
+  ///
+  /// Expects `ImageFormat.NV21` on Android and expects bytes from `NSData`
+  /// provided from a `UIImage` on iOS. (e.g. using
+  /// `UIImageJPEGRepresentation()` on the platform side).
+  factory FirebaseVisionImage.fromBytes(
+    Uint8List bytes,
+    FirebaseVisionImageMetadata metadata,
+  ) {
+    assert(bytes != null);
+    return FirebaseVisionImage._(
+      type: _ImageType.bytes,
+      bytes: bytes,
+      metadata: metadata,
+    );
   }
 
-  /// The file location of the image.
-  final File imageFile;
+  final Uint8List _bytes;
+  final File _imageFile;
+  final FirebaseVisionImageMetadata _metadata;
+  final _ImageType _type;
+
+  Map<String, dynamic> _serialize() => <String, dynamic>{
+        'type': _enumToString(_type),
+        'bytes': _bytes,
+        'path': _imageFile?.path,
+        'metadata': _type == _ImageType.bytes ? _metadata._serialize() : null,
+      };
+}
+
+/// Image metadata used by [FirebaseVision] detectors.
+///
+/// [rotation] defaults to [ImageRotation.rotation_0]. Currently only rotates on
+/// Android.
+class FirebaseVisionImageMetadata {
+  const FirebaseVisionImageMetadata({
+    @required this.size,
+    this.rotation = ImageRotation.rotation_0,
+  }) : assert(size != null);
+
+  final Size size;
+  final ImageRotation rotation;
+
+  Map<String, dynamic> _serialize() => <String, dynamic>{
+        'width': size.width,
+        'height': size.height,
+        'rotation': _imageRotationToInt(rotation),
+      };
 }
 
 /// Abstract class for detectors in [FirebaseVision] API.
@@ -80,6 +148,20 @@ abstract class FirebaseVisionDetector {
   Future<dynamic> detectInImage(FirebaseVisionImage visionImage);
 }
 
+int _imageRotationToInt(ImageRotation rotation) {
+  switch (rotation) {
+    case ImageRotation.rotation_90:
+      return 90;
+    case ImageRotation.rotation_180:
+      return 180;
+    case ImageRotation.rotation_270:
+      return 270;
+    default:
+      assert(rotation == ImageRotation.rotation_0);
+      return 0;
+  }
+}
+
 String _enumToString(dynamic enumValue) {
   final String enumString = enumValue.toString();
   return enumString.substring(enumString.indexOf('.') + 1);

packages/firebase_ml_vision/lib/src/label_detector.dart

@@ -34,11 +34,10 @@ class LabelDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'LabelDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{
           'confidenceThreshold': options.confidenceThreshold,
         },
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Label> labels = <Label>[];
@@ -80,9 +79,8 @@ class CloudLabelDetector extends FirebaseVisionDetector {
     final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
       'CloudLabelDetector#detectInImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
-        'options': options._toMap(),
-      },
+        'options': options._serialize(),
+      }..addAll(visionImage._serialize()),
     );
 
     final List<Label> labels = <Label>[];

packages/firebase_ml_vision/lib/src/text_recognizer.dart

@@ -22,9 +22,8 @@ class TextRecognizer implements FirebaseVisionDetector {
         await FirebaseVision.channel.invokeMethod(
       'TextRecognizer#processImage',
       <String, dynamic>{
-        'path': visionImage.imageFile.path,
         'options': <String, dynamic>{},
-      },
+      }..addAll(visionImage._serialize()),
     );
 
     return VisionText._(reply);

packages/firebase_ml_vision/pubspec.yaml

@@ -3,7 +3,7 @@ description: Flutter plugin for Google ML Vision for Firebase, an SDK that bring
   learning expertise to Android and iOS apps in a powerful yet easy-to-use package.
 author: Flutter Team <flutter-dev@googlegroups.com>
 homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision
-version: 0.2.0+2
+version: 0.2.1
 
 dependencies:
   flutter: