Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.

Commit 21bc07c

Browse files
DennisAlundbparrishMines
DennisAlund
authored and
bparrishMines
committed
Vision cloud label detection (#695)
Adding cloud label detection support
1 parent 9d0e880 commit 21bc07c

File tree

12 files changed

+292
-1
lines changed

12 files changed

+292
-1
lines changed

‎packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/CloudLabelDetector.java

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package io.flutter.plugins.firebasemlvision;
2+
3+
import android.support.annotation.NonNull;
4+
import com.google.android.gms.tasks.OnFailureListener;
5+
import com.google.android.gms.tasks.OnSuccessListener;
6+
import com.google.firebase.ml.vision.FirebaseVision;
7+
import com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions;
8+
import com.google.firebase.ml.vision.cloud.label.FirebaseVisionCloudLabel;
9+
import com.google.firebase.ml.vision.cloud.label.FirebaseVisionCloudLabelDetector;
10+
import com.google.firebase.ml.vision.common.FirebaseVisionImage;
11+
import io.flutter.plugin.common.MethodChannel;
12+
import java.util.ArrayList;
13+
import java.util.HashMap;
14+
import java.util.List;
15+
import java.util.Map;
16+
17+
class CloudLabelDetector implements Detector {
18+
public static final CloudLabelDetector instance = new CloudLabelDetector();
19+
20+
private CloudLabelDetector() {}
21+
22+
@Override
23+
public void handleDetection(
24+
FirebaseVisionImage image, Map<String, Object> options, final MethodChannel.Result result) {
25+
FirebaseVisionCloudLabelDetector detector =
26+
FirebaseVision.getInstance().getVisionCloudLabelDetector(parseOptions(options));
27+
detector
28+
.detectInImage(image)
29+
.addOnSuccessListener(
30+
new OnSuccessListener<List<FirebaseVisionCloudLabel>>() {
31+
@Override
32+
public void onSuccess(List<FirebaseVisionCloudLabel> firebaseVisionCloudLabels) {
33+
List<Map<String, Object>> labels =
34+
new ArrayList<>(firebaseVisionCloudLabels.size());
35+
for (FirebaseVisionCloudLabel label : firebaseVisionCloudLabels) {
36+
Map<String, Object> labelData = new HashMap<>();
37+
labelData.put("confidence", (double) label.getConfidence());
38+
labelData.put("entityId", label.getEntityId());
39+
labelData.put("label", label.getLabel());
40+
41+
labels.add(labelData);
42+
}
43+
44+
result.success(labels);
45+
}
46+
})
47+
.addOnFailureListener(
48+
new OnFailureListener() {
49+
@Override
50+
public void onFailure(@NonNull Exception e) {
51+
result.error("labelDetectorError", e.getLocalizedMessage(), null);
52+
}
53+
});
54+
}
55+
56+
private FirebaseVisionCloudDetectorOptions parseOptions(Map<String, Object> optionsData) {
57+
final int maxResults = (int) optionsData.get("maxResults");
58+
final int modelType = (int) optionsData.get("modelType");
59+
return new FirebaseVisionCloudDetectorOptions.Builder()
60+
.setMaxResults(maxResults)
61+
.setModelType(modelType)
62+
.build();
63+
}
64+
}

‎packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,16 @@ public void onMethodCall(MethodCall call, Result result) {
6161
result.error("labelDetectorError", e.getLocalizedMessage(), null);
6262
}
6363
break;
64+
case "CloudLabelDetector#detectInImage":
65+
try {
66+
image = filePathToVisionImage((String) call.argument("path"));
67+
CloudLabelDetector.instance.handleDetection(image, options, result);
68+
} catch (IOException e) {
69+
result.error("labelDetectorIOError", e.getLocalizedMessage(), null);
70+
} catch (Exception e) {
71+
result.error("labelDetectorError", e.getLocalizedMessage(), null);
72+
}
73+
break;
6474
case "TextRecognizer#detectInImage":
6575
try {
6676
image = filePathToVisionImage((String) call.argument("path"));

‎packages/firebase_ml_vision/example/lib/detector_painters.dart

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import 'dart:ui' as ui;
77
import 'package:firebase_ml_vision/firebase_ml_vision.dart';
88
import 'package:flutter/material.dart';
99

10-
enum Detector { barcode, face, label, text }
10+
enum Detector { barcode, face, label, cloudLabel, text }
1111

1212
class BarcodeDetectorPainter extends CustomPainter {
1313
BarcodeDetectorPainter(this.absoluteImageSize, this.barcodeLocations);

‎packages/firebase_ml_vision/example/lib/main.dart

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ class _MyHomePageState extends State<_MyHomePage> {
8181
case Detector.label:
8282
detector = FirebaseVision.instance.labelDetector();
8383
break;
84+
case Detector.cloudLabel:
85+
detector = FirebaseVision.instance.cloudLabelDetector();
86+
break;
8487
case Detector.text:
8588
detector = FirebaseVision.instance.textDetector();
8689
break;
@@ -109,6 +112,9 @@ class _MyHomePageState extends State<_MyHomePage> {
109112
case Detector.label:
110113
painter = new LabelDetectorPainter(_imageSize, results);
111114
break;
115+
case Detector.cloudLabel:
116+
painter = new LabelDetectorPainter(_imageSize, results);
117+
break;
112118
case Detector.text:
113119
painter = new TextDetectorPainter(_imageSize, results);
114120
break;
@@ -168,6 +174,10 @@ class _MyHomePageState extends State<_MyHomePage> {
168174
child: Text('Detect Label'),
169175
value: Detector.label,
170176
),
177+
const PopupMenuItem<Detector>(
178+
child: Text('Detect Cloud Label'),
179+
value: Detector.cloudLabel,
180+
),
171181
const PopupMenuItem<Detector>(
172182
child: Text('Detect Text'),
173183
value: Detector.text,

‎packages/firebase_ml_vision/ios/Classes/CloudLabelDetector.m

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#import "FirebaseMlVisionPlugin.h"
2+
3+
@implementation CloudLabelDetector
4+
static FIRVisionCloudLabelDetector *detector;
5+
6+
+ (void)handleDetection:(FIRVisionImage *)image
7+
options:(NSDictionary *)options
8+
result:(FlutterResult)result {
9+
FIRVision *vision = [FIRVision vision];
10+
detector = [vision cloudLabelDetectorWithOptions:[CloudLabelDetector parseOptions:options]];
11+
12+
[detector
13+
detectInImage:image
14+
completion:^(NSArray<FIRVisionCloudLabel *> *_Nullable labels, NSError *_Nullable error) {
15+
if (error) {
16+
[FLTFirebaseMlVisionPlugin handleError:error result:result];
17+
return;
18+
} else if (!labels) {
19+
result(@[]);
20+
}
21+
22+
NSMutableArray *labelData = [NSMutableArray array];
23+
for (FIRVisionCloudLabel *label in labels) {
24+
NSDictionary *data = @{
25+
@"confidence" : label.confidence,
26+
@"entityId" : label.entityId,
27+
@"label" : label.label
28+
};
29+
[labelData addObject:data];
30+
}
31+
32+
result(labelData);
33+
}];
34+
}
35+
36+
+ (FIRVisionCloudDetectorOptions *)parseOptions:(NSDictionary *)optionsData {
37+
FIRVisionCloudDetectorOptions *detector = [[FIRVisionCloudDetectorOptions alloc] init];
38+
39+
NSNumber *modelType = optionsData[@"modelType"];
40+
detector.modelType = (FIRVisionCloudModelType)[modelType intValue];
41+
42+
NSNumber *maxResults = optionsData[@"maxResults"];
43+
detector.maxResults = [maxResults unsignedIntegerValue];
44+
45+
return detector;
46+
}
47+
@end

‎packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,8 @@
2323
@interface LabelDetector : NSObject<Detector>
2424
@end
2525

26+
@interface CloudLabelDetector : NSObject<Detector>
27+
@end
28+
2629
@interface TextRecognizer : NSObject<Detector>
2730
@end

‎packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result
4444
[FaceDetector handleDetection:image options:options result:result];
4545
} else if ([@"LabelDetector#detectInImage" isEqualToString:call.method]) {
4646
[LabelDetector handleDetection:image options:options result:result];
47+
} else if ([@"CloudLabelDetector#detectInImage" isEqualToString:call.method]) {
48+
[CloudLabelDetector handleDetection:image options:options result:result];
4749
} else if ([@"TextRecognizer#detectInImage" isEqualToString:call.method]) {
4850
[TextRecognizer handleDetection:image options:options result:result];
4951
} else {

‎packages/firebase_ml_vision/lib/firebase_ml_vision.dart

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@ part 'src/face_detector.dart';
1616
part 'src/firebase_vision.dart';
1717
part 'src/label_detector.dart';
1818
part 'src/text_detector.dart';
19+
part 'src/vision_cloud_detector_options.dart';

‎packages/firebase_ml_vision/lib/src/firebase_vision.dart

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ class FirebaseVision {
4343
return LabelDetector._(options ?? const LabelDetectorOptions());
4444
}
4545

46+
/// Creates an instance of [LabelDetector].
47+
CloudLabelDetector cloudLabelDetector([VisionCloudDetectorOptions options]) {
48+
return CloudLabelDetector._(options ?? const VisionCloudDetectorOptions());
49+
}
50+
4651
/// Creates an instance of [TextDetector].
4752
TextDetector textDetector() => new TextDetector._();
4853
}

‎packages/firebase_ml_vision/lib/src/label_detector.dart

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,41 @@ class LabelDetector extends FirebaseVisionDetector {
5050
}
5151
}
5252

53+
class CloudLabelDetector extends FirebaseVisionDetector {
54+
CloudLabelDetector._(this.options) : assert(options != null);
55+
56+
/// The options for the detector.
57+
///
58+
/// Sets the confidence threshold for detecting entities.
59+
final VisionCloudDetectorOptions options;
60+
61+
/// Detects entities in the input image.
62+
///
63+
/// Performed asynchronously.
64+
@override
65+
Future<List<Label>> detectInImage(FirebaseVisionImage visionImage) async {
66+
debugPrint(
67+
'Options: modelType=${options.modelType}, maxResults=${options.maxResults}');
68+
final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
69+
'CloudLabelDetector#detectInImage',
70+
<String, dynamic>{
71+
'path': visionImage.imageFile.path,
72+
'options': <String, dynamic>{
73+
'maxResults': options.maxResults,
74+
'modelType': options.modelType,
75+
},
76+
},
77+
);
78+
79+
final List<Label> labels = <Label>[];
80+
for (dynamic data in reply) {
81+
labels.add(Label._(data));
82+
}
83+
84+
return labels;
85+
}
86+
}
87+
5388
/// Options for Label detector.
5489
///
5590
/// Confidence threshold could be provided for the label detection. For example,

‎packages/firebase_ml_vision/lib/src/vision_cloud_detector_options.dart

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright 2018 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
part of firebase_ml_vision;
6+
7+
/// Options for cloud vision detectors.
8+
///
9+
class VisionCloudDetectorOptions {
10+
/// Constructor for [VisionCloudDetectorOptions].
11+
///
12+
const VisionCloudDetectorOptions(
13+
{this.maxResults = 10, this.modelType = modelTypeStable})
14+
: assert(maxResults > 0),
15+
assert(modelType == modelTypeLatest || modelType == modelTypeStable);
16+
17+
static const int modelTypeStable = 1;
18+
static const int modelTypeLatest = 2;
19+
20+
/// The number of results to be returned.
21+
///
22+
/// Defaults to 10.
23+
/// Required to be greater than zero.
24+
final int maxResults;
25+
26+
/// The type of model to use for the detection..
27+
///
28+
/// Defaults to [modelTypeStable]
29+
/// Required to be [modelTypeStable] or [modelTypeLatest].
30+
final int modelType;
31+
}

‎packages/firebase_ml_vision/test/firebase_ml_vision_test.dart

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ void main() {
2525
return returnValue;
2626
case 'LabelDetector#detectInImage':
2727
return returnValue;
28+
case 'CloudLabelDetector#detectInImage':
29+
return returnValue;
2830
case 'TextDetector#detectInImage':
2931
return returnValue;
3032
default:
@@ -633,6 +635,87 @@ void main() {
633635
});
634636
});
635637

638+
group('$CloudLabelDetector', () {
639+
test('detectInImage', () async {
640+
final List<dynamic> labelData = <dynamic>[
641+
<dynamic, dynamic>{
642+
'confidence': 0.6,
643+
'entityId': '/m/0',
644+
'label': 'banana',
645+
},
646+
<dynamic, dynamic>{
647+
'confidence': 0.8,
648+
'entityId': '/m/1',
649+
'label': 'apple',
650+
},
651+
];
652+
653+
returnValue = labelData;
654+
655+
final CloudLabelDetector detector =
656+
FirebaseVision.instance.cloudLabelDetector(
657+
const VisionCloudDetectorOptions(
658+
maxResults: 5,
659+
modelType: VisionCloudDetectorOptions.modelTypeLatest),
660+
);
661+
662+
final FirebaseVisionImage image = new FirebaseVisionImage.fromFilePath(
663+
'empty',
664+
);
665+
666+
final List<Label> labels = await detector.detectInImage(image);
667+
668+
expect(log, <Matcher>[
669+
isMethodCall(
670+
'CloudLabelDetector#detectInImage',
671+
arguments: <String, dynamic>{
672+
'path': 'empty',
673+
'options': <String, dynamic>{
674+
'maxResults': 5,
675+
'modelType': VisionCloudDetectorOptions.modelTypeLatest,
676+
},
677+
},
678+
),
679+
]);
680+
681+
expect(labels[0].confidence, 0.6);
682+
expect(labels[0].entityId, '/m/0');
683+
expect(labels[0].label, 'banana');
684+
685+
expect(labels[1].confidence, 0.8);
686+
expect(labels[1].entityId, '/m/1');
687+
expect(labels[1].label, 'apple');
688+
});
689+
690+
test('detectInImage no blocks', () async {
691+
returnValue = <dynamic>[];
692+
693+
final CloudLabelDetector detector =
694+
FirebaseVision.instance.cloudLabelDetector(
695+
const VisionCloudDetectorOptions(),
696+
);
697+
final FirebaseVisionImage image =
698+
new FirebaseVisionImage.fromFilePath('empty');
699+
700+
final List<Label> labels = await detector.detectInImage(image);
701+
702+
expect(log, <Matcher>[
703+
isMethodCall(
704+
'CloudLabelDetector#detectInImage',
705+
arguments: <String, dynamic>{
706+
'path': 'empty',
707+
'options': <String, dynamic>{
708+
'maxResults': 10,
709+
'modelType': VisionCloudDetectorOptions.modelTypeStable,
710+
},
711+
},
712+
),
713+
]);
714+
715+
expect(labels, isEmpty);
716+
});
717+
});
718+
636719
group('$TextDetector', () {
637720
test('detectInImage', () async {
638721
final Map<dynamic, dynamic> textElement = <dynamic, dynamic>{

0 commit comments

Comments
 (0)