Skip to content

Commit 0576dbb

Browse files
azihsoynazihsoyn
authored
ios custom model interpreter (#39)
* dartfmt * fix android cutom model interpreter * format by intellij * format objc code * implement registerCloudModelSource for ios * tmp save * implement ios custom model interpreter * update docs
1 parent 8533fa3 commit 0576dbb

File tree

4 files changed

+128
-29
lines changed

4 files changed

+128
-29
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.7.0
2+
3+
* support custom model interpreter for both OS (cloud hosting model only).
4+
15
## 0.6.4
26

37
* fix android custom model interpreter.

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ The flutter team now has the [firebase_ml_vision](https://pub.dartlang.org/packa
2222
| Label Images(on device) | ✅ | ✅ |
2323
| Label Images(cloud) | yet | yet |
2424
| Recognize landmarks(cloud) | yet | yet |
25-
| Custom model | ✅ | yet |
25+
| Custom model | ✅ | ✅ |
2626

2727
[What features are available on device or in the cloud?](https://firebase.google.com/docs/ml-kit/)
2828

‎ios/Classes/MlkitPlugin.m

Lines changed: 122 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -42,50 +42,48 @@ - (instancetype)init {
4242
}
4343

4444
UIImage* imageFromImageSourceWithData(NSData *data) {
45-
CGImageSourceRef imageSource = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL);
46-
CGImageRef imageRef = CGImageSourceCreateImageAtIndex(imageSource, 0, NULL);
47-
CFRelease(imageSource);
48-
UIImage *image = [UIImage imageWithCGImage:imageRef];
49-
CGImageRelease(imageRef);
50-
return image;
45+
CGImageSourceRef imageSource = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL);
46+
CGImageRef imageRef = CGImageSourceCreateImageAtIndex(imageSource, 0, NULL);
47+
CFRelease(imageSource);
48+
UIImage *image = [UIImage imageWithCGImage:imageRef];
49+
CGImageRelease(imageRef);
50+
return image;
5151
}
5252

5353
- (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result {
5454
FIRVision *vision = [FIRVision vision];
5555
NSMutableArray *ret = [NSMutableArray array];
5656
UIImage* uiImage = NULL;
57-
57+
FIRVisionImage *image = NULL;
58+
5859
if ([call.method hasSuffix:@"#detectFromPath"]) {
5960
NSString *path = call.arguments[@"filepath"];
6061
uiImage = [UIImage imageWithContentsOfFile:path];
62+
image = [[FIRVisionImage alloc] initWithImage:uiImage];
6163
} else if ([call.method hasSuffix:@"#detectFromBinary"]) {
6264
FlutterStandardTypedData* typedData = call.arguments[@"binary"];
6365
uiImage = [UIImage imageWithData: typedData.data];
64-
} else {
65-
result(FlutterMethodNotImplemented);
66-
return;
66+
image = [[FIRVisionImage alloc] initWithImage:uiImage];
6767
}
68-
69-
FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage];
70-
68+
7169
if ([call.method hasPrefix:@"FirebaseVisionTextDetector#detectFrom"]) {
7270
textDetector = [vision onDeviceTextRecognizer];
7371
[textDetector processImage:image
7472
completion:^(FIRVisionText *_Nullable resultText,
7573
NSError *_Nullable error) {
76-
if (error != nil) {
77-
[ret addObject:error.localizedDescription];
78-
result(ret);
79-
return;
80-
} else if (resultText != nil) {
81-
// Recognized text
82-
for (FIRVisionTextBlock *block in resultText.blocks) {
83-
[ret addObject:visionTextBlockToDictionary(block)];
84-
}
85-
}
86-
result(ret);
87-
return;
88-
}];
74+
if (error != nil) {
75+
[ret addObject:error.localizedDescription];
76+
result(ret);
77+
return;
78+
} else if (resultText != nil) {
79+
// Recognized text
80+
for (FIRVisionTextBlock *block in resultText.blocks) {
81+
[ret addObject:visionTextBlockToDictionary(block)];
82+
}
83+
}
84+
result(ret);
85+
return;
86+
}];
8987
} else if ([call.method hasPrefix:@"FirebaseVisionBarcodeDetector#detectFrom"]) {
9088
barcodeDetector = [vision barcodeDetector];
9189
[barcodeDetector detectInImage:image
@@ -159,7 +157,104 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result {
159157
result(ret);
160158
return;
161159
}];
162-
}else {
160+
} else if ([call.method hasPrefix:@"FirebaseModelManager#registerCloudModelSource"]) {
161+
if(call.arguments[@"source"] != [NSNull null] ){
162+
NSString *modeName = call.arguments[@"source"][@"modelName"];
163+
BOOL enableModelUpdates = call.arguments[@"source"][@"enableModelUpdates"];
164+
FIRModelDownloadConditions *initialDownloadConditions = [[FIRModelDownloadConditions alloc] initWithIsWiFiRequired:YES
165+
canDownloadInBackground:YES];
166+
FIRModelDownloadConditions *updatesDownloadConditions = [[FIRModelDownloadConditions alloc] initWithIsWiFiRequired:YES
167+
canDownloadInBackground:YES];
168+
if(call.arguments[@"source"][@"initialDownloadConditions"] != [NSNull null] ){
169+
BOOL requireWifi = call.arguments[@"source"][@"initialDownloadConditions"][@"requireWifi"];
170+
BOOL requireDeviceIdle = call.arguments[@"source"][@"initialDownloadConditions"][@"requireDeviceIdle"];
171+
initialDownloadConditions =
172+
[[FIRModelDownloadConditions alloc] initWithIsWiFiRequired:requireWifi
173+
canDownloadInBackground:requireDeviceIdle];
174+
}
175+
if(call.arguments[@"source"][@"updatesDownloadConditions"] != [NSNull null] ){
176+
BOOL requireWifi = call.arguments[@"source"][@"initialDownloadConditions"][@"requireWifi"];
177+
BOOL requireDeviceIdle = call.arguments[@"source"][@"initialDownloadConditions"][@"requireDeviceIdle"];
178+
initialDownloadConditions =
179+
updatesDownloadConditions =
180+
[[FIRModelDownloadConditions alloc] initWithIsWiFiRequired:requireWifi
181+
canDownloadInBackground:requireDeviceIdle];
182+
}
183+
FIRCloudModelSource *cloudModelSource =
184+
[[FIRCloudModelSource alloc] initWithModelName:modeName
185+
enableModelUpdates:enableModelUpdates
186+
initialConditions:initialDownloadConditions
187+
updateConditions:updatesDownloadConditions];
188+
BOOL registrationSuccess =
189+
[[FIRModelManager modelManager] registerCloudModelSource:cloudModelSource];
190+
}
191+
} else if ([call.method hasPrefix:@"FirebaseModelInterpreter#run"]) {
192+
NSString *cloudModelName = call.arguments[@"cloudModelName"];
193+
// TODO local model
194+
FIRModelOptions *options = [[FIRModelOptions alloc] initWithCloudModelName:cloudModelName localModelName:nil];
195+
FIRModelInterpreter *interpreter = [FIRModelInterpreter modelInterpreterWithOptions:options];
196+
FIRModelInputOutputOptions *ioOptions = [[FIRModelInputOutputOptions alloc] init];
197+
NSError *error;
198+
NSNumber *inputIndex = call.arguments[@"inputOutputOptions"][@"inputIndex"];
199+
NSNumber *inputDataType = call.arguments[@"inputOutputOptions"][@"inputDataType"];
200+
FIRModelElementType inputType = (FIRModelElementType)[inputDataType intValue];
201+
NSArray<NSNumber *> *inputDims = call.arguments[@"inputOutputOptions"][@"inputDims"];
202+
[ioOptions setInputFormatForIndex:[inputIndex unsignedIntegerValue]
203+
type:inputType
204+
dimensions:inputDims
205+
error:&error];
206+
if (error != nil) {
207+
NSLog(@"Failed setInputFormatForIndex with error: %@", error.localizedDescription);
208+
return;
209+
}
210+
211+
NSNumber *outputIndex = call.arguments[@"inputOutputOptions"][@"outputIndex"];
212+
NSNumber *outputDataType = call.arguments[@"inputOutputOptions"][@"outputDataType"];
213+
FIRModelElementType outputType = (FIRModelElementType)[outputDataType intValue];
214+
NSArray<NSNumber *> *outputDims = call.arguments[@"inputOutputOptions"][@"outputDims"];
215+
[ioOptions setOutputFormatForIndex:[outputIndex unsignedIntegerValue]
216+
type:outputType
217+
dimensions:outputDims
218+
error:&error];
219+
if (error != nil) {
220+
NSLog(@"Failed setOutputFormatForIndex with error: %@", error.localizedDescription);
221+
return;
222+
}
223+
FIRModelInputs *inputs = [[FIRModelInputs alloc] init];
224+
FlutterStandardTypedData* typedData = call.arguments[@"inputBytes"];
225+
// ...
226+
[inputs addInput:typedData.data error:&error]; // Repeat as necessary.
227+
if (error != nil) {
228+
NSLog(@"Failed addInput with error: %@", error);
229+
return;
230+
}
231+
[interpreter runWithInputs:inputs
232+
options:ioOptions
233+
completion:^(FIRModelOutputs * _Nullable outputs,
234+
NSError * _Nullable error) {
235+
if (error != nil || outputs == nil) {
236+
NSLog(@"Failed runWithInputs with error: %@", error.localizedDescription);
237+
return;
238+
}
239+
240+
NSArray <NSArray<NSNumber *> *>*outputArrayOfArrays = [outputs outputAtIndex:0 error:&error];
241+
if (error) {
242+
NSLog(@"Failed to process detection outputs with error: %@", error.localizedDescription);
243+
return;
244+
}
245+
246+
// Get the first output from the array of output arrays.
247+
if(!outputArrayOfArrays || !outputArrayOfArrays.firstObject || ![outputArrayOfArrays.firstObject isKindOfClass:[NSArray class]] || !outputArrayOfArrays.firstObject.firstObject || ![outputArrayOfArrays.firstObject.firstObject isKindOfClass:[NSNumber class]]) {
248+
NSLog(@"Failed to get the results array from output.");
249+
return;
250+
}
251+
252+
NSArray<NSNumber *> *ret = outputArrayOfArrays.firstObject;
253+
254+
result(ret);
255+
return;
256+
}];
257+
} else {
163258
result(FlutterMethodNotImplemented);
164259
}
165260
}

‎pubspec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: mlkit
22
description: A Flutter plugin to use the Firebase ML Kit.
3-
version: 0.6.4
3+
version: 0.7.0
44
author: azihsoyn <azihsoyn@gmail.com>
55
homepage: https://github.com/azihsoyn/flutter_mlkit
66

0 commit comments

Comments
 (0)