Skip to content

Commit 5e92511

Browse files
azihsoynazihsoyn
authored
add LocalCustomModel support (#78)
* add LocalCustomModel support for android * support local model for ios and speed up example
1 parent ce757b9 commit 5e92511

File tree

12 files changed

+131
-54
lines changed

12 files changed

+131
-54
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## 0.14.0
2+
3+
* Support Local Custom model.
4+
This includes **Breaking change**.
5+
See detail
6+
https://github.com/azihsoyn/flutter_mlkit/blob/60fc24fc0e26bf0cbe2b74986e9dc28247df6804/lib/mlkit.dart#L255-L259
7+
18
## 0.13.3
29

310
* Migrated to AndroidX by @BugsBunnyBR

‎README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ The flutter team now has the [firebase_ml_vision](https://pub.dartlang.org/packa
2727
| Translation | yet | yet |
2828
| Smart Reply | yet | yet |
2929
| AutoML model inference | yet | yet |
30-
| Custom model | ✅ | ✅ |
30+
| Custom model(on device) | ✅ | ✅ |
31+
| Custom model(cloud) | ✅ | ✅ |
3132

3233
[What features are available on device or in the cloud?](https://firebase.google.com/docs/ml-kit/)
3334

‎android/src/main/java/com/azihsoyn/flutter/mlkit/MlkitPlugin.java

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import android.graphics.Rect;
1010
import android.media.ExifInterface;
1111
import android.net.Uri;
12+
import android.content.res.AssetManager;
13+
import android.content.res.AssetFileDescriptor;
1214

1315
import androidx.annotation.NonNull;
1416
import androidx.annotation.Nullable;
@@ -25,6 +27,7 @@
2527
import com.google.firebase.ml.common.modeldownload.FirebaseModelDownloadConditions;
2628
import com.google.firebase.ml.common.modeldownload.FirebaseModelManager;
2729
import com.google.firebase.ml.common.modeldownload.FirebaseRemoteModel;
30+
import com.google.firebase.ml.common.modeldownload.FirebaseLocalModel;
2831
import com.google.firebase.ml.custom.FirebaseModelDataType;
2932
import com.google.firebase.ml.custom.FirebaseModelInputOutputOptions;
3033
import com.google.firebase.ml.custom.FirebaseModelInputs;
@@ -289,18 +292,33 @@ public void onFailure(@NonNull Exception e) {
289292
}
290293
} else if (call.method.equals("FirebaseModelManager#registerLocalModelSource")) {
291294
FirebaseModelManager manager = FirebaseModelManager.getInstance();
292-
// TODO: next release
295+
296+
if (call.argument("source") != null) {
297+
Map<String, Object> sourceMap = call.argument("source");
298+
String modelName = (String) sourceMap.get("modelName");
299+
String assetFilePath = (String) sourceMap.get("assetFilePath");
300+
FirebaseLocalModel localSource =
301+
new FirebaseLocalModel.Builder(modelName)
302+
.setAssetFilePath("flutter_assets/"+assetFilePath)
303+
.build();
304+
FirebaseModelManager.getInstance().registerLocalModel(localSource);
305+
}
293306

294307
} else if (call.method.equals("FirebaseModelInterpreter#run")) {
295308
FirebaseModelInterpreter mInterpreter;
296309
String remoteModelName = call.argument("remoteModelName");
310+
String localModelName = call.argument("localModelName");
297311
try {
298312

299-
FirebaseModelOptions modelOptions = new FirebaseModelOptions.Builder()
300-
.setRemoteModelName(remoteModelName)
301-
// TODO: local model
302-
// .setLocalModelName("my_local_model")
303-
.build();
313+
FirebaseModelOptions.Builder builder = new FirebaseModelOptions.Builder();
314+
315+
if (remoteModelName != null) {
316+
builder.setRemoteModelName(remoteModelName);
317+
}
318+
if (localModelName != null) {
319+
builder.setLocalModelName(localModelName);
320+
}
321+
FirebaseModelOptions modelOptions = builder.build();
304322
FirebaseModelInputOutputOptions.Builder ioBuilder = new FirebaseModelInputOutputOptions.Builder();
305323
FirebaseModelInputs.Builder inputsBuilder = new FirebaseModelInputs.Builder();
306324

‎example/android/app/build.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ android {
3838
signingConfig signingConfigs.debug
3939
}
4040
}
41+
42+
aaptOptions {
43+
noCompress "tflite" // Your model's file extension: "tflite", "lite", etc.
44+
}
4145
}
4246

4347
flutter {

‎example/assets/mobilenet_quant.tflite

4.08 MB
Binary file not shown.

‎example/ios/Podfile.lock

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ PODS:
7575
- FirebaseMLVisionTextModel (0.16.0):
7676
- GoogleMobileVision/TextDetector (~> 1.4)
7777
- Flutter (1.0.0)
78+
- flutter_native_image (0.0.1):
79+
- Flutter
7880
- GoogleAPIClientForREST/Core (1.3.9):
7981
- GTMSessionFetcher (>= 1.1.7)
8082
- GoogleAPIClientForREST/Vision (1.3.9):
@@ -129,7 +131,7 @@ PODS:
129131
- GTMSessionFetcher/Core (= 1.2.2)
130132
- image_picker (0.0.1):
131133
- Flutter
132-
- mlkit (0.12.0):
134+
- mlkit (0.13.3):
133135
- Firebase/Core (~> 6.1)
134136
- Firebase/MLModelInterpreter (~> 6.1)
135137
- Firebase/MLNaturalLanguage (~> 6.1)
@@ -150,6 +152,7 @@ PODS:
150152

151153
DEPENDENCIES:
152154
- Flutter (from `.symlinks/flutter/ios`)
155+
- flutter_native_image (from `.symlinks/plugins/flutter_native_image/ios`)
153156
- image_picker (from `.symlinks/plugins/image_picker/ios`)
154157
- mlkit (from `.symlinks/plugins/mlkit/ios`)
155158

@@ -181,6 +184,8 @@ SPEC REPOS:
181184
EXTERNAL SOURCES:
182185
Flutter:
183186
:path: ".symlinks/flutter/ios"
187+
flutter_native_image:
188+
:path: ".symlinks/plugins/flutter_native_image/ios"
184189
image_picker:
185190
:path: ".symlinks/plugins/image_picker/ios"
186191
mlkit:
@@ -200,19 +205,20 @@ SPEC CHECKSUMS:
200205
FirebaseMLVisionFaceModel: 897589c0a1d967739cdbb495cf4ca296f8750f75
201206
FirebaseMLVisionLabelModel: e270ea50a47b9bd5f332c3dffa22eb896ee422cf
202207
FirebaseMLVisionTextModel: f5ca6d55bee8f00e08afca69d211fdc4efee1fef
203-
Flutter: 9d0fac939486c9aba2809b7982dfdbb47a7b0296
208+
Flutter: 58dd7d1b27887414a370fcccb9e645c08ffd7a6a
209+
flutter_native_image: 9c0b7451838484458e5b0fae007b86a4c2d4bdfe
204210
GoogleAPIClientForREST: 44c7b678cbab11d26a47eab7309d6179b1a61f0c
205211
GoogleAppMeasurement: 51d8d9ea48f0ca44484d29cfbdef976fbd4fc336
206212
GoogleMobileVision: 31cfb4319fd0c03d80105680abd9eae9da5e3b47
207213
GoogleToolboxForMac: b3553629623a3b1bff17f555e736cd5a6d95ad55
208214
GoogleUtilities: 84df567c76ca84f67b7bb40e769fdd4acc746a10
209215
GTMSessionFetcher: 61bb0f61a4cb560030f1222021178008a5727a23
210-
image_picker: 86b84c4fe89267356a1f17297a45b5d317ebd2e7
211-
mlkit: 9507552971bc285bc5bc260d54acd917994cf38e
216+
image_picker: 16e5fec1fbc87fd3b297c53e4048521eaf17cd06
217+
mlkit: fd0e655c0f29ce1f7d690a3e11149a6a29371136
212218
nanopb: 2901f78ea1b7b4015c860c2fdd1ea2fee1a18d48
213219
Protobuf: 7a877b7f3e5964e3fce995e2eb323dbc6831bb5a
214220
TensorFlowLite: 8b9dc4eb32eac0f8cb660c66bca7604da56dcc5a
215221

216222
PODFILE CHECKSUM: edb4f0c7e166e5d5f0db4fada6af1c826b94662d
217223

218-
COCOAPODS: 1.5.3
224+
COCOAPODS: 1.7.4

‎example/ios/Runner.xcodeproj/project.pbxproj

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = Release.xcconfig; path = Flutter/Release.xcconfig; sourceTree = "<group>"; };
4848
7AFFD8ED1D35381100E5BB4D /* AppDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = "<group>"; };
4949
7AFFD8EE1D35381100E5BB4D /* AppDelegate.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = "<group>"; };
50+
8D37DD099AD60C7021835BEE /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = "<group>"; };
5051
9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Debug.xcconfig; path = Flutter/Debug.xcconfig; sourceTree = "<group>"; };
5152
9740EEB31CF90195004384FC /* Generated.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Generated.xcconfig; path = Flutter/Generated.xcconfig; sourceTree = "<group>"; };
5253
9740EEBA1CF902C7004384FC /* Flutter.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Flutter.framework; path = Flutter/Flutter.framework; sourceTree = "<group>"; };
@@ -56,6 +57,8 @@
5657
97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
5758
97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
5859
97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
60+
A3D1C567AAB07EC1A2E30584 /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Pods/Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = "<group>"; };
61+
AA74D9D94EA42FC8CFC43F46 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = "<group>"; };
5962
DACFE9FE4422E495C3D8F3A0 /* libPods-Runner.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-Runner.a"; sourceTree = BUILT_PRODUCTS_DIR; };
6063
/* End PBXFileReference section */
6164

@@ -76,6 +79,9 @@
7679
43F63C7A32186CA58FC577BC /* Pods */ = {
7780
isa = PBXGroup;
7881
children = (
82+
8D37DD099AD60C7021835BEE /* Pods-Runner.debug.xcconfig */,
83+
AA74D9D94EA42FC8CFC43F46 /* Pods-Runner.release.xcconfig */,
84+
A3D1C567AAB07EC1A2E30584 /* Pods-Runner.profile.xcconfig */,
7985
);
8086
name = Pods;
8187
sourceTree = "<group>";
@@ -182,6 +188,7 @@
182188
TargetAttributes = {
183189
97C146ED1CF9000F007C117D = {
184190
CreatedOnToolsVersion = 7.3.1;
191+
DevelopmentTeam = 3TWY9JRL2P;
185192
};
186193
};
187194
};
@@ -262,23 +269,19 @@
262269
buildActionMask = 2147483647;
263270
files = (
264271
);
265-
inputFileListPaths = (
266-
);
267272
inputPaths = (
268-
"${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-resources.sh",
273+
"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources.sh",
269274
"${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVFaceDetectorResources.bundle",
270275
"${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVTextDetectorResources.bundle",
271276
);
272277
name = "[CP] Copy Pods Resources";
273-
outputFileListPaths = (
274-
);
275278
outputPaths = (
276279
"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVFaceDetectorResources.bundle",
277280
"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVTextDetectorResources.bundle",
278281
);
279282
runOnlyForDeploymentPostprocessing = 0;
280283
shellPath = /bin/sh;
281-
shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-resources.sh\"\n";
284+
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources.sh\"\n";
282285
showEnvVarsInLog = 0;
283286
};
284287
9740EEB61CF901F6004384FC /* Run Script */ = {
@@ -300,21 +303,17 @@
300303
buildActionMask = 2147483647;
301304
files = (
302305
);
303-
inputFileListPaths = (
304-
);
305306
inputPaths = (
306-
"${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh",
307+
"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh",
307308
"${PODS_ROOT}/../.symlinks/flutter/ios/Flutter.framework",
308309
);
309310
name = "[CP] Embed Pods Frameworks";
310-
outputFileListPaths = (
311-
);
312311
outputPaths = (
313312
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Flutter.framework",
314313
);
315314
runOnlyForDeploymentPostprocessing = 0;
316315
shellPath = /bin/sh;
317-
shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n";
316+
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n";
318317
showEnvVarsInLog = 0;
319318
};
320319
/* End PBXShellScriptBuildPhase section */
@@ -406,7 +405,7 @@
406405
buildSettings = {
407406
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
408407
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
409-
DEVELOPMENT_TEAM = S8QB4VV633;
408+
DEVELOPMENT_TEAM = 3TWY9JRL2P;
410409
ENABLE_BITCODE = NO;
411410
FRAMEWORK_SEARCH_PATHS = (
412411
"$(inherited)",
@@ -533,6 +532,7 @@
533532
buildSettings = {
534533
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
535534
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
535+
DEVELOPMENT_TEAM = 3TWY9JRL2P;
536536
ENABLE_BITCODE = NO;
537537
FRAMEWORK_SEARCH_PATHS = (
538538
"$(inherited)",
@@ -557,6 +557,7 @@
557557
buildSettings = {
558558
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
559559
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
560+
DEVELOPMENT_TEAM = 3TWY9JRL2P;
560561
ENABLE_BITCODE = NO;
561562
FRAMEWORK_SEARCH_PATHS = (
562563
"$(inherited)",

‎example/lib/custom-models.dart

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import 'dart:typed_data';
44

55
import 'package:flutter/material.dart';
66
import 'package:flutter/services.dart' show rootBundle;
7+
import 'package:flutter_native_image/flutter_native_image.dart';
78
import 'package:image/image.dart' as img;
89
import 'package:image_picker/image_picker.dart';
910
import 'package:mlkit/mlkit.dart';
@@ -34,17 +35,18 @@ class ObjectDetectionLabel {
3435

3536
class _CustomModelWidgetState extends State<CustomModelWidget> {
3637
List<String> _models = ["mobilenet_quant", "mobilenet_float", "coco"];
38+
List<String> _localModels = ["mobilenet_quant"];
3739

3840
File _file;
39-
int _currentModel = 2;
41+
int _currentModel = 0;
4042
List<ObjectDetectionLabel> _currentLabels = <ObjectDetectionLabel>[];
4143

4244
FirebaseModelInterpreter interpreter = FirebaseModelInterpreter.instance;
4345
FirebaseModelManager manager = FirebaseModelManager.instance;
4446
Map<String, List<String>> labels = {
4547
"mobilenet_quant": null,
4648
"mobilenet_float": null,
47-
"coco": null
49+
"coco": null,
4850
};
4951

5052
Map<String, FirebaseModelInputOutputOptions> _ioOptions = {
@@ -84,6 +86,10 @@ class _CustomModelWidgetState extends State<CustomModelWidget> {
8486
updatesDownloadConditions:
8587
FirebaseModelDownloadConditions(requireWifi: true)));
8688
});
89+
_localModels.forEach((model) {
90+
manager.registerLocalModelSource(FirebaseLocalModelSource(
91+
modelName: model, assetFilePath: "assets/" + model + ".tflite"));
92+
});
8793

8894
rootBundle.loadString('assets/labels_mobilenet.txt').then((string) {
8995
var _l = string.split('\n');
@@ -129,12 +135,18 @@ class _CustomModelWidgetState extends State<CustomModelWidget> {
129135

130136
if (options.inputOptions[0].dataType ==
131137
FirebaseModelDataType.BYTE) {
132-
results = await interpreter.run(_models[_currentModel],
133-
options, (imageToByteListInt(_file, dim)));
138+
var bytes = await imageToByteListInt(_file, dim);
139+
results = await interpreter.run(
140+
localModelName: _localModels[_currentModel],
141+
inputOutputOptions: options,
142+
inputBytes: bytes);
134143
factor = 2.55;
135144
} else {
136-
results = await interpreter.run(_models[_currentModel],
137-
options, (imageToByteListFloat(_file, dim)));
145+
var bytes = await imageToByteListFloat(_file, dim);
146+
results = await interpreter.run(
147+
localModelName: _localModels[_currentModel],
148+
inputOutputOptions: options,
149+
inputBytes: bytes);
138150
}
139151

140152
print(results);
@@ -189,11 +201,12 @@ class _CustomModelWidgetState extends State<CustomModelWidget> {
189201
);
190202
}
191203

192-
Uint8List imageToByteListInt(File file, int _inputSize) {
193-
var bytes = file.readAsBytesSync();
204+
Future<Uint8List> imageToByteListInt(File file, int _inputSize) async {
205+
File compressedFile = await FlutterNativeImage.compressImage(file.path,
206+
quality: 80, targetWidth: _inputSize, targetHeight: _inputSize);
207+
var bytes = compressedFile.readAsBytesSync();
194208
var decoder = img.findDecoderForData(bytes);
195209
img.Image image = decoder.decodeImage(bytes);
196-
image = img.copyResize(image, width: _inputSize, height: _inputSize);
197210
var convertedBytes = new Uint8List(1 * _inputSize * _inputSize * 3);
198211
var buffer = new ByteData.view(convertedBytes.buffer);
199212
int pixelIndex = 0;
@@ -211,12 +224,12 @@ class _CustomModelWidgetState extends State<CustomModelWidget> {
211224
return convertedBytes;
212225
}
213226

214-
Uint8List imageToByteListFloat(File file, int _inputSize) {
215-
var bytes = file.readAsBytesSync();
227+
Future<Uint8List> imageToByteListFloat(File file, int _inputSize) async {
228+
File compressedFile = await FlutterNativeImage.compressImage(file.path,
229+
quality: 80, targetWidth: _inputSize, targetHeight: _inputSize);
230+
var bytes = compressedFile.readAsBytesSync();
216231
var decoder = img.findDecoderForData(bytes);
217232
img.Image image = decoder.decodeImage(bytes);
218-
image = img.copyResize(image, width: _inputSize, height: _inputSize);
219-
//return image.getBytes(format: img.Format.rgb);
220233
var convertedBytes = Float32List(1 * _inputSize * _inputSize * 3);
221234
var buffer = Float32List.view(convertedBytes.buffer);
222235
int pixelIndex = 0;

‎example/pubspec.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ dependencies:
2020
flutter:
2121
sdk: flutter
2222
image_picker:
23+
flutter_native_image:
24+
git: https://github.com/btastic/flutter_native_image.git
2325
image:
2426
# The following adds the Cupertino Icons font to your application.
2527
# Use with the CupertinoIcons class for iOS style icons.
@@ -47,6 +49,7 @@ flutter:
4749
assets:
4850
- assets/labels_mobilenet.txt
4951
- assets/labels_coco.txt
52+
- assets/mobilenet_quant.tflite
5053

5154
# An image asset can refer to one or more resolution-specific "variants", see
5255
# https://flutter.dev/assets-and-images/#resolution-aware.

0 commit comments

Comments
 (0)