Skip to content

Commit

Permalink
add recognition with hardcoded path to shape predictor and resnet model
Browse files Browse the repository at this point in the history
  • Loading branch information
Vladimir Kocheryzhkin authored and Vladimir Kocheryzhkin committed Mar 21, 2018
1 parent 0d7c146 commit 48b38cf
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 13 deletions.
8 changes: 7 additions & 1 deletion app/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,13 @@ add_library( # Sets the name of the library.
../dlib/dlib/entropy_decoder/entropy_decoder_kernel_2.cpp
../dlib/dlib/base64/base64_kernel_1.cpp
../dlib/dlib/threads/threads_kernel_1.cpp
../dlib/dlib/threads/threads_kernel_2.cpp )
../dlib/dlib/threads/threads_kernel_2.cpp
../dlib/dlib/threads/thread_pool_extension.cpp
../dlib/dlib/threads/async.cpp
../dlib/dlib/dnn/cpu_dlib.cpp
../dlib/dlib/dnn/tensor_tools.cpp
)


add_library( # Sets the name of the library.
native-lib
Expand Down
2 changes: 2 additions & 0 deletions app/src/main/AndroidManifest.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package="ch.hepia.iti.opencvnativeandroidstudio">

<uses-permission android:name="android.permission.CAMERA"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />

<uses-feature android:name="android.hardware.camera"/>
<uses-feature android:name="android.hardware.camera.autofocus"/>
Expand Down
87 changes: 76 additions & 11 deletions app/src/main/cpp/native-lib.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/features2d/features2d.hpp>

#include <dlib/dnn.h>
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing.h>
Expand All @@ -12,32 +13,96 @@

using namespace std;
using namespace cv;
using namespace dlib;

#define AppTag "OCVSample::Activity"

template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual = dlib::add_prev1<block<N,BN,1,dlib::tag1<SUBNET>>>;

template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual_down = dlib::add_prev2<dlib::avg_pool<2,2,2,2,dlib::skip1<dlib::tag2<block<N,BN,2,dlib::tag1<SUBNET>>>>>>;

template <int N, template <typename> class BN, int stride, typename SUBNET>
using block = BN<dlib::con<N,3,3,1,1,dlib::relu<BN<dlib::con<N,3,3,stride,stride,SUBNET>>>>>;

template <int N, typename SUBNET> using ares = dlib::relu<residual<block,N,dlib::affine,SUBNET>>;
template <int N, typename SUBNET> using ares_down = dlib::relu<residual_down<block,N,dlib::affine,SUBNET>>;

template <typename SUBNET> using alevel0 = ares_down<256,SUBNET>;
template <typename SUBNET> using alevel1 = ares<256,ares<256,ares_down<256,SUBNET>>>;
template <typename SUBNET> using alevel2 = ares<128,ares<128,ares_down<128,SUBNET>>>;
template <typename SUBNET> using alevel3 = ares<64,ares<64,ares<64,ares_down<64,SUBNET>>>>;
template <typename SUBNET> using alevel4 = ares<32,ares<32,ares<32,SUBNET>>>;

using anet_type = dlib::loss_metric<dlib::fc_no_bias<128, dlib::avg_pool_everything<
alevel0<
alevel1<
alevel2<
alevel3<
alevel4<
dlib::max_pool<3,3,2,2,dlib::relu<dlib::affine<dlib::con<32,7,7,2,2,
dlib::input_rgb_image_sized<150>
>>>>>>>>>>>>;

dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
dlib::shape_predictor sp;
anet_type net;

extern "C"
{
dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
void JNICALL Java_ch_hepia_iti_opencvnativeandroidstudio_MainActivity_loadResources(JNIEnv *env, jobject instance) {
__android_log_print(ANDROID_LOG_DEBUG, AppTag, "Load resources started");
FILE* file = fopen("/storage/emulated/0/Movies/shape_predictor_5_face_landmarks.dat","r+");
FILE* file2 = fopen("/storage/emulated/0/Movies/dlib_face_recognition_resnet_model_v1.dat","r+");

if (file != NULL && file2 != NULL)
{
dlib::deserialize("/storage/emulated/0/Movies/shape_predictor_5_face_landmarks.dat") >> sp;
dlib::deserialize("/storage/emulated/0/Movies/dlib_face_recognition_resnet_model_v1.dat") >> net;
__android_log_print(ANDROID_LOG_DEBUG, AppTag, "Resources found");
} else{
__android_log_print(ANDROID_LOG_DEBUG, AppTag, "Resources NOT found");
}
__android_log_print(ANDROID_LOG_DEBUG, AppTag, "Load resources completed");
}

void JNICALL Java_ch_hepia_iti_opencvnativeandroidstudio_MainActivity_salt(JNIEnv *env, jobject instance,
jlong matAddrGray,
jint nbrElem) {
Mat &temp = *(Mat *) matAddrGray;
std::vector<matrix<rgb_pixel>> faces;
dlib::cv_image<unsigned char> cimg(temp);
auto start = std::chrono::high_resolution_clock::now();

std::vector<dlib::rectangle> dets = detector(cimg);

auto finish = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> elapsed = finish - start;
__android_log_print(ANDROID_LOG_INFO, "App", "Elapsed time: = %0.4f sec", elapsed.count());
__android_log_print(ANDROID_LOG_INFO, "App", "Number of faces: = %d", dets.size());

for(auto i : dets)
for(auto face : dets)
{
int x = i.left();
int y = i.top();
int width = i.width();
int height = i.height();
auto shape = sp(cimg, face);
int x = face.left();
int y = face.top();
int width = face.width();
int height = face.height();
cv::Rect rect(x, y, width, height);
cv::rectangle(temp, rect, cv::Scalar(255));

matrix<rgb_pixel> face_chip;
extract_image_chip(cimg, get_face_chip_details(shape, 150, 0.25), face_chip);
faces.push_back(move(face_chip));
}

std::vector<matrix<float, 0, 1>> face_descriptors = net(faces);

auto finish = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> elapsed = finish - start;
__android_log_print(ANDROID_LOG_INFO, AppTag, "Elapsed time: = %0.2f sec", elapsed.count());
// __android_log_print(ANDROID_LOG_INFO, AppTag, "Number of faces: = %d", face_descriptors.size());
// for (size_t i = 0; i < face_descriptors.size(); ++i)
// {
// for (auto j : face_descriptors[i]) {
// __android_log_print(ANDROID_LOG_INFO, AppTag, "Desc: %f", j);
// }
// }
}
}
36 changes: 35 additions & 1 deletion app/src/main/java/ch/hepia/iti/opencvnativeandroidstudio/MainActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import android.Manifest;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Environment;
import android.support.v4.app.ActivityCompat;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;
Expand All @@ -15,6 +16,18 @@
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Mat;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;

import static android.os.Environment.getExternalStorageDirectory;

public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2 {

private static final String TAG = "OCVSample::Activity";
Expand All @@ -26,6 +39,8 @@ public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
Log.i(TAG, "OpenCV loaded successfully");


// Load ndk built module, as specified in moduleName in build.gradle
// after opencv initialization
System.loadLibrary("native-lib");
Expand All @@ -48,12 +63,16 @@ public void onCreate(Bundle savedInstanceState) {

// Permissions for Android 6+
ActivityCompat.requestPermissions(MainActivity.this,
new String[]{Manifest.permission.CAMERA},
new String[]{Manifest.permission.CAMERA,
Manifest.permission.READ_EXTERNAL_STORAGE,
Manifest.permission.WRITE_EXTERNAL_STORAGE},
1);

_cameraBridgeViewBase = (CameraBridgeViewBase) findViewById(R.id.main_surface);
_cameraBridgeViewBase.setVisibility(SurfaceView.VISIBLE);
_cameraBridgeViewBase.setCvCameraViewListener(this);


}

@Override
Expand All @@ -76,6 +95,13 @@ public void onResume() {

@Override
public void onRequestPermissionsResult(int requestCode, String permissions[], int[] grantResults) {
// for(int i = 0; i < permissions.length; i++){
// Log.d(TAG, permissions[i]);
// }
// for(int i = 0; i < grantResults.length; i++){
// Log.d(TAG, String.format("value = %d",grantResults[i]));
// }

switch (requestCode) {
case 1: {
// If request is cancelled, the result arrays are empty.
Expand All @@ -88,6 +114,12 @@ public void onRequestPermissionsResult(int requestCode, String permissions[], in
// functionality that depends on this permission.
Toast.makeText(MainActivity.this, "Permission denied to read your External storage", Toast.LENGTH_SHORT).show();
}

if (grantResults.length > 1 && grantResults[2] == PackageManager.PERMISSION_GRANTED) {
Log.d(TAG, "I got permissions");
Log.d(TAG, permissions[2]);
loadResources();
}
return;
}
// other 'case' lines to check for other
Expand All @@ -114,9 +146,11 @@ public void onCameraViewStopped() {
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
Mat matGray = inputFrame.gray();
salt(matGray.getNativeObjAddr(), 2000);
//loadResources();
return matGray;
}

public native void salt(long matAddrGray, int nbrElem);
public native void loadResources();
}

0 comments on commit 48b38cf

Please sign in to comment.