diff --git a/deploy/NCNN/Android/README.md b/deploy/NCNN/Android/README.md new file mode 100644 index 00000000..4fe8a7ae --- /dev/null +++ b/deploy/NCNN/Android/README.md @@ -0,0 +1,37 @@ +# ncnn-android-yolov6 + +The yolov6 object detection in `Android` . + +This is a sample ncnn android project, it depends on ncnn library and opencv + +https://github.com/Tencent/ncnn + +https://github.com/nihui/opencv-mobile + + +## how to build and run +### step1 +https://github.com/Tencent/ncnn/releases + +* Download ncnn-YYYYMMDD-android-vulkan.zip or build ncnn for android yourself +* Extract ncnn-YYYYMMDD-android-vulkan.zip into **app/src/main/jni** and change the **ncnn_DIR** path to yours in **app/src/main/jni/CMakeLists.txt** + +### step2 +https://github.com/nihui/opencv-mobile + +* Download opencv-mobile-XYZ-android.zip +* Extract opencv-mobile-XYZ-android.zip into **app/src/main/jni** and change the **OpenCV_DIR** path to yours in **app/src/main/jni/CMakeLists.txt** + +### step3 +* Open this project with Android Studio, build it and enjoy! + +## some notes +* Android ndk camera is used for best efficiency +* Crash may happen on very old devices for lacking HAL3 camera interface +* All models are manually modified to accept dynamic input shape +* Most small models run slower on GPU than on CPU, this is common +* FPS may be lower in dark environment because of longer camera exposure time + +## Reference： +https://github.com/nihui/ncnn-android-nanodet +https://github.com/Tencent/ncnn diff --git a/deploy/NCNN/Android/app/build.gradle b/deploy/NCNN/Android/app/build.gradle new file mode 100644 index 00000000..8f40b22f --- /dev/null +++ b/deploy/NCNN/Android/app/build.gradle @@ -0,0 +1,29 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 24 + + defaultConfig { + applicationId "com.tencent.yolov6ncnn" + archivesBaseName = "$applicationId" + + minSdkVersion 24 + } + + externalNativeBuild { + cmake { + version "3.10.2" + path file('src/main/jni/CMakeLists.txt') + } + } + + dependencies { + implementation 'com.android.support:support-v4:24.0.0' + } + ndkVersion '24.0.8215888' + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + namespace 'com.tencent.yolov6ncnn' +} diff --git a/deploy/NCNN/Android/app/src/main/AndroidManifest.xml b/deploy/NCNN/Android/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..25f64bdb --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/MainActivity.java b/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/MainActivity.java new file mode 100644 index 00000000..5ccacc29 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/MainActivity.java @@ -0,0 +1,162 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package com.tencent.yolov6ncnn; + +import android.Manifest; +import android.app.Activity; +import android.content.pm.PackageManager; +import android.graphics.PixelFormat; +import android.os.Bundle; +import android.util.Log; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import android.view.View; +import android.view.WindowManager; +import android.widget.AdapterView; +import android.widget.Button; +import android.widget.Spinner; + +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; + +public class MainActivity extends Activity implements SurfaceHolder.Callback +{ + public static final int REQUEST_CAMERA = 100; + + private Yolov6Ncnn yolov6ncnn = new Yolov6Ncnn(); + private int facing = 0; + + private Spinner spinnerModel; + private Spinner spinnerCPUGPU; + private int current_model = 0; + private int current_cpugpu = 0; + + private SurfaceView cameraView; + + /** Called when the activity is first created. */ + @Override + public void onCreate(Bundle savedInstanceState) + { + super.onCreate(savedInstanceState); + setContentView(R.layout.main); + + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + cameraView = (SurfaceView) findViewById(R.id.cameraview); + + cameraView.getHolder().setFormat(PixelFormat.RGBA_8888); + cameraView.getHolder().addCallback(this); + + Button buttonSwitchCamera = (Button) findViewById(R.id.buttonSwitchCamera); + buttonSwitchCamera.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View arg0) { + + int new_facing = 1 - facing; + + yolov6ncnn.closeCamera(); + + yolov6ncnn.openCamera(new_facing); + + facing = new_facing; + } + }); + + spinnerModel = (Spinner) findViewById(R.id.spinnerModel); + spinnerModel.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView arg0, View arg1, int position, long id) + { + if (position != current_model) + { + current_model = position; + reload(); + } + } + + @Override + public void onNothingSelected(AdapterView arg0) + { + } + }); + + spinnerCPUGPU = (Spinner) findViewById(R.id.spinnerCPUGPU); + spinnerCPUGPU.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView arg0, View arg1, int position, long id) + { + if (position != current_cpugpu) + { + current_cpugpu = position; + reload(); + } + } + + @Override + public void onNothingSelected(AdapterView arg0) + { + } + }); + + reload(); + } + + private void reload() + { + boolean ret_init = yolov6ncnn.loadModel(getAssets(), current_model, current_cpugpu); + if (!ret_init) + { + Log.e("MainActivity", "yolov6ncnn loadModel failed"); + } + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) + { + yolov6ncnn.setOutputWindow(holder.getSurface()); + } + + @Override + public void surfaceCreated(SurfaceHolder holder) + { + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) + { + } + + @Override + public void onResume() + { + super.onResume(); + + if (ContextCompat.checkSelfPermission(getApplicationContext(), Manifest.permission.CAMERA) == PackageManager.PERMISSION_DENIED) + { + ActivityCompat.requestPermissions(this, new String[] {Manifest.permission.CAMERA}, REQUEST_CAMERA); + } + + yolov6ncnn.openCamera(facing); + } + + @Override + public void onPause() + { + super.onPause(); + + yolov6ncnn.closeCamera(); + } +} diff --git a/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/Yolov6Ncnn.java b/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/Yolov6Ncnn.java new file mode 100644 index 00000000..36038ef8 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/java/com/tencent/yolov6ncnn/Yolov6Ncnn.java @@ -0,0 +1,30 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package com.tencent.yolov6ncnn; + +import android.content.res.AssetManager; +import android.view.Surface; + +public class Yolov6Ncnn +{ + public native boolean loadModel(AssetManager mgr, int modelid, int cpugpu); + public native boolean openCamera(int facing); + public native boolean closeCamera(); + public native boolean setOutputWindow(Surface surface); + + static { + System.loadLibrary("yolov6ncnn"); + } +} diff --git a/deploy/NCNN/Android/app/src/main/jni/CMakeLists.txt b/deploy/NCNN/Android/app/src/main/jni/CMakeLists.txt new file mode 100644 index 00000000..7eb4a559 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/CMakeLists.txt @@ -0,0 +1,13 @@ +project(yolov6ncnn) + +cmake_minimum_required(VERSION 3.10) + +set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-4.6.0-android/sdk/native/jni) +find_package(OpenCV REQUIRED core imgproc) + +set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20230223-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn) +find_package(ncnn REQUIRED) + +add_library(yolov6ncnn SHARED yolov6ncnn.cpp yolo.cpp ndkcamera.cpp) + +target_link_libraries(yolov6ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk) diff --git a/deploy/NCNN/Android/app/src/main/jni/ndkcamera.cpp b/deploy/NCNN/Android/app/src/main/jni/ndkcamera.cpp new file mode 100644 index 00000000..b4776de6 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/ndkcamera.cpp @@ -0,0 +1,771 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "ndkcamera.h" + +#include + +#include + +#include + +#include "mat.h" + +static void onDisconnected(void* context, ACameraDevice* device) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); +} + +static void onError(void* context, ACameraDevice* device, int error) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error); +} + +static void onImageAvailable(void* context, AImageReader* reader) +{ +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader); + + AImage* image = 0; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) + { + // error + return; + } + + int32_t format; + AImage_getFormat(image, &format); + + // assert format == AIMAGE_FORMAT_YUV_420_888 + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t* y_data = 0; + uint8_t* u_data = 0; + uint8_t* v_data = 0; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && v_rowStride == width) + { + // already nv21 :) + ((NdkCamera*)context)->on_image((unsigned char*)y_data, (int)width, (int)height); + } + else + { + // construct nv21 + unsigned char* nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char* yptr = nv21; + for (int y=0; yon_image((unsigned char*)nv21, (int)width, (int)height); + + delete[] nv21; + } + + AImage_delete(image); +} + +static void onSessionActive(void* context, ACameraCaptureSession *session) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session); +} + +static void onSessionReady(void* context, ACameraCaptureSession *session) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session); +} + +static void onSessionClosed(void* context, ACameraCaptureSession *session) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session); +} + +void onCaptureFailed(void* context, ACameraCaptureSession* session, ACaptureRequest* request, ACameraCaptureFailure* failure) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, failure); +} + +void onCaptureSequenceCompleted(void* context, ACameraCaptureSession* session, int sequenceId, int64_t frameNumber) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", session, sequenceId, frameNumber); +} + +void onCaptureSequenceAborted(void* context, ACameraCaptureSession* session, int sequenceId) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, sequenceId); +} + +void onCaptureCompleted(void* context, ACameraCaptureSession* session, ACaptureRequest* request, const ACameraMetadata* result) +{ +// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result); +} + +NdkCamera::NdkCamera() +{ + camera_facing = 0; + camera_orientation = 0; + + camera_manager = 0; + camera_device = 0; + image_reader = 0; + image_reader_surface = 0; + image_reader_target = 0; + capture_request = 0; + capture_session_output_container = 0; + capture_session_output = 0; + capture_session = 0; + + + // setup imagereader and its surface + { + AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader); + + AImageReader_ImageListener listener; + listener.context = this; + listener.onImageAvailable = onImageAvailable; + + AImageReader_setImageListener(image_reader, &listener); + + AImageReader_getWindow(image_reader, &image_reader_surface); + + ANativeWindow_acquire(image_reader_surface); + } +} + +NdkCamera::~NdkCamera() +{ + close(); + + if (image_reader) + { + AImageReader_delete(image_reader); + image_reader = 0; + } + + if (image_reader_surface) + { + ANativeWindow_release(image_reader_surface); + image_reader_surface = 0; + } +} + +int NdkCamera::open(int _camera_facing) +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open"); + + camera_facing = _camera_facing; + + camera_manager = ACameraManager_create(); + + // find front camera + std::string camera_id; + { + ACameraIdList* camera_id_list = 0; + ACameraManager_getCameraIdList(camera_manager, &camera_id_list); + + for (int i = 0; i < camera_id_list->numCameras; ++i) + { + const char* id = camera_id_list->cameraIds[i]; + ACameraMetadata* camera_metadata = 0; + ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata); + + // query faceing + acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT; + { + ACameraMetadata_const_entry e = { 0 }; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e); + facing = (acamera_metadata_enum_android_lens_facing_t)e.data.u8[0]; + } + + if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT) + { + ACameraMetadata_free(camera_metadata); + continue; + } + + if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK) + { + ACameraMetadata_free(camera_metadata); + continue; + } + + camera_id = id; + + // query orientation + int orientation = 0; + { + ACameraMetadata_const_entry e = { 0 }; + ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e); + + orientation = (int)e.data.i32[0]; + } + + camera_orientation = orientation; + + ACameraMetadata_free(camera_metadata); + + break; + } + + ACameraManager_deleteCameraIdList(camera_id_list); + } + + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), camera_orientation); + + // open camera + { + ACameraDevice_StateCallbacks camera_device_state_callbacks; + camera_device_state_callbacks.context = this; + camera_device_state_callbacks.onDisconnected = onDisconnected; + camera_device_state_callbacks.onError = onError; + + ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, &camera_device); + } + + // capture request + { + ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request); + + ACameraOutputTarget_create(image_reader_surface, &image_reader_target); + ACaptureRequest_addTarget(capture_request, image_reader_target); + } + + // capture session + { + ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks; + camera_capture_session_state_callbacks.context = this; + camera_capture_session_state_callbacks.onActive = onSessionActive; + camera_capture_session_state_callbacks.onReady = onSessionReady; + camera_capture_session_state_callbacks.onClosed = onSessionClosed; + + ACaptureSessionOutputContainer_create(&capture_session_output_container); + + ACaptureSessionOutput_create(image_reader_surface, &capture_session_output); + + ACaptureSessionOutputContainer_add(capture_session_output_container, capture_session_output); + + ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, &camera_capture_session_state_callbacks, &capture_session); + + ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks; + camera_capture_session_capture_callbacks.context = this; + camera_capture_session_capture_callbacks.onCaptureStarted = 0; + camera_capture_session_capture_callbacks.onCaptureProgressed = 0; + camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted; + camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed; + camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted; + camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted; + camera_capture_session_capture_callbacks.onCaptureBufferLost = 0; + + ACameraCaptureSession_setRepeatingRequest(capture_session, &camera_capture_session_capture_callbacks, 1, &capture_request, nullptr); + } + + return 0; +} + +void NdkCamera::close() +{ + __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close"); + + if (capture_session) + { + ACameraCaptureSession_stopRepeating(capture_session); + ACameraCaptureSession_close(capture_session); + capture_session = 0; + } + + if (camera_device) + { + ACameraDevice_close(camera_device); + camera_device = 0; + } + + if (capture_session_output_container) + { + ACaptureSessionOutputContainer_free(capture_session_output_container); + capture_session_output_container = 0; + } + + if (capture_session_output) + { + ACaptureSessionOutput_free(capture_session_output); + capture_session_output = 0; + } + + if (capture_request) + { + ACaptureRequest_free(capture_request); + capture_request = 0; + } + + if (image_reader_target) + { + ACameraOutputTarget_free(image_reader_target); + image_reader_target = 0; + } + + if (camera_manager) + { + ACameraManager_delete(camera_manager); + camera_manager = 0; + } +} + +void NdkCamera::on_image(const cv::Mat& rgb) const +{ +} + +void NdkCamera::on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const +{ + // rotate nv21 + int w = 0; + int h = 0; + int rotate_type = 0; + { + if (camera_orientation == 0) + { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 2 : 1; + } + if (camera_orientation == 90) + { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 5 : 6; + } + if (camera_orientation == 180) + { + w = nv21_width; + h = nv21_height; + rotate_type = camera_facing == 0 ? 4 : 3; + } + if (camera_orientation == 270) + { + w = nv21_height; + h = nv21_width; + rotate_type = camera_facing == 0 ? 7 : 8; + } + } + + cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1); + ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, rotate_type); + + // nv21_rotated to rgb + cv::Mat rgb(h, w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data); + + on_image(rgb); +} + +static const int NDKCAMERAWINDOW_ID = 233; + +NdkCameraWindow::NdkCameraWindow() : NdkCamera() +{ + sensor_manager = 0; + sensor_event_queue = 0; + accelerometer_sensor = 0; + win = 0; + + accelerometer_orientation = 0; + + // sensor + sensor_manager = ASensorManager_getInstance(); + + accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, ASENSOR_TYPE_ACCELEROMETER); +} + +NdkCameraWindow::~NdkCameraWindow() +{ + if (accelerometer_sensor) + { + ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor); + accelerometer_sensor = 0; + } + + if (sensor_event_queue) + { + ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue); + sensor_event_queue = 0; + } + + if (win) + { + ANativeWindow_release(win); + } +} + +void NdkCameraWindow::set_window(ANativeWindow* _win) +{ + if (win) + { + ANativeWindow_release(win); + } + + win = _win; + ANativeWindow_acquire(win); +} + +void NdkCameraWindow::on_image_render(cv::Mat& rgb) const +{ +} + +void NdkCameraWindow::on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const +{ + // resolve orientation from camera_orientation and accelerometer_sensor + { + if (!sensor_event_queue) + { + sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0); + + ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor); + } + + int id = ALooper_pollAll(0, 0, 0, 0); + if (id == NDKCAMERAWINDOW_ID) + { + ASensorEvent e[8]; + ssize_t num_event = 0; + while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1) + { + num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8); + if (num_event < 0) + break; + } + + if (num_event > 0) + { + float acceleration_x = e[num_event - 1].acceleration.x; + float acceleration_y = e[num_event - 1].acceleration.y; + float acceleration_z = e[num_event - 1].acceleration.z; +// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z); + + if (acceleration_y > 7) + { + accelerometer_orientation = 0; + } + if (acceleration_x < -7) + { + accelerometer_orientation = 90; + } + if (acceleration_y < -7) + { + accelerometer_orientation = 180; + } + if (acceleration_x > 7) + { + accelerometer_orientation = 270; + } + } + } + } + + // roi crop and rotate nv21 + int nv21_roi_x = 0; + int nv21_roi_y = 0; + int nv21_roi_w = 0; + int nv21_roi_h = 0; + int roi_x = 0; + int roi_y = 0; + int roi_w = 0; + int roi_h = 0; + int rotate_type = 0; + int render_w = 0; + int render_h = 0; + int render_rotate_type = 0; + { + int win_w = ANativeWindow_getWidth(win); + int win_h = ANativeWindow_getHeight(win); + + if (accelerometer_orientation == 90 || accelerometer_orientation == 270) + { + std::swap(win_w, win_h); + } + + const int final_orientation = (camera_orientation + accelerometer_orientation) % 360; + + if (final_orientation == 0 || final_orientation == 180) + { + if (win_w * nv21_height > win_h * nv21_width) + { + roi_w = nv21_width; + roi_h = (nv21_width * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_height - roi_h) / 2) / 2 * 2; + } + else + { + roi_h = nv21_height; + roi_w = (nv21_height * win_w / win_h) / 2 * 2; + roi_x = ((nv21_width - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_x; + nv21_roi_y = roi_y; + nv21_roi_w = roi_w; + nv21_roi_h = roi_h; + } + if (final_orientation == 90 || final_orientation == 270) + { + if (win_w * nv21_width > win_h * nv21_height) + { + roi_w = nv21_height; + roi_h = (nv21_height * win_h / win_w) / 2 * 2; + roi_x = 0; + roi_y = ((nv21_width - roi_h) / 2) / 2 * 2; + } + else + { + roi_h = nv21_width; + roi_w = (nv21_width * win_w / win_h) / 2 * 2; + roi_x = ((nv21_height - roi_w) / 2) / 2 * 2; + roi_y = 0; + } + + nv21_roi_x = roi_y; + nv21_roi_y = roi_x; + nv21_roi_w = roi_h; + nv21_roi_h = roi_w; + } + + if (camera_facing == 0) + { + if (camera_orientation == 0 && accelerometer_orientation == 0) + { + rotate_type = 2; + } + if (camera_orientation == 0 && accelerometer_orientation == 90) + { + rotate_type = 7; + } + if (camera_orientation == 0 && accelerometer_orientation == 180) + { + rotate_type = 4; + } + if (camera_orientation == 0 && accelerometer_orientation == 270) + { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 0) + { + rotate_type = 5; + } + if (camera_orientation == 90 && accelerometer_orientation == 90) + { + rotate_type = 2; + } + if (camera_orientation == 90 && accelerometer_orientation == 180) + { + rotate_type = 7; + } + if (camera_orientation == 90 && accelerometer_orientation == 270) + { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 0) + { + rotate_type = 4; + } + if (camera_orientation == 180 && accelerometer_orientation == 90) + { + rotate_type = 5; + } + if (camera_orientation == 180 && accelerometer_orientation == 180) + { + rotate_type = 2; + } + if (camera_orientation == 180 && accelerometer_orientation == 270) + { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 0) + { + rotate_type = 7; + } + if (camera_orientation == 270 && accelerometer_orientation == 90) + { + rotate_type = 4; + } + if (camera_orientation == 270 && accelerometer_orientation == 180) + { + rotate_type = 5; + } + if (camera_orientation == 270 && accelerometer_orientation == 270) + { + rotate_type = 2; + } + } + else + { + if (final_orientation == 0) + { + rotate_type = 1; + } + if (final_orientation == 90) + { + rotate_type = 6; + } + if (final_orientation == 180) + { + rotate_type = 3; + } + if (final_orientation == 270) + { + rotate_type = 8; + } + } + + if (accelerometer_orientation == 0) + { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 1; + } + if (accelerometer_orientation == 90) + { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 8; + } + if (accelerometer_orientation == 180) + { + render_w = roi_w; + render_h = roi_h; + render_rotate_type = 3; + } + if (accelerometer_orientation == 270) + { + render_w = roi_h; + render_h = roi_w; + render_rotate_type = 6; + } + } + + // crop and rotate nv21 + cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1); + { + const unsigned char* srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x; + unsigned char* dstY = nv21_croprotated.data; + ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, rotate_type); + + const unsigned char* srcUV = nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x; + unsigned char* dstUV = nv21_croprotated.data + roi_w * roi_h; + ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, roi_h / 2, roi_w, rotate_type); + } + + // nv21_croprotated to rgb + cv::Mat rgb(roi_h, roi_w, CV_8UC3); + ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data); + + on_image_render(rgb); + + // rotate to native window orientation + cv::Mat rgb_render(render_h, render_w, CV_8UC3); + ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, render_rotate_type); + + ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); + + ANativeWindow_Buffer buf; + ANativeWindow_lock(win, &buf, NULL); + + // scale to target size + if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM) + { + for (int y = 0; y < render_h; y++) + { + const unsigned char* ptr = rgb_render.ptr(y); + unsigned char* outptr = (unsigned char*)buf.bits + buf.stride * 4 * y; + + int x = 0; +#if __ARM_NEON + for (; x + 7 < render_w; x += 8) + { + uint8x8x3_t _rgb = vld3_u8(ptr); + uint8x8x4_t _rgba; + _rgba.val[0] = _rgb.val[0]; + _rgba.val[1] = _rgb.val[1]; + _rgba.val[2] = _rgb.val[2]; + _rgba.val[3] = vdup_n_u8(255); + vst4_u8(outptr, _rgba); + + ptr += 24; + outptr += 32; + } +#endif // __ARM_NEON + for (; x < render_w; x++) + { + outptr[0] = ptr[0]; + outptr[1] = ptr[1]; + outptr[2] = ptr[2]; + outptr[3] = 255; + + ptr += 3; + outptr += 4; + } + } + } + + ANativeWindow_unlockAndPost(win); +} diff --git a/deploy/NCNN/Android/app/src/main/jni/ndkcamera.h b/deploy/NCNN/Android/app/src/main/jni/ndkcamera.h new file mode 100644 index 00000000..ddd30eb8 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/ndkcamera.h @@ -0,0 +1,80 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef NDKCAMERA_H +#define NDKCAMERA_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +class NdkCamera +{ +public: + NdkCamera(); + virtual ~NdkCamera(); + + // facing 0=front 1=back + int open(int camera_facing = 0); + void close(); + + virtual void on_image(const cv::Mat& rgb) const; + + virtual void on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const; + +public: + int camera_facing; + int camera_orientation; + +private: + ACameraManager* camera_manager; + ACameraDevice* camera_device; + AImageReader* image_reader; + ANativeWindow* image_reader_surface; + ACameraOutputTarget* image_reader_target; + ACaptureRequest* capture_request; + ACaptureSessionOutputContainer* capture_session_output_container; + ACaptureSessionOutput* capture_session_output; + ACameraCaptureSession* capture_session; +}; + +class NdkCameraWindow : public NdkCamera +{ +public: + NdkCameraWindow(); + virtual ~NdkCameraWindow(); + + void set_window(ANativeWindow* win); + + virtual void on_image_render(cv::Mat& rgb) const; + + virtual void on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const; + +public: + mutable int accelerometer_orientation; + +private: + ASensorManager* sensor_manager; + mutable ASensorEventQueue* sensor_event_queue; + const ASensor* accelerometer_sensor; + ANativeWindow* win; +}; + +#endif // NDKCAMERA_H diff --git a/deploy/NCNN/Android/app/src/main/jni/yolo.cpp b/deploy/NCNN/Android/app/src/main/jni/yolo.cpp new file mode 100644 index 00000000..ce01888c --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/yolo.cpp @@ -0,0 +1,416 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "yolo.h" + +#include +#include + +#include "cpu.h" + +static float fast_exp(float x) +{ + union { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; +} + +static float sigmoid(float x) +{ + return 1.0f / (1.0f + fast_exp(-x)); +} +static float intersection_area(const Object& a, const Object& b) +{ + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); +} + +static void qsort_descent_inplace(std::vector& faceobjects, int left, int right) +{ + int i = left; + int j = right; + float p = faceobjects[(left + right) / 2].prob; + + while (i <= j) + { + while (faceobjects[i].prob > p) + i++; + + while (faceobjects[j].prob < p) + j--; + + if (i <= j) + { + // swap + std::swap(faceobjects[i], faceobjects[j]); + + i++; + j--; + } + } + + // #pragma omp parallel sections + { + // #pragma omp section + { + if (left < j) qsort_descent_inplace(faceobjects, left, j); + } + // #pragma omp section + { + if (i < right) qsort_descent_inplace(faceobjects, i, right); + } + } +} + +static void qsort_descent_inplace(std::vector& faceobjects) +{ + if (faceobjects.empty()) + return; + + qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1); +} + +static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold) +{ + picked.clear(); + + const int n = faceobjects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) + { + areas[i] = faceobjects[i].rect.width * faceobjects[i].rect.height; + } + + for (int i = 0; i < n; i++) + { + const Object& a = faceobjects[i]; + + int keep = 1; + for (int j = 0; j < (int)picked.size(); j++) + { + const Object& b = faceobjects[picked[j]]; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } +} + +static void generate_proposals(int stride, const ncnn::Mat& pred, float prob_threshold, std::vector& objects) +{ + const int num_c = pred.c; + const int num_grid_y = pred.h; + const int num_grid_x = pred.w; + const int num_classes = num_c - 4; + + for (int i = 0; i < num_grid_y; i++) { + for (int j = 0; j < num_grid_x; j++) { + const float *ptr = (float *) pred.data; + int class_index = 0; + float class_score = -1.f; + for (int c = 0; c < num_classes; c++) { + float score = ptr[c * num_grid_y * num_grid_x + i * num_grid_x + j]; + if (score > class_score) { + class_index = c; + class_score = score; + } + } + if (class_score >= prob_threshold && class_score < 1.f) { + float x0 = ptr[num_classes * num_grid_y * num_grid_x + i * num_grid_x + j]; + float y0 = ptr[(num_classes + 1) * num_grid_y * num_grid_x + i * num_grid_x + j]; + float x1 = ptr[(num_classes + 2) * num_grid_y * num_grid_x + i * num_grid_x + j]; + float y1 = ptr[(num_classes + 3) * num_grid_y * num_grid_x + i * num_grid_x + j]; + + x0 = (j + 0.5f - x0) * stride; + y0 = (i + 0.5f - y0) * stride; + x1 = (j + 0.5f + x1) * stride; + y1 = (i + 0.5f + y1) * stride; + + Object obj; + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + obj.label = class_index; + obj.prob = class_score; + + objects.push_back(obj); + + } + } + } +} + +Yolo::Yolo() +{ + blob_pool_allocator.set_size_compare_ratio(0.f); + workspace_pool_allocator.set_size_compare_ratio(0.f); +} + + +int Yolo::load(AAssetManager* mgr, const char* modeltype, const int *target_size, const float* _mean_vals, const float* _norm_vals, bool use_gpu) +{ + yolo.clear(); + blob_pool_allocator.clear(); + workspace_pool_allocator.clear(); + + ncnn::set_cpu_powersave(2); + ncnn::set_omp_num_threads(ncnn::get_big_cpu_count()); + + yolo.opt = ncnn::Option(); + +#if NCNN_VULKAN + yolo.opt.use_vulkan_compute = use_gpu; +#endif + + yolo.opt.num_threads = ncnn::get_big_cpu_count(); + yolo.opt.blob_allocator = &blob_pool_allocator; + yolo.opt.workspace_allocator = &workspace_pool_allocator; + + char parampath[256]; + char modelpath[256]; + sprintf(parampath, "yolov6-%s.param", modeltype); + sprintf(modelpath, "yolov6-%s.bin", modeltype); + + yolo.load_param(mgr, parampath); + yolo.load_model(mgr, modelpath); + + net_h = target_size[0]; + net_w = target_size[1]; + mean_vals[0] = _mean_vals[0]; + mean_vals[1] = _mean_vals[1]; + mean_vals[2] = _mean_vals[2]; + norm_vals[0] = _norm_vals[0]; + norm_vals[1] = _norm_vals[1]; + norm_vals[2] = _norm_vals[2]; + + return 0; +} + +int Yolo::detect(const cv::Mat& rgb, std::vector& objects, float prob_threshold, float nms_threshold) +{ + int width = rgb.cols; + int height = rgb.rows; + + // pad to multiple of 64 + int w = width; + int h = height; + float scale = 1.f; + if (w > h) + { + scale = net_w / (float)w; + w = net_w; + h = h * scale; + } + else + { + scale = net_h / (float)h; + h = net_h; + w = w * scale; + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgb.data, ncnn::Mat::PIXEL_RGB2BGR, width, height, w, h); + + // pad to net_h net_w rectangle + int wpad = (w + 63) / 64 * 64 - w; + int hpad = (h + 63) / 64 * 64 - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, ncnn::BORDER_CONSTANT, 114.f); + + in_pad.substract_mean_normalize(mean_vals, norm_vals); + + ncnn::Extractor ex = yolo.create_extractor(); + + ex.input("in0", in_pad); + + std::vector proposals; + + // stride 8 + { + ncnn::Mat out; + ex.extract("out0", out); + + std::vector objects8; + generate_proposals(8, out, prob_threshold, objects8); + + proposals.insert(proposals.end(), objects8.begin(), objects8.end()); + } + + // stride 16 + { + ncnn::Mat out; + ex.extract("out1", out); + + std::vector objects16; + generate_proposals(16, out, prob_threshold, objects16); + + proposals.insert(proposals.end(), objects16.begin(), objects16.end()); + } + + // stride 32 + { + ncnn::Mat out; + ex.extract("out2", out); + + std::vector objects32; + generate_proposals(32, out, prob_threshold, objects32); + + proposals.insert(proposals.end(), objects32.begin(), objects32.end()); + } + + // stride 64 + { + ncnn::Mat out; + ex.extract("out3", out); + + std::vector objects64; + generate_proposals(64, out, prob_threshold, objects64); + + proposals.insert(proposals.end(), objects64.begin(), objects64.end()); + } + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + +// objects.resize(count); + for (int i = 0; i < count; i++) + { + Object obj = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (obj.rect.x - (wpad / 2)) / scale; + float y0 = (obj.rect.y - (hpad / 2)) / scale; + float x1 = (obj.rect.x + obj.rect.width - (wpad / 2)) / scale; + float y1 = (obj.rect.y + obj.rect.height - (hpad / 2)) / scale; + + // clip + x0 = std::floor(std::max(std::min(x0, (float)(width - 1)), 1.f)); + y0 = std::floor(std::max(std::min(y0, (float)(height - 1)), 1.f)); + x1 = std::ceil(std::max(std::min(x1, (float)(width - 1)), 1.f)); + y1 = std::ceil(std::max(std::min(y1, (float)(height - 1)), 1.f)); + + + obj.rect.x = x0; + obj.rect.y = y0; + obj.rect.width = x1 - x0; + obj.rect.height = y1 - y0; + if (obj.rect.width > 10.f && obj.rect.height > 10.f) { + objects.push_back(obj); + } + } + + // sort objects by area + struct + { + bool operator()(const Object& a, const Object& b) const + { + return a.rect.area() > b.rect.area(); + } + } objects_area_greater; + std::sort(objects.begin(), objects.end(), objects_area_greater); + + return 0; +} + +int Yolo::draw(cv::Mat& rgb, const std::vector& objects) +{ + static const char* class_names[] = { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", + "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", + "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", + "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", + "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", + "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", + "hair drier", "toothbrush" + }; + + static const unsigned char colors[19][3] = { + { 54, 67, 244}, + { 99, 30, 233}, + {176, 39, 156}, + {183, 58, 103}, + {181, 81, 63}, + {243, 150, 33}, + {244, 169, 3}, + {212, 188, 0}, + {136, 150, 0}, + { 80, 175, 76}, + { 74, 195, 139}, + { 57, 220, 205}, + { 59, 235, 255}, + { 7, 193, 255}, + { 0, 152, 255}, + { 34, 87, 255}, + { 72, 85, 121}, + {158, 158, 158}, + {139, 125, 96} + }; + + int color_index = 0; + + for (int i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + +// fprintf(stderr, "%d = %.5f at %.2f %.2f %.2f x %.2f\n", obj.label, obj.prob, +// obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height); + + const unsigned char* color = colors[color_index % 19]; + color_index++; + + cv::Scalar cc(color[0], color[1], color[2]); + + cv::rectangle(rgb, obj.rect, cc, 2); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > rgb.cols) + x = rgb.cols - label_size.width; + + cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), cc, -1); + + cv::Scalar textcc = (color[0] + color[1] + color[2] >= 381) ? cv::Scalar(0, 0, 0) : cv::Scalar(255, 255, 255); + + cv::putText(rgb, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, textcc, 1); + } + + return 0; +} diff --git a/deploy/NCNN/Android/app/src/main/jni/yolo.h b/deploy/NCNN/Android/app/src/main/jni/yolo.h new file mode 100644 index 00000000..d00977b2 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/yolo.h @@ -0,0 +1,52 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef YOLO_H +#define YOLO_H + +#include + +#include + +struct Object +{ + cv::Rect_ rect; + int label; + float prob; +}; + +class Yolo +{ +public: + Yolo(); + + int load(const char* modeltype, const int *target_size, const float* mean_vals, const float* norm_vals, bool use_gpu = false); + + int load(AAssetManager* mgr, const char* modeltype, const int *target_size, const float* mean_vals, const float* norm_vals, bool use_gpu = false); + + int detect(const cv::Mat& rgb, std::vector& objects, float prob_threshold = 0.25f, float nms_threshold = 0.45f); + + int draw(cv::Mat& rgb, const std::vector& objects); + +private: + ncnn::Net yolo; + int net_h; + int net_w; + float mean_vals[3]; + float norm_vals[3]; + ncnn::UnlockedPoolAllocator blob_pool_allocator; + ncnn::PoolAllocator workspace_pool_allocator; +}; + +#endif // NANODET_H diff --git a/deploy/NCNN/Android/app/src/main/jni/yolov6ncnn.cpp b/deploy/NCNN/Android/app/src/main/jni/yolov6ncnn.cpp new file mode 100644 index 00000000..88510783 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/jni/yolov6ncnn.cpp @@ -0,0 +1,281 @@ +// Tencent is pleased to support the open source community by making ncnn available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// https://opensource.org/licenses/BSD-3-Clause +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include + +#include "yolo.h" + +#include "ndkcamera.h" + +#include +#include + +#if __ARM_NEON +#include +#endif // __ARM_NEON + +static int draw_unsupported(cv::Mat& rgb) +{ + const char text[] = "unsupported"; + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 1.0, 1, &baseLine); + + int y = (rgb.rows - label_size.height) / 2; + int x = (rgb.cols - label_size.width) / 2; + + cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(rgb, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 0, 0)); + + return 0; +} + +static int draw_fps(cv::Mat& rgb) +{ + // resolve moving average + float avg_fps = 0.f; + { + static double t0 = 0.f; + static float fps_history[10] = {0.f}; + + double t1 = ncnn::get_current_time(); + if (t0 == 0.f) + { + t0 = t1; + return 0; + } + + float fps = 1000.f / (t1 - t0); + t0 = t1; + + for (int i = 9; i >= 1; i--) + { + fps_history[i] = fps_history[i - 1]; + } + fps_history[0] = fps; + + if (fps_history[9] == 0.f) + { + return 0; + } + + for (int i = 0; i < 10; i++) + { + avg_fps += fps_history[i]; + } + avg_fps /= 10.f; + } + + char text[32]; + sprintf(text, "FPS=%.2f", avg_fps); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int y = 0; + int x = rgb.cols - label_size.width; + + cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cv::Scalar(255, 255, 255), -1); + + cv::putText(rgb, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); + + return 0; +} + +static Yolo* g_yolo = 0; +static ncnn::Mutex lock; + +class MyNdkCamera : public NdkCameraWindow +{ +public: + virtual void on_image_render(cv::Mat& rgb) const; +}; + +void MyNdkCamera::on_image_render(cv::Mat& rgb) const +{ + // nanodet + { + ncnn::MutexLockGuard g(lock); + + if (g_yolo) + { + std::vector objects; + + g_yolo->detect(rgb, objects); + + g_yolo->draw(rgb, objects); + } + else + { + draw_unsupported(rgb); + } + } + + draw_fps(rgb); +} + +static MyNdkCamera* g_camera = 0; + +extern "C" { + +JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) +{ + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad"); + + g_camera = new MyNdkCamera; + + return JNI_VERSION_1_4; +} + +JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved) +{ + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnUnload"); + + { + ncnn::MutexLockGuard g(lock); + + delete g_yolo; + g_yolo = 0; + } + + delete g_camera; + g_camera = 0; +} + +// public native boolean loadModel(AssetManager mgr, int modelid, int cpugpu); +JNIEXPORT jboolean JNICALL Java_com_tencent_yolov6ncnn_Yolov6Ncnn_loadModel(JNIEnv* env, jobject thiz, jobject assetManager, jint modelid, jint cpugpu) +{ + if (modelid < 0 || modelid > 4 || cpugpu < 0 || cpugpu > 1) + { + return JNI_FALSE; + } + + AAssetManager* mgr = AAssetManager_fromJava(env, assetManager); + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "loadModel %p", mgr); + + const char* modeltypes[] = + { + "lite-s", + "lite-m", + "lite-l0", + "lite-l1", + "lite-l2", + }; + + const int target_sizes[][2] = + { + {320, 320}, + {320, 320}, + {320, 320}, + {320, 192}, + {224, 128} + }; + + const float mean_vals[][3] = + { + {0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f}, + {0.f, 0.f, 0.f} + }; + + const float norm_vals[][3] = + { + { 1 / 255.f, 1 / 255.f, 1 / 255.f }, + { 1 / 255.f, 1 / 255.f, 1 / 255.f }, + { 1 / 255.f, 1 / 255.f, 1 / 255.f }, + { 1 / 255.f, 1 / 255.f, 1 / 255.f }, + { 1 / 255.f, 1 / 255.f, 1 / 255.f } + }; + + const char* modeltype = modeltypes[(int)modelid]; + const int *target_size = target_sizes[(int)modelid]; + bool use_gpu = (int)cpugpu == 1; + + // reload + { + ncnn::MutexLockGuard g(lock); + + if (use_gpu && ncnn::get_gpu_count() == 0) + { + // no gpu + delete g_yolo; + g_yolo = 0; + } + else + { + if (!g_yolo) + g_yolo = new Yolo; + g_yolo->load(mgr, modeltype, target_size, mean_vals[(int)modelid], norm_vals[(int)modelid], use_gpu); + } + } + + return JNI_TRUE; +} + +// public native boolean openCamera(int facing); +JNIEXPORT jboolean JNICALL Java_com_tencent_yolov6ncnn_Yolov6Ncnn_openCamera(JNIEnv* env, jobject thiz, jint facing) +{ + if (facing < 0 || facing > 1) + return JNI_FALSE; + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "openCamera %d", facing); + + g_camera->open((int)facing); + + return JNI_TRUE; +} + +// public native boolean closeCamera(); +JNIEXPORT jboolean JNICALL Java_com_tencent_yolov6ncnn_Yolov6Ncnn_closeCamera(JNIEnv* env, jobject thiz) +{ + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "closeCamera"); + + g_camera->close(); + + return JNI_TRUE; +} + +// public native boolean setOutputWindow(Surface surface); +JNIEXPORT jboolean JNICALL Java_com_tencent_yolov6ncnn_Yolov6Ncnn_setOutputWindow(JNIEnv* env, jobject thiz, jobject surface) +{ + ANativeWindow* win = ANativeWindow_fromSurface(env, surface); + + __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win); + + g_camera->set_window(win); + + return JNI_TRUE; +} + +} diff --git a/deploy/NCNN/Android/app/src/main/res/layout/main.xml b/deploy/NCNN/Android/app/src/main/res/layout/main.xml new file mode 100644 index 00000000..e8a45125 --- /dev/null +++ b/deploy/NCNN/Android/app/src/main/res/layout/main.xml @@ -0,0 +1,60 @@ + + + + + +