Skip to content

Commit 3ee3f41

Browse files
dkurtdkurt
committed
Added Halide backend support for deep learning layers
1 parent a4cc801 commit 3ee3f41

25 files changed

+2185
-29
lines changed

‎modules/dnn/CMakeLists.txt

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,34 @@ if(NOT PROTOBUF_FOUND)
5454
)
5555
endif()
5656

57-
ocv_source_group("Src\\protobuf" FILES ${PROTOBUF_SRCS} ${PROTOBUF_HDRS})
58-
ocv_module_include_directories(include ${PROTOBUF_INCLUDE_DIR})
57+
set(DNN_INCLUDE_DIRS
58+
include
59+
${PROTOBUF_INCLUDE_DIR}
60+
)
61+
62+
set(DNN_LIBRARIES
63+
${PROTOBUF_LIBRARIES}
64+
${LAPACK_LIBRARIES}
65+
)
5966

67+
# ----------------------------------------------------------------------------
68+
# Halide library
69+
# ----------------------------------------------------------------------------
70+
if(HALIDE_FOUND)
71+
set(DNN_INCLUDE_DIRS
72+
${DNN_INCLUDE_DIRS}
73+
${HALIDE_INCLUDE_DIRS}
74+
)
75+
set(DNN_LIBRARIES
76+
${DNN_LIBRARIES}
77+
${HALIDE_LIBRARIES}
78+
)
79+
endif()
80+
81+
ocv_source_group("Src\\protobuf" FILES ${PROTOBUF_SRCS} ${PROTOBUF_HDRS})
82+
ocv_module_include_directories(${DNN_INCLUDE_DIRS})
6083
ocv_glob_module_sources(${PROTOBUF_SRCS} ${PROTOBUF_HDRS} ${CBLAS_H_PROXY_PATH})
61-
ocv_create_module(${PROTOBUF_LIBRARIES} ${LAPACK_LIBRARIES})
84+
ocv_create_module(${DNN_LIBRARIES})
6285
ocv_add_samples()
6386
ocv_add_accuracy_tests()
6487
ocv_add_perf_tests()

‎modules/dnn/include/opencv2/dnn/dnn.hpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,15 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
5555

5656
typedef std::vector<int> MatShape;
5757

58+
/**
59+
* @brief Enum of computation backends supported by layers.
60+
*/
61+
enum Backend
62+
{
63+
DEFAULT,
64+
HALIDE
65+
};
66+
5867
/** @brief Initialize dnn module and built-in layers.
5968
*
6069
* This function automatically called on most of OpenCV builds,
@@ -131,6 +140,32 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
131140
*/
132141
virtual int outputNameToIndex(String outputName);
133142

143+
/**
144+
* @brief Ask layer if it support specific backend for doing computations.
145+
* @param[in] backendId computation backend identifier.
146+
*/
147+
virtual bool supportBackend(int backendId);
148+
149+
/**
150+
* @brief Returns pointers to Halide functions.
151+
* @param[in] inputs blobs that will be used in forward calls.
152+
* @param[in] outputs blobs that will be used in forward calls.
153+
*
154+
* This functions are Halide pipeline in order from bottom to top.
155+
*/
156+
virtual std::vector<void*> initHalide(const std::vector<Mat*> &inputs,
157+
const std::vector<Mat> &outputs);
158+
159+
/**
160+
* @brief Implement layers fusing.
161+
* @param[in] node pointer to backend node of bottom layer.
162+
* @param[in] backendId specific backend identifier.
163+
*
164+
* Actual for graph-based backends. If layer attached successfully,
165+
* returns pointer to fused node.
166+
*/
167+
virtual void* tryAttach(void* node, int backendId);
168+
134169
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
135170
const int requiredOutputs,
136171
std::vector<MatShape> &outputs,
@@ -251,6 +286,18 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
251286
/** @overload */
252287
void forwardOpt(const std::vector<LayerId> &toLayers);
253288

289+
/**
290+
* @brief Schedule layers that support Halide backend.
291+
* @param[in] path to YAML file with scheduling directives.
292+
*/
293+
void applyHalideScheduler(const std::string& configFile);
294+
295+
/**
296+
* @brief Ask network to use specific computation backend where it supported.
297+
* @param[in] backend identifier.
298+
*/
299+
void setPreferableBackend(int backendId);
300+
254301
/** @brief Sets the new value for the layer output blob
255302
* @param outputName descriptor of the updating layer output blob.
256303
* @param blob new blob.

‎modules/dnn/perf/perf_halide_net.cpp

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
//
5+
// Copyright (C) 2017, Intel Corporation, all rights reserved.
6+
// Third party copyrights are property of their respective owners.
7+
8+
namespace cvtest
9+
{
10+
11+
using namespace cv;
12+
using namespace dnn;
13+
14+
#define PERF_HALIDE_NET(weights, proto, scheduler, inWidth, inHeight, \
15+
outputLayer, frameworkName) \
16+
Mat input(inHeight, inWidth, CV_32FC3); \
17+
randu(input, 0.0f, 1.0f); \
18+
\
19+
Net net; \
20+
std::string framework = frameworkName; \
21+
if (framework == "caffe") \
22+
{ \
23+
net = cv::dnn::readNetFromCaffe(proto, weights); \
24+
net.setBlob(".data", cv::dnn::blobFromImage(input, 1.0, false)); \
25+
} \
26+
else if (framework == "torch") \
27+
{ \
28+
net = cv::dnn::readNetFromTorch(weights); \
29+
net.setBlob("", cv::dnn::blobFromImage(input, 1.0, false)); \
30+
} \
31+
else if (framework == "tensorflow") \
32+
{ \
33+
cv::Ptr<cv::dnn::Importer> importer = \
34+
cv::dnn::createTensorflowImporter(weights); \
35+
importer->populateNet(net); \
36+
importer.release(); \
37+
net.setBlob("", cv::dnn::blobFromImage(input, 1.0, false)); \
38+
} \
39+
net.setPreferableBackend(HALIDE); \
40+
net.allocate(); \
41+
net.applyHalideScheduler(scheduler); \
42+
net.forward(net.getLayerId(outputLayer)); \
43+
\
44+
TEST_CYCLE_N(10) \
45+
{ \
46+
net.forward(net.getLayerId(outputLayer)); \
47+
} \
48+
SANITY_CHECK_NOTHING();
49+
50+
inline std::string getOpenCVExtraDir()
51+
{
52+
return "/home/dkurtaev/opencv_extra/testdata/";
53+
}
54+
55+
PERF_TEST(AlexNet, HalidePerfTest)
56+
{
57+
PERF_HALIDE_NET(getOpenCVExtraDir() + "/dnn/bvlc_alexnet.caffemodel",
58+
getOpenCVExtraDir() + "/dnn/bvlc_alexnet.prototxt",
59+
getOpenCVExtraDir() + "/dnn/halide_scheduler_alexnet.yml",
60+
227, 227, "prob", "caffe");
61+
}
62+
63+
PERF_TEST(ResNet50, HalidePerfTest)
64+
{
65+
PERF_HALIDE_NET(getOpenCVExtraDir() + "/dnn/ResNet-50-model.caffemodel",
66+
getOpenCVExtraDir() + "/dnn/ResNet-50-deploy.prototxt",
67+
getOpenCVExtraDir() + "/dnn/halide_scheduler_resnet_50.yml",
68+
224, 224, "prob", "caffe");
69+
}
70+
71+
PERF_TEST(SqueezeNet_v1_1, HalidePerfTest)
72+
{
73+
PERF_HALIDE_NET(getOpenCVExtraDir() + "/dnn/squeezenet_v1_1.caffemodel",
74+
getOpenCVExtraDir() + "/dnn/squeezenet_v1_1.prototxt",
75+
getOpenCVExtraDir() + "/dnn/halide_scheduler_squeezenet_v1_1.yml",
76+
227, 227, "prob", "caffe");
77+
}
78+
79+
PERF_TEST(Inception_5h, HalidePerfTest)
80+
{
81+
PERF_HALIDE_NET(getOpenCVExtraDir() + "/dnn/tensorflow_inception_graph.pb", "",
82+
getOpenCVExtraDir() + "/dnn/halide_scheduler_inception_5h.yml",
83+
224, 224, "softmax2", "tensorflow");
84+
}
85+
86+
PERF_TEST(ENet, HalidePerfTest)
87+
{
88+
PERF_HALIDE_NET("/home/dkurtaev/Documents/enet/model-best.net", "",
89+
getOpenCVExtraDir() + "/dnn/halide_scheduler_enet.yml",
90+
512, 256, "l367_Deconvolution", "torch");
91+
}
92+
93+
} // namespace cvtest

‎modules/dnn/samples/squeezenet_halide.cpp

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
//
5+
// Copyright (C) 2017, Intel Corporation, all rights reserved.
6+
// Third party copyrights are property of their respective owners.
7+
8+
// Sample of using Halide backend in OpenCV deep learning module.
9+
// Based on dnn/samples/caffe_googlenet.cpp.
10+
11+
#include <opencv2/dnn.hpp>
12+
#include <opencv2/imgproc.hpp>
13+
#include <opencv2/highgui.hpp>
14+
using namespace cv;
15+
using namespace cv::dnn;
16+
17+
#include <fstream>
18+
#include <iostream>
19+
#include <cstdlib>
20+
21+
/* Find best class for the blob (i. e. class with maximal probability) */
22+
void getMaxClass(const Mat &probBlob, int *classId, double *classProb)
23+
{
24+
Mat probMat = probBlob.reshape(1, 1); //reshape the blob to 1x1000 matrix
25+
Point classNumber;
26+
27+
minMaxLoc(probMat, NULL, classProb, NULL, &classNumber);
28+
*classId = classNumber.x;
29+
}
30+
31+
std::vector<std::string> readClassNames(const char *filename = "synset_words.txt")
32+
{
33+
std::vector<std::string> classNames;
34+
35+
std::ifstream fp(filename);
36+
if (!fp.is_open())
37+
{
38+
std::cerr << "File with classes labels not found: " << filename << std::endl;
39+
exit(-1);
40+
}
41+
42+
std::string name;
43+
while (!fp.eof())
44+
{
45+
std::getline(fp, name);
46+
if (name.length())
47+
classNames.push_back( name.substr(name.find(' ')+1) );
48+
}
49+
50+
fp.close();
51+
return classNames;
52+
}
53+
54+
int main(int argc, char **argv)
55+
{
56+
initModule(); // Required if OpenCV is built as static libs.
57+
58+
std::string modelTxt = "train_val.prototxt";
59+
std::string modelBin = "squeezenet_v1.1.caffemodel";
60+
std::string scheduler = "halide_scheduler_squeezenet_v1_1.yml";
61+
std::string imageFile = (argc > 1) ? argv[1] : "space_shuttle.jpg";
62+
63+
//! [Read and initialize network]
64+
Net net = dnn::readNetFromCaffe(modelTxt, modelBin);
65+
//! [Read and initialize network]
66+
67+
//! [Check that network was read successfully]
68+
if (net.empty())
69+
{
70+
std::cerr << "Can't load network by using the following files: " << std::endl;
71+
std::cerr << "prototxt: " << modelTxt << std::endl;
72+
std::cerr << "caffemodel: " << modelBin << std::endl;
73+
std::cerr << "SqueezeNet v1.1 can be downloaded from:" << std::endl;
74+
std::cerr << "https://github.com/DeepScale/SqueezeNet/tree/master/SqueezeNet_v1.1" << std::endl;
75+
exit(-1);
76+
}
77+
//! [Check that network was read successfully]
78+
79+
//! [Prepare blob]
80+
Mat img = imread(imageFile);
81+
if (img.empty())
82+
{
83+
std::cerr << "Can't read image from the file: " << imageFile << std::endl;
84+
exit(-1);
85+
}
86+
if (img.channels() != 3)
87+
{
88+
std::cerr << "Image " << imageFile << " isn't 3-channel" << std::endl;
89+
exit(-1);
90+
}
91+
92+
resize(img, img, Size(227, 227)); // SqueezeNet v1.1 predict class by 3x227x227 input image.
93+
Mat inputBlob = blobFromImage(img, 1.0, false); // Convert Mat to 4-dimensional batch.
94+
//! [Prepare blob]
95+
96+
//! [Set input blob]
97+
net.setBlob(".data", inputBlob); // Set the network input.
98+
//! [Set input blob]
99+
100+
//! [Enable Halide backend]
101+
net.setPreferableBackend(HALIDE); // Tell engine to use Halide where it possible.
102+
//! [Enable Halide backend]
103+
104+
//! [Initialize Halide backend]
105+
net.allocate(); // Initialize internal state of network.
106+
//! [Initialize Halide backend]
107+
108+
//! [Schedule network]
109+
net.applyHalideScheduler(scheduler); // Apply scheduling directives.
110+
//! [Schedule network]
111+
112+
//! [Make forward pass]
113+
net.forward(); // Compute output.
114+
//! [Make forward pass]
115+
116+
//! [Gather output]
117+
Mat prob = net.getBlob("prob"); // Gather output of "prob" layer.
118+
119+
int classId;
120+
double classProb;
121+
getMaxClass(prob, &classId, &classProb); // Find the best class.
122+
//! [Gather output]
123+
124+
//! [Print results]
125+
std::vector<std::string> classNames = readClassNames();
126+
std::cout << "Best class: #" << classId << " '" << classNames.at(classId) << "'" << std::endl;
127+
std::cout << "Probability: " << classProb * 100 << "%" << std::endl;
128+
//! [Print results]
129+
130+
return 0;
131+
} //main

0 commit comments

Comments
 (0)