Skip to content

Commit 1070e98

Browse files
dkurtdkurt
committed
Added Halide backend support for deep learning layers
1 parent 27f6d4e commit 1070e98

26 files changed

+2810
-11
lines changed

‎modules/dnn/include/opencv2/dnn/dnn.hpp‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,23 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
5555

5656
typedef std::vector<int> MatShape;
5757

58+
/**
59+
* @brief Enum of computation backends supported by layers.
60+
*/
61+
enum Backend
62+
{
63+
DEFAULT,
64+
HALIDE
65+
};
66+
67+
/**
68+
* @brief Compilation targets of Halide code.
69+
*/
70+
enum HalideTarget
71+
{
72+
HALIDE_CPU
73+
};
74+
5875
/** @brief Initialize dnn module and built-in layers.
5976
*
6077
* This function automatically called on most of OpenCV builds,
@@ -131,6 +148,47 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
131148
*/
132149
virtual int outputNameToIndex(String outputName);
133150

151+
/**
152+
* @brief Ask layer if it support specific backend for doing computations.
153+
* @param[in] backendId computation backend identifier.
154+
* @see Backend
155+
*/
156+
virtual bool supportBackend(int backendId);
157+
158+
/**
159+
* @brief Returns pointers to Halide functions.
160+
* @param[in] inputs Pointers to input Halide buffers.
161+
*
162+
* Input buffers should be exactly the same that will be used in forward invocations.
163+
* Despite we can use Halide::ImageParam based on input shape only,
164+
* it helps prevent some memory management issues (if something wrong,
165+
* Halide tests will be failed).
166+
*/
167+
virtual std::vector<void*> initHalide(const std::vector<void*> &inputs);
168+
169+
/**
170+
* @brief Automatic Halide scheduling based on layer hyper-parameters.
171+
* @param[in] funcs Halide functions.
172+
* @param[in] inputs Blobs that will be used in forward invocations.
173+
* @param[in] outputs Blobs that will be used in forward invocations.
174+
*
175+
* Layer don't use own Halide::Func members because we can have applied
176+
* layers fusing. In this way the fused function should be scheduled.
177+
*/
178+
virtual void applyHalideScheduler(std::vector<void*>& funcs,
179+
const std::vector<Mat*> &inputs,
180+
const std::vector<Mat> &outputs) const;
181+
182+
/**
183+
* @brief Implement layers fusing.
184+
* @param[in] node pointer to backend node of bottom layer.
185+
* @param[in] backendId specific backend identifier.
186+
*
187+
* Actual for graph-based backends. If layer attached successfully,
188+
* returns pointer to fused node.
189+
*/
190+
virtual void* tryAttach(void* node, int backendId);
191+
134192
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
135193
const int requiredOutputs,
136194
std::vector<MatShape> &outputs,
@@ -251,6 +309,25 @@ namespace dnn //! This namespace is used for dnn module functionlaity.
251309
/** @overload */
252310
void forwardOpt(const std::vector<LayerId> &toLayers);
253311

312+
/**
313+
* @brief Compile Halide layers.
314+
* @param[in] targetId Compilation target of Halide code.
315+
* @param[in] scheduler Path to YAML file with scheduling directives.
316+
* @see HalideTarget
317+
*
318+
* Schedule layers that support Halide backend. Then compile them for
319+
* specific target. For layers that not represented in scheduling file
320+
* or if no manual scheduling used at all, automatic scheduling will be applied.
321+
*/
322+
void compileHalide(int targetId = HALIDE_CPU, const std::string& scheduler = "");
323+
324+
/**
325+
* @brief Ask network to use specific computation backend where it supported.
326+
* @param[in] backendId backend identifier.
327+
* @see Backend
328+
*/
329+
void setPreferableBackend(int backendId);
330+
254331
/** @brief Sets the new value for the layer output blob
255332
* @param outputName descriptor of the updating layer output blob.
256333
* @param blob new blob.

‎modules/dnn/perf/perf_halide_net.cpp‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
//
5+
// Copyright (C) 2017, Intel Corporation, all rights reserved.
6+
// Third party copyrights are property of their respective owners.
7+
8+
namespace cvtest
9+
{
10+
11+
#ifdef HAVE_HALIDE
12+
using namespace cv;
13+
using namespace dnn;
14+
15+
static void loadNet(const std::string& weights, const std::string& proto,
16+
const std::string& scheduler, int inWidth, int inHeight,
17+
const std::string& outputLayer, const std::string& framework,
18+
Net* net, int* outputLayerId)
19+
{
20+
Mat input(inHeight, inWidth, CV_32FC3);
21+
randu(input, 0.0f, 1.0f);
22+
23+
if (framework == "caffe")
24+
{
25+
*net = cv::dnn::readNetFromCaffe(proto, weights);
26+
}
27+
else if (framework == "torch")
28+
{
29+
*net = cv::dnn::readNetFromTorch(weights);
30+
}
31+
else if (framework == "tensorflow")
32+
{
33+
*net = cv::dnn::readNetFromTensorflow(weights);
34+
}
35+
else
36+
CV_Error(Error::StsNotImplemented, "Unknown framework " + framework);
37+
38+
net->setBlob("", cv::dnn::blobFromImage(input, 1.0, false));
39+
net->setPreferableBackend(HALIDE);
40+
net->compileHalide(HALIDE_CPU, scheduler);
41+
*outputLayerId = net->getLayerId(outputLayer);
42+
net->forward(*outputLayerId);
43+
}
44+
45+
PERF_TEST(GoogLeNet, HalidePerfTest)
46+
{
47+
Net net;
48+
int outputLayerId;
49+
loadNet(findDataFile("dnn/bvlc_googlenet.caffemodel"),
50+
findDataFile("dnn/bvlc_googlenet.prototxt"),
51+
"", 227, 227, "prob", "caffe", &net, &outputLayerId);
52+
53+
TEST_CYCLE_N(10)
54+
{
55+
net.forward(outputLayerId);
56+
}
57+
SANITY_CHECK_NOTHING();
58+
}
59+
60+
PERF_TEST(AlexNet, HalidePerfTest)
61+
{
62+
Net net;
63+
int outputLayerId;
64+
loadNet(findDataFile("dnn/bvlc_alexnet.caffemodel"),
65+
findDataFile("dnn/bvlc_alexnet.prototxt"),
66+
findDataFile("dnn/halide_scheduler_alexnet.yml"),
67+
227, 227, "prob", "caffe", &net, &outputLayerId);
68+
69+
TEST_CYCLE_N(10)
70+
{
71+
net.forward(outputLayerId);
72+
}
73+
SANITY_CHECK_NOTHING();
74+
}
75+
76+
// PERF_TEST(ResNet50, HalidePerfTest)
77+
// {
78+
// Net net;
79+
// int outputLayerId;
80+
// loadNet(findDataFile("dnn/ResNet-50-model.caffemodel"),
81+
// findDataFile("dnn/ResNet-50-deploy.prototxt"),
82+
// findDataFile("dnn/halide_scheduler_resnet_50.yml"),
83+
// 224, 224, "prob", "caffe", &net, &outputLayerId);
84+
//
85+
// TEST_CYCLE_N(10)
86+
// {
87+
// net.forward(outputLayerId);
88+
// }
89+
// SANITY_CHECK_NOTHING();
90+
// }
91+
92+
// PERF_TEST(SqueezeNet_v1_1, HalidePerfTest)
93+
// {
94+
// Net net;
95+
// int outputLayerId;
96+
// loadNet(findDataFile("dnn/squeezenet_v1_1.caffemodel"),
97+
// findDataFile("dnn/squeezenet_v1_1.prototxt"),
98+
// findDataFile("dnn/halide_scheduler_squeezenet_v1_1.yml"),
99+
// 227, 227, "prob", "caffe", &net, &outputLayerId);
100+
//
101+
// TEST_CYCLE_N(10)
102+
// {
103+
// net.forward(outputLayerId);
104+
// }
105+
// SANITY_CHECK_NOTHING();
106+
// }
107+
108+
PERF_TEST(Inception_5h, HalidePerfTest)
109+
{
110+
Net net;
111+
int outputLayerId;
112+
loadNet(findDataFile("dnn/tensorflow_inception_graph.pb"), "",
113+
findDataFile("dnn/halide_scheduler_inception_5h.yml"),
114+
224, 224, "softmax2", "tensorflow", &net, &outputLayerId);
115+
116+
TEST_CYCLE_N(10)
117+
{
118+
net.forward(outputLayerId);
119+
}
120+
SANITY_CHECK_NOTHING();
121+
}
122+
123+
PERF_TEST(ENet, HalidePerfTest)
124+
{
125+
Net net;
126+
int outputLayerId;
127+
loadNet(findDataFile("dnn/Enet-model-best.net"), "",
128+
findDataFile("dnn/halide_scheduler_enet.yml"),
129+
512, 256, "l367_Deconvolution", "torch", &net, &outputLayerId);
130+
131+
TEST_CYCLE_N(10)
132+
{
133+
net.forward(outputLayerId);
134+
}
135+
SANITY_CHECK_NOTHING();
136+
}
137+
#endif // HAVE_HALIDE
138+
139+
} // namespace cvtest

‎modules/dnn/perf/perf_main.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,12 @@
11
#include "perf_precomp.hpp"
22

3-
CV_PERF_TEST_MAIN(dnn)
3+
static const char* extraTestDataPath =
4+
#ifdef WINRT
5+
NULL;
6+
#else
7+
getenv("OPENCV_DNN_TEST_DATA_PATH");
8+
#endif
9+
10+
CV_PERF_TEST_MAIN(dnn,
11+
extraTestDataPath ? (void)cvtest::addDataSearchPath(extraTestDataPath) : (void)0
12+
)

‎modules/dnn/samples/squeezenet_halide.cpp‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
//
5+
// Copyright (C) 2017, Intel Corporation, all rights reserved.
6+
// Third party copyrights are property of their respective owners.
7+
8+
// Sample of using Halide backend in OpenCV deep learning module.
9+
// Based on dnn/samples/caffe_googlenet.cpp.
10+
11+
#include <opencv2/dnn.hpp>
12+
#include <opencv2/imgproc.hpp>
13+
#include <opencv2/highgui.hpp>
14+
using namespace cv;
15+
using namespace cv::dnn;
16+
17+
#include <fstream>
18+
#include <iostream>
19+
#include <cstdlib>
20+
21+
/* Find best class for the blob (i. e. class with maximal probability) */
22+
void getMaxClass(const Mat &probBlob, int *classId, double *classProb)
23+
{
24+
Mat probMat = probBlob.reshape(1, 1); //reshape the blob to 1x1000 matrix
25+
Point classNumber;
26+
27+
minMaxLoc(probMat, NULL, classProb, NULL, &classNumber);
28+
*classId = classNumber.x;
29+
}
30+
31+
std::vector<std::string> readClassNames(const char *filename = "synset_words.txt")
32+
{
33+
std::vector<std::string> classNames;
34+
35+
std::ifstream fp(filename);
36+
if (!fp.is_open())
37+
{
38+
std::cerr << "File with classes labels not found: " << filename << std::endl;
39+
exit(-1);
40+
}
41+
42+
std::string name;
43+
while (!fp.eof())
44+
{
45+
std::getline(fp, name);
46+
if (name.length())
47+
classNames.push_back( name.substr(name.find(' ')+1) );
48+
}
49+
50+
fp.close();
51+
return classNames;
52+
}
53+
54+
int main(int argc, char **argv)
55+
{
56+
initModule(); // Required if OpenCV is built as static libs.
57+
58+
std::string modelTxt = "train_val.prototxt";
59+
std::string modelBin = "squeezenet_v1.1.caffemodel";
60+
std::string scheduler = "halide_scheduler_squeezenet_v1_1.yml";
61+
std::string imageFile = (argc > 1) ? argv[1] : "space_shuttle.jpg";
62+
63+
//! [Read and initialize network]
64+
Net net = dnn::readNetFromCaffe(modelTxt, modelBin);
65+
//! [Read and initialize network]
66+
67+
//! [Check that network was read successfully]
68+
if (net.empty())
69+
{
70+
std::cerr << "Can't load network by using the following files: " << std::endl;
71+
std::cerr << "prototxt: " << modelTxt << std::endl;
72+
std::cerr << "caffemodel: " << modelBin << std::endl;
73+
std::cerr << "SqueezeNet v1.1 can be downloaded from:" << std::endl;
74+
std::cerr << "https://github.com/DeepScale/SqueezeNet/tree/master/SqueezeNet_v1.1" << std::endl;
75+
exit(-1);
76+
}
77+
//! [Check that network was read successfully]
78+
79+
//! [Prepare blob]
80+
Mat img = imread(imageFile);
81+
if (img.empty())
82+
{
83+
std::cerr << "Can't read image from the file: " << imageFile << std::endl;
84+
exit(-1);
85+
}
86+
if (img.channels() != 3)
87+
{
88+
std::cerr << "Image " << imageFile << " isn't 3-channel" << std::endl;
89+
exit(-1);
90+
}
91+
92+
resize(img, img, Size(227, 227)); // SqueezeNet v1.1 predict class by 3x227x227 input image.
93+
Mat inputBlob = blobFromImage(img, 1.0, false); // Convert Mat to 4-dimensional batch.
94+
//! [Prepare blob]
95+
96+
//! [Set input blob]
97+
net.setBlob("", inputBlob); // Set the network input.
98+
//! [Set input blob]
99+
100+
//! [Enable Halide backend]
101+
net.setPreferableBackend(HALIDE); // Tell engine to use Halide where it possible.
102+
//! [Enable Halide backend]
103+
104+
//! [Compile Halide pipeline]
105+
net.compileHalide(); // Compile Halide pipeline.
106+
//! [Compile Halide pipeline]
107+
108+
//! [Make forward pass]
109+
net.forward(); // Compute output.
110+
//! [Make forward pass]
111+
112+
//! [Gather output]
113+
Mat prob = net.getBlob("prob"); // Gather output of "prob" layer.
114+
115+
int classId;
116+
double classProb;
117+
getMaxClass(prob, &classId, &classProb); // Find the best class.
118+
//! [Gather output]
119+
120+
//! [Print results]
121+
std::vector<std::string> classNames = readClassNames();
122+
std::cout << "Best class: #" << classId << " '" << classNames.at(classId) << "'" << std::endl;
123+
std::cout << "Probability: " << classProb * 100 << "%" << std::endl;
124+
//! [Print results]
125+
126+
return 0;
127+
} //main

0 commit comments

Comments
 (0)