Skip to content

Commit

Permalink
Add Tracing to Perf Analyzer GRPC/HTTP Client (triton-inference-serve…
Browse files Browse the repository at this point in the history 
…r#136)

* Added tracing for GRPC/HTTPperf_analyzer clients

* Cleaned up options

* Fixed flag

* Wording
  • Loading branch information
@dyastremsky
dyastremsky authored Jul 22, 2022
1 parent 1173e95 commit f4c75d8
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 10 deletions.
18 changes: 11 additions & 7 deletions src/c++/perf_analyzer/client_backend/client_backend.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,17 @@ Error
ClientBackendFactory::Create(
const BackendKind kind, const std::string& url, const ProtocolType protocol,
const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const GrpcCompressionAlgorithm compression_algorithm,
std::shared_ptr<Headers> http_headers,
const std::string& triton_server_path,
const std::string& model_repository_path, const std::string& memory_type,
const bool verbose, std::shared_ptr<ClientBackendFactory>* factory)
{
factory->reset(new ClientBackendFactory(
kind, url, protocol, ssl_options, compression_algorithm, http_headers,
triton_server_path, model_repository_path, memory_type, verbose));
kind, url, protocol, ssl_options, trace_options, compression_algorithm,
http_headers, triton_server_path, model_repository_path, memory_type,
verbose));
return Error::Success;
}

Expand All @@ -132,9 +134,9 @@ ClientBackendFactory::CreateClientBackend(
std::unique_ptr<ClientBackend>* client_backend)
{
RETURN_IF_CB_ERROR(ClientBackend::Create(
kind_, url_, protocol_, ssl_options_, compression_algorithm_,
http_headers_, verbose_, triton_server_path, model_repository_path_,
memory_type_, client_backend));
kind_, url_, protocol_, ssl_options_, trace_options_,
compression_algorithm_, http_headers_, verbose_, triton_server_path,
model_repository_path_, memory_type_, client_backend));
return Error::Success;
}

Expand All @@ -145,6 +147,7 @@ Error
ClientBackend::Create(
const BackendKind kind, const std::string& url, const ProtocolType protocol,
const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const GrpcCompressionAlgorithm compression_algorithm,
std::shared_ptr<Headers> http_headers, const bool verbose,
const std::string& triton_server_path,
Expand All @@ -154,8 +157,9 @@ ClientBackend::Create(
std::unique_ptr<ClientBackend> local_backend;
if (kind == TRITON) {
RETURN_IF_CB_ERROR(tritonremote::TritonClientBackend::Create(
url, protocol, ssl_options, BackendToGrpcType(compression_algorithm),
http_headers, verbose, &local_backend));
url, protocol, ssl_options, trace_options,
BackendToGrpcType(compression_algorithm), http_headers, verbose,
&local_backend));
}
#ifdef TRITON_ENABLE_PERF_ANALYZER_TFS
else if (kind == TENSORFLOW_SERVING) {
Expand Down
5 changes: 5 additions & 0 deletions src/c++/perf_analyzer/client_backend/client_backend.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ class ClientBackendFactory {
static Error Create(
const BackendKind kind, const std::string& url,
const ProtocolType protocol, const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const GrpcCompressionAlgorithm compression_algorithm,
std::shared_ptr<Headers> http_headers,
const std::string& triton_server_path,
Expand All @@ -272,12 +273,14 @@ class ClientBackendFactory {
ClientBackendFactory(
const BackendKind kind, const std::string& url,
const ProtocolType protocol, const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const GrpcCompressionAlgorithm compression_algorithm,
const std::shared_ptr<Headers> http_headers,
const std::string& triton_server_path,
const std::string& model_repository_path, const std::string& memory_type,
const bool verbose)
: kind_(kind), url_(url), protocol_(protocol), ssl_options_(ssl_options),
trace_options_(trace_options),
compression_algorithm_(compression_algorithm),
http_headers_(http_headers), triton_server_path(triton_server_path),
model_repository_path_(model_repository_path),
Expand All @@ -289,6 +292,7 @@ class ClientBackendFactory {
const std::string url_;
const ProtocolType protocol_;
const SslOptionsBase& ssl_options_;
const std::map<std::string, std::vector<std::string>> trace_options_;
const GrpcCompressionAlgorithm compression_algorithm_;
std::shared_ptr<Headers> http_headers_;
std::string triton_server_path;
Expand All @@ -305,6 +309,7 @@ class ClientBackend {
static Error Create(
const BackendKind kind, const std::string& url,
const ProtocolType protocol, const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const GrpcCompressionAlgorithm compression_algorithm,
std::shared_ptr<Headers> http_headers, const bool verbose,
const std::string& library_directory, const std::string& model_repository,
Expand Down
13 changes: 13 additions & 0 deletions src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ Error
TritonClientBackend::Create(
const std::string& url, const ProtocolType protocol,
const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const grpc_compression_algorithm compression_algorithm,
std::shared_ptr<Headers> http_headers, const bool verbose,
std::unique_ptr<ClientBackend>* client_backend)
Expand All @@ -97,6 +98,12 @@ TritonClientBackend::Create(
RETURN_IF_TRITON_ERROR(tc::InferenceServerHttpClient::Create(
&(triton_client_backend->client_.http_client_), url, verbose,
http_ssl_options));
if (!trace_options.empty()) {
std::string response;
RETURN_IF_TRITON_ERROR(
triton_client_backend->client_.http_client_->UpdateTraceSettings(
&response, "", trace_options));
}
} else {
std::pair<bool, triton::client::SslOptions> grpc_ssl_options_pair =
ParseGrpcSslOptions(ssl_options);
Expand All @@ -105,6 +112,12 @@ TritonClientBackend::Create(
RETURN_IF_TRITON_ERROR(tc::InferenceServerGrpcClient::Create(
&(triton_client_backend->client_.grpc_client_), url, verbose, use_ssl,
grpc_ssl_options));
if (!trace_options.empty()) {
inference::TraceSettingResponse response;
RETURN_IF_TRITON_ERROR(
triton_client_backend->client_.grpc_client_->UpdateTraceSettings(
&response, "", trace_options));
}
}

*client_backend = std::move(triton_client_backend);
Expand Down
1 change: 1 addition & 0 deletions src/c++/perf_analyzer/client_backend/triton/triton_client_backend.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class TritonClientBackend : public ClientBackend {
static Error Create(
const std::string& url, const ProtocolType protocol,
const SslOptionsBase& ssl_options,
const std::map<std::string, std::vector<std::string>> trace_options,
const grpc_compression_algorithm compression_algorithm,
std::shared_ptr<tc::Headers> http_headers, const bool verbose,
std::unique_ptr<ClientBackend>* client_backend);
Expand Down
80 changes: 77 additions & 3 deletions src/c++/perf_analyzer/perf_analyzer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,11 @@ Usage(char** argv, const std::string& msg = std::string())
std::cerr << "\t--streaming" << std::endl;
std::cerr << "\t--grpc-compression-algorithm <compression_algorithm>"
<< std::endl;
std::cerr << "\t--trace-file" << std::endl;
std::cerr << "\t--trace-level" << std::endl;
std::cerr << "\t--trace-rate" << std::endl;
std::cerr << "\t--trace-count" << std::endl;
std::cerr << "\t--log-frequency" << std::endl;
std::cerr << std::endl;
std::cerr << "==== OPTIONS ==== \n \n";

Expand Down Expand Up @@ -732,6 +737,47 @@ Usage(char** argv, const std::string& msg = std::string())
18)
<< std::endl;

std::cerr
<< FormatMessage(
" --trace-file: Set the file where trace output will be saved."
" If --trace-log-frequency is also specified, this argument "
"value will be the prefix of the files to save the trace "
"output. See --trace-log-frequency for details. Only used for "
"service-kind of triton. Default value is none.",
18)
<< std::endl;
std::cerr
<< FormatMessage(
"--trace-level: Specify a trace level. OFF to disable tracing, "
"TIMESTAMPS to trace timestamps, TENSORS to trace tensors. It "
"may be specified multiple times to trace multiple "
"informations. Default is OFF.",
18)
<< std::endl;
std::cerr
<< FormatMessage(
" --trace-rate: Set the trace sampling rate. Default is 1000.", 18)
<< std::endl;
std::cerr << FormatMessage(
" --trace-count: Set the number of traces to be sampled. "
"If the value is -1, the number of traces to be sampled "
"will not be limited. Default is -1.",
18)
<< std::endl;
std::cerr
<< FormatMessage(
" --log-frequency: Set the trace log frequency. If the "
"value is 0, Triton will only log the trace output to "
"<trace-file> when shutting down. Otherwise, Triton will log "
"the trace output to <trace-file>.<idx> when it collects the "
"specified number of traces. For example, if the log frequency "
"is 100, when Triton collects the 100-th trace, it logs the "
"traces to file <trace-file>.0, and when it collects the 200-th "
"trace, it logs the 101-th to the 200-th traces to file "
"<trace-file>.1. Default is 0.",
18)
<< std::endl;

std::cerr << FormatMessage(
" --triton-server-directory: The Triton server install "
"path. Required by and only used when C API "
Expand Down Expand Up @@ -825,6 +871,9 @@ PerfAnalyzer::Run(int argc, char** argv)
// gRPC and HTTP SSL options
cb::SslOptionsBase ssl_options;

// Trace options
std::map<std::string, std::vector<std::string>> trace_options;

// Verbose csv option for including additional information
bool verbose_csv = false;

Expand Down Expand Up @@ -877,6 +926,11 @@ PerfAnalyzer::Run(int argc, char** argv)
{"ssl-https-private-key-type", 1, 0, 41},
{"verbose-csv", 0, 0, 42},
{"enable-mpi", 0, 0, 43},
{"trace-file", 1, 0, 44},
{"trace-level", 1, 0, 45},
{"trace-rate", 1, 0, 46},
{"trace-count", 1, 0, 47},
{"log-frequency", 1, 0, 48},
{0, 0, 0, 0}};

// Parse commandline...
Expand Down Expand Up @@ -1286,6 +1340,26 @@ PerfAnalyzer::Run(int argc, char** argv)
enable_mpi = true;
break;
}
case 44: {
trace_options["trace_file"] = {optarg};
break;
}
case 45: {
trace_options["trace_level"] = {optarg};
break;
}
case 46: {
trace_options["trace_rate"] = {optarg};
break;
}
case 47: {
trace_options["trace_count"] = {optarg};
break;
}
case 48: {
trace_options["log_frequency"] = {optarg};
break;
}
case 'v':
extra_verbose = verbose;
verbose = true;
Expand Down Expand Up @@ -1562,9 +1636,9 @@ PerfAnalyzer::Run(int argc, char** argv)
std::shared_ptr<cb::ClientBackendFactory> factory;
FAIL_IF_ERR(
cb::ClientBackendFactory::Create(
kind, url, protocol, ssl_options, compression_algorithm, http_headers,
triton_server_path, model_repository_path, memory_type, extra_verbose,
&factory),
kind, url, protocol, ssl_options, trace_options,
compression_algorithm, http_headers, triton_server_path,
model_repository_path, memory_type, extra_verbose, &factory),
"failed to create client factory");

std::unique_ptr<cb::ClientBackend> backend;
Expand Down

0 comments on commit f4c75d8

Please sign in to comment.