From f4c75d8bb8e224590e863298c97493b3333584e6 Mon Sep 17 00:00:00 2001 From: dyastremsky <58150256+dyastremsky@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:15:46 -0700 Subject: [PATCH] Add Tracing to Perf Analyzer GRPC/HTTP Client (#136) * Added tracing for GRPC/HTTPperf_analyzer clients * Cleaned up options * Fixed flag * Wording --- .../client_backend/client_backend.cc | 18 +++-- .../client_backend/client_backend.h | 5 ++ .../triton/triton_client_backend.cc | 13 +++ .../triton/triton_client_backend.h | 1 + src/c++/perf_analyzer/perf_analyzer.cc | 80 ++++++++++++++++++- 5 files changed, 107 insertions(+), 10 deletions(-) diff --git a/src/c++/perf_analyzer/client_backend/client_backend.cc b/src/c++/perf_analyzer/client_backend/client_backend.cc index 901306d0b..f039195e2 100644 --- a/src/c++/perf_analyzer/client_backend/client_backend.cc +++ b/src/c++/perf_analyzer/client_backend/client_backend.cc @@ -115,6 +115,7 @@ Error ClientBackendFactory::Create( const BackendKind kind, const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const GrpcCompressionAlgorithm compression_algorithm, std::shared_ptr http_headers, const std::string& triton_server_path, @@ -122,8 +123,9 @@ ClientBackendFactory::Create( const bool verbose, std::shared_ptr* factory) { factory->reset(new ClientBackendFactory( - kind, url, protocol, ssl_options, compression_algorithm, http_headers, - triton_server_path, model_repository_path, memory_type, verbose)); + kind, url, protocol, ssl_options, trace_options, compression_algorithm, + http_headers, triton_server_path, model_repository_path, memory_type, + verbose)); return Error::Success; } @@ -132,9 +134,9 @@ ClientBackendFactory::CreateClientBackend( std::unique_ptr* client_backend) { RETURN_IF_CB_ERROR(ClientBackend::Create( - kind_, url_, protocol_, ssl_options_, compression_algorithm_, - http_headers_, verbose_, triton_server_path, model_repository_path_, - memory_type_, client_backend)); + kind_, url_, protocol_, ssl_options_, trace_options_, + compression_algorithm_, http_headers_, verbose_, triton_server_path, + model_repository_path_, memory_type_, client_backend)); return Error::Success; } @@ -145,6 +147,7 @@ Error ClientBackend::Create( const BackendKind kind, const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const GrpcCompressionAlgorithm compression_algorithm, std::shared_ptr http_headers, const bool verbose, const std::string& triton_server_path, @@ -154,8 +157,9 @@ ClientBackend::Create( std::unique_ptr local_backend; if (kind == TRITON) { RETURN_IF_CB_ERROR(tritonremote::TritonClientBackend::Create( - url, protocol, ssl_options, BackendToGrpcType(compression_algorithm), - http_headers, verbose, &local_backend)); + url, protocol, ssl_options, trace_options, + BackendToGrpcType(compression_algorithm), http_headers, verbose, + &local_backend)); } #ifdef TRITON_ENABLE_PERF_ANALYZER_TFS else if (kind == TENSORFLOW_SERVING) { diff --git a/src/c++/perf_analyzer/client_backend/client_backend.h b/src/c++/perf_analyzer/client_backend/client_backend.h index 936493a0f..0eb1b9a3c 100644 --- a/src/c++/perf_analyzer/client_backend/client_backend.h +++ b/src/c++/perf_analyzer/client_backend/client_backend.h @@ -258,6 +258,7 @@ class ClientBackendFactory { static Error Create( const BackendKind kind, const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const GrpcCompressionAlgorithm compression_algorithm, std::shared_ptr http_headers, const std::string& triton_server_path, @@ -272,12 +273,14 @@ class ClientBackendFactory { ClientBackendFactory( const BackendKind kind, const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const GrpcCompressionAlgorithm compression_algorithm, const std::shared_ptr http_headers, const std::string& triton_server_path, const std::string& model_repository_path, const std::string& memory_type, const bool verbose) : kind_(kind), url_(url), protocol_(protocol), ssl_options_(ssl_options), + trace_options_(trace_options), compression_algorithm_(compression_algorithm), http_headers_(http_headers), triton_server_path(triton_server_path), model_repository_path_(model_repository_path), @@ -289,6 +292,7 @@ class ClientBackendFactory { const std::string url_; const ProtocolType protocol_; const SslOptionsBase& ssl_options_; + const std::map> trace_options_; const GrpcCompressionAlgorithm compression_algorithm_; std::shared_ptr http_headers_; std::string triton_server_path; @@ -305,6 +309,7 @@ class ClientBackend { static Error Create( const BackendKind kind, const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const GrpcCompressionAlgorithm compression_algorithm, std::shared_ptr http_headers, const bool verbose, const std::string& library_directory, const std::string& model_repository, diff --git a/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc b/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc index bd3e604d3..a4d7bd1e8 100644 --- a/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc +++ b/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.cc @@ -85,6 +85,7 @@ Error TritonClientBackend::Create( const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const grpc_compression_algorithm compression_algorithm, std::shared_ptr http_headers, const bool verbose, std::unique_ptr* client_backend) @@ -97,6 +98,12 @@ TritonClientBackend::Create( RETURN_IF_TRITON_ERROR(tc::InferenceServerHttpClient::Create( &(triton_client_backend->client_.http_client_), url, verbose, http_ssl_options)); + if (!trace_options.empty()) { + std::string response; + RETURN_IF_TRITON_ERROR( + triton_client_backend->client_.http_client_->UpdateTraceSettings( + &response, "", trace_options)); + } } else { std::pair grpc_ssl_options_pair = ParseGrpcSslOptions(ssl_options); @@ -105,6 +112,12 @@ TritonClientBackend::Create( RETURN_IF_TRITON_ERROR(tc::InferenceServerGrpcClient::Create( &(triton_client_backend->client_.grpc_client_), url, verbose, use_ssl, grpc_ssl_options)); + if (!trace_options.empty()) { + inference::TraceSettingResponse response; + RETURN_IF_TRITON_ERROR( + triton_client_backend->client_.grpc_client_->UpdateTraceSettings( + &response, "", trace_options)); + } } *client_backend = std::move(triton_client_backend); diff --git a/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.h b/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.h index 620c0f075..1d4f830cc 100644 --- a/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.h +++ b/src/c++/perf_analyzer/client_backend/triton/triton_client_backend.h @@ -75,6 +75,7 @@ class TritonClientBackend : public ClientBackend { static Error Create( const std::string& url, const ProtocolType protocol, const SslOptionsBase& ssl_options, + const std::map> trace_options, const grpc_compression_algorithm compression_algorithm, std::shared_ptr http_headers, const bool verbose, std::unique_ptr* client_backend); diff --git a/src/c++/perf_analyzer/perf_analyzer.cc b/src/c++/perf_analyzer/perf_analyzer.cc index bbbc9854b..07a4785bc 100644 --- a/src/c++/perf_analyzer/perf_analyzer.cc +++ b/src/c++/perf_analyzer/perf_analyzer.cc @@ -296,6 +296,11 @@ Usage(char** argv, const std::string& msg = std::string()) std::cerr << "\t--streaming" << std::endl; std::cerr << "\t--grpc-compression-algorithm " << std::endl; + std::cerr << "\t--trace-file" << std::endl; + std::cerr << "\t--trace-level" << std::endl; + std::cerr << "\t--trace-rate" << std::endl; + std::cerr << "\t--trace-count" << std::endl; + std::cerr << "\t--log-frequency" << std::endl; std::cerr << std::endl; std::cerr << "==== OPTIONS ==== \n \n"; @@ -732,6 +737,47 @@ Usage(char** argv, const std::string& msg = std::string()) 18) << std::endl; + std::cerr + << FormatMessage( + " --trace-file: Set the file where trace output will be saved." + " If --trace-log-frequency is also specified, this argument " + "value will be the prefix of the files to save the trace " + "output. See --trace-log-frequency for details. Only used for " + "service-kind of triton. Default value is none.", + 18) + << std::endl; + std::cerr + << FormatMessage( + "--trace-level: Specify a trace level. OFF to disable tracing, " + "TIMESTAMPS to trace timestamps, TENSORS to trace tensors. It " + "may be specified multiple times to trace multiple " + "informations. Default is OFF.", + 18) + << std::endl; + std::cerr + << FormatMessage( + " --trace-rate: Set the trace sampling rate. Default is 1000.", 18) + << std::endl; + std::cerr << FormatMessage( + " --trace-count: Set the number of traces to be sampled. " + "If the value is -1, the number of traces to be sampled " + "will not be limited. Default is -1.", + 18) + << std::endl; + std::cerr + << FormatMessage( + " --log-frequency: Set the trace log frequency. If the " + "value is 0, Triton will only log the trace output to " + " when shutting down. Otherwise, Triton will log " + "the trace output to . when it collects the " + "specified number of traces. For example, if the log frequency " + "is 100, when Triton collects the 100-th trace, it logs the " + "traces to file .0, and when it collects the 200-th " + "trace, it logs the 101-th to the 200-th traces to file " + ".1. Default is 0.", + 18) + << std::endl; + std::cerr << FormatMessage( " --triton-server-directory: The Triton server install " "path. Required by and only used when C API " @@ -825,6 +871,9 @@ PerfAnalyzer::Run(int argc, char** argv) // gRPC and HTTP SSL options cb::SslOptionsBase ssl_options; + // Trace options + std::map> trace_options; + // Verbose csv option for including additional information bool verbose_csv = false; @@ -877,6 +926,11 @@ PerfAnalyzer::Run(int argc, char** argv) {"ssl-https-private-key-type", 1, 0, 41}, {"verbose-csv", 0, 0, 42}, {"enable-mpi", 0, 0, 43}, + {"trace-file", 1, 0, 44}, + {"trace-level", 1, 0, 45}, + {"trace-rate", 1, 0, 46}, + {"trace-count", 1, 0, 47}, + {"log-frequency", 1, 0, 48}, {0, 0, 0, 0}}; // Parse commandline... @@ -1286,6 +1340,26 @@ PerfAnalyzer::Run(int argc, char** argv) enable_mpi = true; break; } + case 44: { + trace_options["trace_file"] = {optarg}; + break; + } + case 45: { + trace_options["trace_level"] = {optarg}; + break; + } + case 46: { + trace_options["trace_rate"] = {optarg}; + break; + } + case 47: { + trace_options["trace_count"] = {optarg}; + break; + } + case 48: { + trace_options["log_frequency"] = {optarg}; + break; + } case 'v': extra_verbose = verbose; verbose = true; @@ -1562,9 +1636,9 @@ PerfAnalyzer::Run(int argc, char** argv) std::shared_ptr factory; FAIL_IF_ERR( cb::ClientBackendFactory::Create( - kind, url, protocol, ssl_options, compression_algorithm, http_headers, - triton_server_path, model_repository_path, memory_type, extra_verbose, - &factory), + kind, url, protocol, ssl_options, trace_options, + compression_algorithm, http_headers, triton_server_path, + model_repository_path, memory_type, extra_verbose, &factory), "failed to create client factory"); std::unique_ptr backend;