Skip to content

Commit

Permalink
WIP: Bazel client: add a simple profiler
Browse files Browse the repository at this point in the history 
Add a simple profiler that can measure function
call counts and durations, and report statistics
by printing to stdout.

The enable the profiler, set the
BAZEL_CLIENT_PROFILE preprocessor definition.

See bazelbuild#5444

Change-Id: I43f0afd124b486c694f451e8455a66ffca8137b6
  • Loading branch information
@laszlocsomor
laszlocsomor committed Jun 25, 2018
1 parent 88d1cae commit 0dfe551
Show file tree
Hide file tree
Showing 11 changed files with 457 additions and 41 deletions.
1 change: 1 addition & 0 deletions src/main/cpp/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ cc_binary(
"//src/main/cpp/util:logging",
"//src/main/cpp/util:strings",
"//src/main/protobuf:command_server_cc_proto",
"//src/main/cpp/util:profiler",
"//third_party/ijar:zip",
],
)
Expand Down
65 changes: 37 additions & 28 deletions src/main/cpp/blaze.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
#include "src/main/cpp/util/path.h"
#include "src/main/cpp/util/path_platform.h"
#include "src/main/cpp/util/port.h"
#include "src/main/cpp/util/profiler.h"
#include "src/main/cpp/util/strings.h"
#include "src/main/cpp/workspace_layout.h"
#include "third_party/ijar/zip.h"
Expand All @@ -77,11 +78,11 @@ using blaze_util::GetLastErrorString;

namespace blaze {

using command_server::CommandServer;
using std::map;
using std::set;
using std::string;
using std::vector;
using command_server::CommandServer;

// The following is a treatise on how the interaction between the client and the
// server works.
Expand Down Expand Up @@ -268,7 +269,7 @@ class PureZipExtractorProcessor : public devtools_ijar::ZipExtractorProcessor {
// the Blaze binary to the given vector.
class NoteAllFilesZipProcessor : public PureZipExtractorProcessor {
public:
explicit NoteAllFilesZipProcessor(std::vector<std::string>* files)
explicit NoteAllFilesZipProcessor(std::vector<std::string> *files)
: files_(files) {}

bool AcceptPure(const char *filename,
Expand All @@ -286,21 +287,22 @@ class NoteAllFilesZipProcessor : public PureZipExtractorProcessor {
BAZEL_DIE(blaze_exit_code::INTERNAL_ERROR)
<< "NoteAllFilesZipProcessor::Process shouldn't be called";
}

private:
std::vector<std::string>* files_;
std::vector<std::string> *files_;
};

// A devtools_ijar::ZipExtractorProcessor that processes the ZIP entries using
// the given PureZipExtractorProcessors.
class CompoundZipProcessor : public devtools_ijar::ZipExtractorProcessor {
public:
explicit CompoundZipProcessor(
const vector<PureZipExtractorProcessor*>& processors)
const vector<PureZipExtractorProcessor *> &processors)
: processors_(processors) {}

bool Accept(const char *filename, const devtools_ijar::u4 attr) override {
bool should_accept = false;
for (auto* processor : processors_) {
for (auto *processor : processors_) {
if (processor->Accept(filename, attr)) {
// ZipExtractorProcessor::Accept is allowed to be side-effectful, so
// we don't want to break out on the first true here.
Expand All @@ -312,15 +314,15 @@ class CompoundZipProcessor : public devtools_ijar::ZipExtractorProcessor {

void Process(const char *filename, const devtools_ijar::u4 attr,
const devtools_ijar::u1 *data, const size_t size) override {
for (auto* processor : processors_) {
for (auto *processor : processors_) {
if (processor->AcceptPure(filename, attr)) {
processor->Process(filename, attr, data, size);
}
}
}

private:
const vector<PureZipExtractorProcessor*> processors_;
const vector<PureZipExtractorProcessor *> processors_;
};

static map<string, EnvVarValue> PrepareEnvironmentForJvm();
Expand Down Expand Up @@ -363,8 +365,8 @@ static void ComputeInstallMd5AndNoteAllFiles(const string &self_path) {
NoteAllFilesZipProcessor note_all_files_processor(
&globals->extracted_binaries);
GetInstallKeyFileProcessor install_key_processor(&globals->install_md5);
CompoundZipProcessor processor({&note_all_files_processor,
&install_key_processor});
CompoundZipProcessor processor(
{&note_all_files_processor, &install_key_processor});
std::unique_ptr<devtools_ijar::ZipExtractor> extractor(
devtools_ijar::ZipExtractor::Create(self_path.c_str(), &processor));
if (extractor.get() == NULL) {
Expand Down Expand Up @@ -663,7 +665,7 @@ static void GoToWorkspace(const WorkspaceLayout *workspace_layout) {

// Starts the Blaze server.
static int StartServer(const WorkspaceLayout *workspace_layout,
BlazeServerStartup **server_startup) {
BlazeServerStartup **server_startup) {
vector<string> jvm_args_vector = GetArgumentArray(workspace_layout);
string argument_string = GetArgumentString(jvm_args_vector);
string server_dir =
Expand Down Expand Up @@ -916,8 +918,8 @@ static void ActuallyExtractData(const string &argv0,
std::string install_md5;
GetInstallKeyFileProcessor install_key_processor(&install_md5);
ExtractBlazeZipProcessor extract_blaze_processor(embedded_binaries);
CompoundZipProcessor processor({&extract_blaze_processor,
&install_key_processor});
CompoundZipProcessor processor(
{&extract_blaze_processor, &install_key_processor});
if (!blaze_util::MakeDirectories(embedded_binaries, 0777)) {
BAZEL_DIE(blaze_exit_code::INTERNAL_ERROR)
<< "couldn't create '" << embedded_binaries
Expand Down Expand Up @@ -1000,6 +1002,9 @@ static void ActuallyExtractData(const string &argv0,
blaze_util::SyncFile(embedded_binaries);
}

static blaze_util::profiler::Stats extract_data_stats_(__FILE__
": ExtractData");

// Installs Blaze by extracting the embedded data files, iff necessary.
// The MD5-named install_base directory on disk is trusted; we assume
// no-one has modified the extracted files beneath this directory once
Expand All @@ -1008,6 +1013,7 @@ static void ActuallyExtractData(const string &argv0,
// becomes visible automically at the new path.
// Populates globals->extracted_binaries with their extracted locations.
static void ExtractData(const string &self_path) {
blaze_util::profiler::ScopeProfiler prof(&extract_data_stats_);
// If the install dir doesn't exist, create it, if it does, we know it's good.
if (!blaze_util::PathExists(globals->options->install_base)) {
uint64_t st = GetMillisecondsMonotonic();
Expand Down Expand Up @@ -1271,8 +1277,8 @@ static void ParseOptions(int argc, const char *argv[]) {
std::vector<std::string> args;
args.insert(args.end(), argv, argv + argc);
const blaze_exit_code::ExitCode parse_exit_code =
globals->option_processor->ParseOptions(
args, globals->workspace, globals->cwd, &error);
globals->option_processor->ParseOptions(args, globals->workspace,
globals->cwd, &error);

if (parse_exit_code != blaze_exit_code::SUCCESS) {
globals->option_processor->PrintStartupOptionsProvenanceMessage();
Expand Down Expand Up @@ -1311,8 +1317,8 @@ static void ComputeBaseDirectories(const WorkspaceLayout *workspace_layout,
string install_user_root =
blaze_util::JoinPath(globals->options->output_user_root, "install");
ComputeInstallMd5AndNoteAllFiles(self_path);
globals->options->install_base = blaze_util::JoinPath(install_user_root,
globals->install_md5);
globals->options->install_base =
blaze_util::JoinPath(install_user_root, globals->install_md5);
} else {
// We still need to populate globals->install_md5 and
// globals->extracted_binaries.
Expand Down Expand Up @@ -1359,7 +1365,7 @@ static void ComputeBaseDirectories(const WorkspaceLayout *workspace_layout,
globals->jvm_log_file_append = true;
} else {
globals->jvm_log_file =
blaze_util::JoinPath(globals->options->output_base, "server/jvm.out");
blaze_util::JoinPath(globals->options->output_base, "server/jvm.out");
globals->jvm_log_file_append = false;
}
}
Expand Down Expand Up @@ -1467,6 +1473,8 @@ int GetExitCodeForAbruptExit(const GlobalVariables &globals) {

int Main(int argc, const char *argv[], WorkspaceLayout *workspace_layout,
OptionProcessor *option_processor) {
blaze_util::profiler::StopWatch prof;

// Logging must be set first to assure no log statements are missed.
std::unique_ptr<blaze_util::BazelLogHandler> default_handler(
new blaze_util::BazelLogHandler());
Expand Down Expand Up @@ -1520,10 +1528,13 @@ int Main(int argc, const char *argv[], WorkspaceLayout *workspace_layout,

WarnFilesystemType(globals->options->output_base);

prof.PrintAndReset("main 1");

ExtractData(self_path);
globals->jvm_path = globals->options->GetJvm();

blaze_server->Connect();
prof.PrintAndReset("main 2");

if (!globals->options->batch &&
"shutdown" == globals->option_processor->GetCommand() &&
Expand All @@ -1537,8 +1548,10 @@ int Main(int argc, const char *argv[], WorkspaceLayout *workspace_layout,
if (globals->options->batch) {
SetScheduling(globals->options->batch_cpu_scheduling,
globals->options->io_nice_level);
prof.PrintAndReset("main 3");
StartStandalone(workspace_layout, blaze_server);
} else {
prof.PrintAndReset("main 4");
SendServerRequest(workspace_layout, blaze_server);
}
return 0;
Expand All @@ -1564,8 +1577,7 @@ GrpcBlazeServer::~GrpcBlazeServer() {
pipe_ = NULL;
}

bool GrpcBlazeServer::TryConnect(
CommandServer::Stub *client) {
bool GrpcBlazeServer::TryConnect(CommandServer::Stub *client) {
grpc::ClientContext context;
context.set_deadline(std::chrono::system_clock::now() +
std::chrono::seconds(connect_timeout_secs_));
Expand Down Expand Up @@ -1630,8 +1642,7 @@ bool GrpcBlazeServer::Connect() {

std::shared_ptr<grpc::Channel> channel(
grpc::CreateChannel(port, grpc::InsecureChannelCredentials()));
std::unique_ptr<CommandServer::Stub> client(
CommandServer::NewStub(channel));
std::unique_ptr<CommandServer::Stub> client(CommandServer::NewStub(channel));

if (!TryConnect(client.get())) {
return false;
Expand Down Expand Up @@ -1778,8 +1789,7 @@ unsigned int GrpcBlazeServer::Communicate() {
const vector<string> command_args =
globals->option_processor->GetCommandArguments();
if (!command_args.empty()) {
arg_vector.insert(arg_vector.end(),
command_args.begin(),
arg_vector.insert(arg_vector.end(), command_args.begin(),
command_args.end());
}

Expand Down Expand Up @@ -1899,7 +1909,7 @@ unsigned int GrpcBlazeServer::Communicate() {
<< globals->jvm_log_file << "')\n";
return GetExitCodeForAbruptExit(*globals);
} else if (final_response.has_exec_request()) {
const command_server::ExecRequest& request = final_response.exec_request();
const command_server::ExecRequest &request = final_response.exec_request();
if (request.argv_size() < 1) {
BAZEL_LOG(USER)
<< "\nServer requested exec() but did not pass a binary to execute\n";
Expand All @@ -1908,7 +1918,7 @@ unsigned int GrpcBlazeServer::Communicate() {

vector<string> argv;
argv.insert(argv.begin(), request.argv().begin(), request.argv().end());
for (const auto& variable : request.environment_variable()) {
for (const auto &variable : request.environment_variable()) {
SetEnv(variable.name(), variable.value());
}

Expand All @@ -1921,9 +1931,8 @@ unsigned int GrpcBlazeServer::Communicate() {
}

// We'll exit with exit code SIGPIPE on Unixes due to PropagateSignalOnExit()
return pipe_broken
? blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR
: final_response.exit_code();
return pipe_broken ? blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR
: final_response.exit_code();
}

void GrpcBlazeServer::Disconnect() {
Expand Down
22 changes: 22 additions & 0 deletions src/main/cpp/util/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ cc_library(
":blaze_exit_code",
":errors",
":logging",
":profiler",
":strings",
] + select({
"//src/conditions:windows": ["//src/main/native/windows:lib-file"],
Expand Down Expand Up @@ -147,6 +148,27 @@ cc_library(
deps = [":blaze_exit_code"],
)

cc_library(
name = "profiler",
srcs = [
"profiler.cc",
"profiler.h",
] + select({
"//src/conditions:windows": ["profiler_windows.cc"],
"//conditions:default": ["profiler_posix.cc"],
}),
hdrs = ["profiler.h"],
visibility = [
"//src/main/cpp:__pkg__",
"//src/test/cpp/util:__pkg__",
],
deps = [
":blaze_exit_code",
":errors",
":logging",
],
)

cc_library(
name = "blaze_exit_code",
hdrs = ["exit_code.h"],
Expand Down
26 changes: 14 additions & 12 deletions src/main/cpp/util/file_windows.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ctype.h> // isalpha
#include <wchar.h> // wcslen
#include <ctype.h> // isalpha
#include <wchar.h> // wcslen
#include <wctype.h> // iswalpha
#include <windows.h>

Expand All @@ -26,23 +26,22 @@
#include "src/main/cpp/util/logging.h"
#include "src/main/cpp/util/path.h"
#include "src/main/cpp/util/path_platform.h"
#include "src/main/cpp/util/profiler.h"
#include "src/main/cpp/util/strings.h"
#include "src/main/native/windows/file.h"
#include "src/main/native/windows/util.h"

namespace blaze_util {

using bazel::windows::AutoHandle;
using bazel::windows::GetLongPath;
using bazel::windows::HasUncPrefix;
using bazel::windows::OpenDirectory;
using std::basic_string;
using std::pair;
using std::string;
using std::unique_ptr;
using std::wstring;
using bazel::windows::AutoHandle;
using bazel::windows::GetLongPath;
using bazel::windows::HasUncPrefix;
using bazel::windows::OpenDirectory;



// Returns true if `path` refers to a directory or (non-dangling) junction.
// `path` must be a normalized Windows path, with UNC prefix (and absolute) if
Expand Down Expand Up @@ -73,8 +72,8 @@ class WindowsPipe : public IPipe {

bool Send(const void* buffer, int size) override {
DWORD actually_written = 0;
return ::WriteFile(_write_handle, buffer, size, &actually_written,
NULL) == TRUE;
return ::WriteFile(_write_handle, buffer, size, &actually_written, NULL) ==
TRUE;
}

int Receive(void* buffer, int size, int* error) override {
Expand Down Expand Up @@ -361,7 +360,10 @@ int WriteToStdOutErr(const void* data, size_t size, bool to_stdout) {
}
}

blaze_util::profiler::Stats rename_directory_(__FILE__ ": RenameDirectory");

int RenameDirectory(const std::string& old_name, const std::string& new_name) {
blaze_util::profiler::ScopeProfiler stats(&rename_directory_);
wstring wold_name;
string error;
if (!AsAbsoluteWindowsPath(old_name, &wold_name, &error)) {
Expand Down Expand Up @@ -866,8 +868,8 @@ bool ChangeDirectory(const string& path) {
return ::SetCurrentDirectoryA(spath.c_str()) == TRUE;
}

void ForEachDirectoryEntry(const string &path,
DirectoryEntryConsumer *consume) {
void ForEachDirectoryEntry(const string& path,
DirectoryEntryConsumer* consume) {
wstring wpath;
if (path.empty() || IsDevNull(path.c_str())) {
return;
Expand Down
Loading

0 comments on commit 0dfe551

Please sign in to comment.