-
Notifications
You must be signed in to change notification settings - Fork 13.6k
[clang-doc] Reenable time trace support #141139
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conversation
@llvm/pr-subscribers-clang-tools-extra Author: Paul Kirth (ilovepi) ChangesThis patch re-enables -ftime-trace support in clang-doc. Initial support Patch is 23.59 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/141139.diff 6 Files Affected:
diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index f8e338eb7c6ed..546dd0254ec01 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -9,6 +9,7 @@
#include "BitcodeReader.h"
#include "llvm/ADT/IndexedMap.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
@@ -672,6 +673,7 @@ llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, T I) {
template <>
llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, Reference *I) {
+ llvm::TimeTraceScope("Reducing infos", "readRecord");
Record R;
llvm::StringRef Blob;
llvm::Expected<unsigned> MaybeRecID = Stream.readRecord(ID, R, &Blob);
@@ -683,6 +685,7 @@ llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, Reference *I) {
// Read a block of records into a single info.
template <typename T>
llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
+ llvm::TimeTraceScope("Reducing infos", "readBlock");
if (llvm::Error Err = Stream.EnterSubBlock(ID))
return Err;
@@ -713,6 +716,7 @@ llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
template <typename T>
llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
+ llvm::TimeTraceScope("Reducing infos", "readSubBlock");
switch (ID) {
// Blocks can only have certain types of sub blocks.
case BI_COMMENT_BLOCK_ID: {
@@ -819,6 +823,7 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
ClangDocBitcodeReader::Cursor
ClangDocBitcodeReader::skipUntilRecordOrBlock(unsigned &BlockOrRecordID) {
+ llvm::TimeTraceScope("Reducing infos", "skipUntilRecordOrBlock");
BlockOrRecordID = 0;
while (!Stream.AtEndOfStream()) {
@@ -880,6 +885,7 @@ llvm::Error ClangDocBitcodeReader::validateStream() {
}
llvm::Error ClangDocBitcodeReader::readBlockInfoBlock() {
+ llvm::TimeTraceScope("Reducing infos", "readBlockInfoBlock");
Expected<std::optional<llvm::BitstreamBlockInfo>> MaybeBlockInfo =
Stream.ReadBlockInfoBlock();
if (!MaybeBlockInfo)
@@ -895,6 +901,7 @@ llvm::Error ClangDocBitcodeReader::readBlockInfoBlock() {
template <typename T>
llvm::Expected<std::unique_ptr<Info>>
ClangDocBitcodeReader::createInfo(unsigned ID) {
+ llvm::TimeTraceScope("Reducing infos", "createInfo");
std::unique_ptr<Info> I = std::make_unique<T>();
if (auto Err = readBlock(ID, static_cast<T *>(I.get())))
return std::move(Err);
@@ -903,6 +910,7 @@ ClangDocBitcodeReader::createInfo(unsigned ID) {
llvm::Expected<std::unique_ptr<Info>>
ClangDocBitcodeReader::readBlockToInfo(unsigned ID) {
+ llvm::TimeTraceScope("Reducing infos", "readBlockToInfo");
switch (ID) {
case BI_NAMESPACE_BLOCK_ID:
return createInfo<NamespaceInfo>(ID);
diff --git a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index 366deb55b77b9..e34e1d052f197 100644
--- a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -18,6 +18,8 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Mustache.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TimeProfiler.h"
using namespace llvm;
using namespace llvm::json;
@@ -81,13 +83,18 @@ static Error setupTemplateFiles(const clang::doc::ClangDocContext &CDCtx) {
Error MustacheHTMLGenerator::generateDocs(
StringRef RootDir, StringMap<std::unique_ptr<doc::Info>> Infos,
const clang::doc::ClangDocContext &CDCtx) {
- if (auto Err = setupTemplateFiles(CDCtx))
- return Err;
+ {
+ llvm::TimeTraceScope TS("Setup Templates");
+ if (auto Err = setupTemplateFiles(CDCtx))
+ return Err;
+ }
+
// Track which directories we already tried to create.
StringSet<> CreatedDirs;
// Collect all output by file name and create the necessary directories.
StringMap<std::vector<doc::Info *>> FileToInfos;
for (const auto &Group : Infos) {
+ llvm::TimeTraceScope TS("setup directories");
doc::Info *Info = Group.getValue().get();
SmallString<128> Path;
@@ -104,15 +111,19 @@ Error MustacheHTMLGenerator::generateDocs(
FileToInfos[Path].push_back(Info);
}
- for (const auto &Group : FileToInfos) {
- std::error_code FileErr;
- raw_fd_ostream InfoOS(Group.getKey(), FileErr, sys::fs::OF_None);
- if (FileErr)
- return createFileOpenError(Group.getKey(), FileErr);
-
- for (const auto &Info : Group.getValue())
- if (Error Err = generateDocForInfo(Info, InfoOS, CDCtx))
- return Err;
+ {
+ llvm::TimeTraceScope TS("Generate Docs");
+ for (const auto &Group : FileToInfos) {
+ llvm::TimeTraceScope TS("Info to Doc");
+ std::error_code FileErr;
+ raw_fd_ostream InfoOS(Group.getKey(), FileErr, sys::fs::OF_None);
+ if (FileErr)
+ return createFileOpenError(Group.getKey(), FileErr);
+
+ for (const auto &Info : Group.getValue())
+ if (Error Err = generateDocForInfo(Info, InfoOS, CDCtx))
+ return Err;
+ }
}
return Error::success();
}
diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp
index a17645c3f3a31..bbc8dc54ed4a3 100644
--- a/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/clang-tools-extra/clang-doc/Mapper.cpp
@@ -13,7 +13,9 @@
#include "clang/Index/USRGeneration.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/Mutex.h"
+#include "llvm/Support/TimeProfiler.h"
namespace clang {
namespace doc {
@@ -40,48 +42,66 @@ Location MapASTVisitor::getDeclLocation(const NamedDecl *D) const {
}
void MapASTVisitor::HandleTranslationUnit(ASTContext &Context) {
+ if (CDCtx.FTimeTrace)
+ llvm::timeTraceProfilerInitialize(200, "clang-doc");
TraverseDecl(Context.getTranslationUnitDecl());
+ if (CDCtx.FTimeTrace)
+ llvm::timeTraceProfilerFinishThread();
}
template <typename T>
bool MapASTVisitor::mapDecl(const T *D, bool IsDefinition) {
- // If we're looking a decl not in user files, skip this decl.
- if (D->getASTContext().getSourceManager().isInSystemHeader(D->getLocation()))
- return true;
+ llvm::TimeTraceScope TS("Mapping declaration");
+ {
+ llvm::TimeTraceScope TS("Preamble");
+ // If we're looking a decl not in user files, skip this decl.
+ if (D->getASTContext().getSourceManager().isInSystemHeader(
+ D->getLocation()))
+ return true;
- // Skip function-internal decls.
- if (D->getParentFunctionOrMethod())
- return true;
+ // Skip function-internal decls.
+ if (D->getParentFunctionOrMethod())
+ return true;
+ }
+
+ std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>> CP;
- llvm::SmallString<128> USR;
- // If there is an error generating a USR for the decl, skip this decl.
- if (index::generateUSRForDecl(D, USR))
- return true;
- // Prevent Visiting USR twice
{
- llvm::sys::SmartScopedLock<true> Guard(USRVisitedGuard);
- StringRef Visited = USR.str();
- if (USRVisited.count(Visited) && !isTypedefAnonRecord<T>(D))
+ llvm::TimeTraceScope TS("emit info from astnode");
+ llvm::SmallString<128> USR;
+ // If there is an error generating a USR for the decl, skip this decl.
+ if (index::generateUSRForDecl(D, USR))
return true;
- // We considered a USR to be visited only when its defined
- if (IsDefinition)
- USRVisited.insert(Visited);
+ // Prevent Visiting USR twice
+ {
+ llvm::sys::SmartScopedLock<true> Guard(USRVisitedGuard);
+ StringRef Visited = USR.str();
+ if (USRVisited.count(Visited) && !isTypedefAnonRecord<T>(D))
+ return true;
+ // We considered a USR to be visited only when its defined
+ if (IsDefinition)
+ USRVisited.insert(Visited);
+ }
+ bool IsFileInRootDir;
+ llvm::SmallString<128> File =
+ getFile(D, D->getASTContext(), CDCtx.SourceRoot, IsFileInRootDir);
+ CP = serialize::emitInfo(D, getComment(D, D->getASTContext()),
+ getDeclLocation(D), CDCtx.PublicOnly);
+ }
+
+ auto &[Child, Parent] = CP;
+
+ {
+ llvm::TimeTraceScope TS("serialized info into bitcode");
+ // A null in place of a valid Info indicates that the serializer is skipping
+ // this decl for some reason (e.g. we're only reporting public decls).
+ if (Child)
+ CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Child->USR)),
+ serialize::serialize(Child));
+ if (Parent)
+ CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Parent->USR)),
+ serialize::serialize(Parent));
}
- bool IsFileInRootDir;
- llvm::SmallString<128> File =
- getFile(D, D->getASTContext(), CDCtx.SourceRoot, IsFileInRootDir);
- auto [Child, Parent] =
- serialize::emitInfo(D, getComment(D, D->getASTContext()),
- getDeclLocation(D), CDCtx.PublicOnly);
-
- // A null in place of a valid Info indicates that the serializer is skipping
- // this decl for some reason (e.g. we're only reporting public decls).
- if (Child)
- CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Child->USR)),
- serialize::serialize(Child));
- if (Parent)
- CDCtx.ECtx->reportResult(llvm::toHex(llvm::toStringRef(Parent->USR)),
- serialize::serialize(Parent));
return true;
}
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 9ab2f342d969a..ad93ed66b86a1 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -369,9 +369,11 @@ ClangDocContext::ClangDocContext(tooling::ExecutionContext *ECtx,
StringRef OutDirectory, StringRef SourceRoot,
StringRef RepositoryUrl,
StringRef RepositoryLinePrefix, StringRef Base,
- std::vector<std::string> UserStylesheets)
+ std::vector<std::string> UserStylesheets,
+ bool FTimeTrace)
: ECtx(ECtx), ProjectName(ProjectName), PublicOnly(PublicOnly),
- OutDirectory(OutDirectory), UserStylesheets(UserStylesheets), Base(Base) {
+ FTimeTrace(FTimeTrace), OutDirectory(OutDirectory),
+ UserStylesheets(UserStylesheets), Base(Base) {
llvm::SmallString<128> SourceRootDir(SourceRoot);
if (SourceRoot.empty())
// If no SourceRoot was provided the current path is used as the default
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index a2e01719eb59e..e7c67574da1e4 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -518,10 +518,13 @@ struct ClangDocContext {
ClangDocContext(tooling::ExecutionContext *ECtx, StringRef ProjectName,
bool PublicOnly, StringRef OutDirectory, StringRef SourceRoot,
StringRef RepositoryUrl, StringRef RepositoryCodeLinePrefix,
- StringRef Base, std::vector<std::string> UserStylesheets);
+ StringRef Base, std::vector<std::string> UserStylesheets,
+ bool FTimeTrace = false);
tooling::ExecutionContext *ECtx;
std::string ProjectName; // Name of project clang-doc is documenting.
bool PublicOnly; // Indicates if only public declarations are documented.
+ bool FTimeTrace; // Indicates if ftime trace is turned on
+ int Granularity; // Granularity of ftime trace
std::string OutDirectory; // Directory for outputting generated files.
std::string SourceRoot; // Directory where processed files are stored. Links
// to definition locations will only be generated if
diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index 8e8f7053a8f87..a205c8f031cfe 100644
--- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -41,6 +41,7 @@
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
#include <atomic>
#include <mutex>
@@ -110,6 +111,11 @@ static llvm::cl::opt<std::string> RepositoryCodeLinePrefix(
llvm::cl::desc("Prefix of line code for repository."),
llvm::cl::cat(ClangDocCategory));
+static llvm::cl::opt<bool> FTimeTrace("ftime-trace", llvm::cl::desc(R"(
+Turn on time profiler. Generates clang-doc-tracing.json)"),
+ llvm::cl::init(false),
+ llvm::cl::cat(ClangDocCategory));
+
enum OutputFormatTy {
md,
yaml,
@@ -256,140 +262,195 @@ Example usage for a project using a compile commands database:
return 1;
}
- // Fail early if an invalid format was provided.
- std::string Format = getFormatString();
- llvm::outs() << "Emiting docs in " << Format << " format.\n";
- auto G = doc::findGeneratorByName(Format);
- if (!G) {
- llvm::errs() << toString(G.takeError()) << "\n";
- return 1;
- }
-
- ArgumentsAdjuster ArgAdjuster;
- if (!DoxygenOnly)
- ArgAdjuster = combineAdjusters(
- getInsertArgumentAdjuster("-fparse-all-comments",
- tooling::ArgumentInsertPosition::END),
- ArgAdjuster);
-
- clang::doc::ClangDocContext CDCtx = {
- Executor->get()->getExecutionContext(),
- ProjectName,
- PublicOnly,
- OutDirectory,
- SourceRoot,
- RepositoryUrl,
- RepositoryCodeLinePrefix,
- BaseDirectory,
- {UserStylesheets.begin(), UserStylesheets.end()}};
-
- if (Format == "html") {
- if (auto Err = getHtmlAssetFiles(argv[0], CDCtx)) {
- llvm::errs() << toString(std::move(Err)) << "\n";
+ // turns on ftime trace profiling
+ if (FTimeTrace)
+ llvm::timeTraceProfilerInitialize(200, "clang-doc");
+ {
+ llvm::TimeTraceScope("main");
+
+ // Fail early if an invalid format was provided.
+ std::string Format = getFormatString();
+ llvm::outs() << "Emiting docs in " << Format << " format.\n";
+ auto G = doc::findGeneratorByName(Format);
+ if (!G) {
+ llvm::errs() << toString(G.takeError()) << "\n";
return 1;
}
- }
- // Mapping phase
- llvm::outs() << "Mapping decls...\n";
- auto Err =
- Executor->get()->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster);
- if (Err) {
- if (IgnoreMappingFailures)
- llvm::errs() << "Error mapping decls in files. Clang-doc will ignore "
- "these files and continue:\n"
- << toString(std::move(Err)) << "\n";
- else {
- llvm::errs() << toString(std::move(Err)) << "\n";
- return 1;
+ ArgumentsAdjuster ArgAdjuster;
+ if (!DoxygenOnly)
+ ArgAdjuster = combineAdjusters(
+ getInsertArgumentAdjuster("-fparse-all-comments",
+ tooling::ArgumentInsertPosition::END),
+ ArgAdjuster);
+
+ clang::doc::ClangDocContext CDCtx = {
+ Executor->get()->getExecutionContext(),
+ ProjectName,
+ PublicOnly,
+ OutDirectory,
+ SourceRoot,
+ RepositoryUrl,
+ RepositoryCodeLinePrefix,
+ BaseDirectory,
+ {UserStylesheets.begin(), UserStylesheets.end()},
+ FTimeTrace};
+
+ if (Format == "html") {
+ if (auto Err = getHtmlAssetFiles(argv[0], CDCtx)) {
+ llvm::errs() << toString(std::move(Err)) << "\n";
+ return 1;
+ }
}
- }
- // Collect values into output by key.
- // In ToolResults, the Key is the hashed USR and the value is the
- // bitcode-encoded representation of the Info object.
- llvm::outs() << "Collecting infos...\n";
- llvm::StringMap<std::vector<StringRef>> USRToBitcode;
- Executor->get()->getToolResults()->forEachResult(
- [&](StringRef Key, StringRef Value) {
- USRToBitcode[Key].emplace_back(Value);
- });
-
- // Collects all Infos according to their unique USR value. This map is added
- // to from the thread pool below and is protected by the USRToInfoMutex.
- llvm::sys::Mutex USRToInfoMutex;
- llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo;
-
- // First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
- std::atomic<bool> Error;
- Error = false;
- llvm::sys::Mutex IndexMutex;
- // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
- llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency));
- for (auto &Group : USRToBitcode) {
- Pool.async([&]() {
- std::vector<std::unique_ptr<doc::Info>> Infos;
- for (auto &Bitcode : Group.getValue()) {
- llvm::BitstreamCursor Stream(Bitcode);
- doc::ClangDocBitcodeReader Reader(Stream);
- auto ReadInfos = Reader.readBitcode();
- if (!ReadInfos) {
- llvm::errs() << toString(ReadInfos.takeError()) << "\n";
- Error = true;
- return;
- }
- std::move(ReadInfos->begin(), ReadInfos->end(),
- std::back_inserter(Infos));
+ if (Format == "mustache") {
+ if (auto Err = getMustacheHtmlFiles(argv[0], CDCtx)) {
+ llvm::errs() << toString(std::move(Err)) << "\n";
+ return 1;
}
+ }
- auto Reduced = doc::mergeInfos(Infos);
- if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- return;
+ llvm::timeTraceProfilerBegin("Executor Launch", "total runtime");
+ // Mapping phase
+ llvm::outs() << "Mapping decls...\n";
+ auto Err = Executor->get()->execute(doc::newMapperActionFactory(CDCtx),
+ ArgAdjuster);
+ llvm::timeTraceProfilerEnd();
+ if (Err) {
+ if (IgnoreMappingFailures)
+ llvm::errs() << "Error mapping decls in files. Clang-doc will ignore "
+ "these files and continue:\n"
+ << toString(std::move(Err)) << "\n";
+ else {
+ llvm::errs() << toString(std::move(Err)) << "\n";
+ return 1;
}
+ }
- // Add a reference to this Info in the Index
- {
- std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex);
- clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced.get().get());
+ // Collect values into output by key.
+ // In ToolResults, the Key is the hashed USR and the value is the
+ // bitcode-encoded representation of the Info object.
+ llvm::timeTraceProfilerBegin("Collect Info", "total runtime");
+ llvm::outs() << "Collecting infos...\n";
+ llvm::StringMap<std::vector<StringRef>> USRToBitcode;
+ Executor->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+ USRToBitcode[Key].emplace_back(Value);
+ });
+ llvm::timeTraceProfilerEnd();
+
+ // Collects all Infos according to their unique USR value. This map is added
+ // to from the thread pool below and is protected by the USRToInfoMutex.
+ llvm::sys::Mutex USRToInfoMutex;
+ llvm::StringMap<std::unique_ptr<doc::Info>> USRToInfo;
+
+ // First reducing phase (reduce all decls into one info per decl).
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic<bool> Error;
+ Error = false;
+ llvm::sys::Mutex IndexMutex;
+ // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
+ llvm::DefaultThreadPool Pool(
+ llvm::hardware_concurrency(ExecutorConcurrency));
+ {
+ llvm::TimeTraceScope TS("Reduce");
+ for (auto &Group : USRToBitcode) {
+ Pool.async([&]() {
+ if (FTimeTrace)
+ llvm::timeTraceProfilerInitialize(200, "clang-doc");
+
+ std::vector<std::unique_ptr<doc::Info>> Infos;
+ {
+ llvm::TimeTraceScope Red("decoding bitcode");
+ for (auto &Bitcode : Group.getValue()) {
+ llvm::BitstreamCursor Stream(Bitcode);
+ doc::ClangDocBitcodeReader Reader(Stream);
+ auto ReadInfos = Reader.readBitcode();
+ ...
[truncated]
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm not sure I want to land this before I land the main support for Mustache templates. I think its easier to rebase this patch, than that whole stack.
7ac1e05
to
7f51d82
Compare
41c65c1
to
7f346f4
Compare
Sorry about that I was trying to get the PR changes via Github CLI... |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
LGTM
20536fe
to
80db854
Compare
This patch re-enables -ftime-trace support in clang-doc. Initial support in #97644 was reverted, and never relanded. This patch adds back the command line option, and leverages the RAII tracing infrastructure more thoroughly.
80db854
to
feed17a
Compare
This patch re-enables -ftime-trace support in clang-doc. Initial support
in #97644 was reverted, and never relanded. This patch adds back the
command line option, and leverages the RAII tracing infrastructure more
thoroughly.