Skip to content

Commit d89622c

Browse files
mtrofinmtrofin
committed
[ctx_profile] Profile reader and writer
Utility converting a profile coming from `compiler_rt` to bitstream. `PGOCtxProfileWriter::write` would be used as the `Writer` parameter for `__llvm_ctx_profile_fetch` API. This is expected to happen in user code, for example in the RPC hanler tasked with collecting a profile, and would look like this: ``` // set up an output stream "Out", which could contain other stuff { // constructing the Writer will start the section, in Out, containing // the collected contextual profiles. PGOCtxProfWriter Writer(Out); __llvm_ctx_profile_fetch(&Writer, +[](void* W, const ContextNode &N) { reinterpret_cast<PGOCtxProfWriter*>(W)->write(N); }); // Writer going out of scope will finish up the section. } ```
1 parent 91feb13 commit d89622c

File tree

7 files changed

+665
-0
lines changed

7 files changed

+665
-0
lines changed

‎llvm/include/llvm/ProfileData/PGOCtxProfReader.h‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
///
9+
/// \file
10+
///
11+
/// Reader for contextual iFDO profile, which comes in bitstream format.
12+
///
13+
//===----------------------------------------------------------------------===//
14+
15+
#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
16+
#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
17+
18+
#include "llvm/ADT/DenseSet.h"
19+
#include "llvm/Bitstream/BitstreamReader.h"
20+
#include "llvm/IR/GlobalValue.h"
21+
#include "llvm/ProfileData/PGOCtxProfWriter.h"
22+
#include "llvm/Support/Error.h"
23+
#include <map>
24+
#include <vector>
25+
26+
namespace llvm {
27+
/// The loaded contextual profile, suitable for mutation during IPO passes. We
28+
/// generally expect a fraction of counters and of callsites to be populated.
29+
/// We continue to model counters as vectors, but callsites are modeled as a map
30+
/// of a map. The expectation is that, typically, there is a small number of
31+
/// indirect targets (usually, 1 for direct calls); but potentially a large
32+
/// number of callsites, and, as inlining progresses, the callsite count of a
33+
/// caller will grow.
34+
class PGOContextualProfile final {
35+
public:
36+
using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
37+
using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
38+
39+
private:
40+
friend class PGOCtxProfileReader;
41+
GlobalValue::GUID GUID = 0;
42+
SmallVector<uint64_t, 16> Counters;
43+
CallsiteMapTy Callsites;
44+
45+
PGOContextualProfile(GlobalValue::GUID G,
46+
SmallVectorImpl<uint64_t> &&Counters)
47+
: GUID(G), Counters(std::move(Counters)) {}
48+
49+
Expected<PGOContextualProfile &>
50+
getOrEmplace(uint32_t Index, GlobalValue::GUID G,
51+
SmallVectorImpl<uint64_t> &&Counters);
52+
53+
public:
54+
PGOContextualProfile(const PGOContextualProfile &) = delete;
55+
PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
56+
PGOContextualProfile(PGOContextualProfile &&) = default;
57+
PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
58+
59+
GlobalValue::GUID guid() const { return GUID; }
60+
const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
61+
const CallsiteMapTy &callsites() const { return Callsites; }
62+
CallsiteMapTy &callsites() { return Callsites; }
63+
64+
bool hasCallsite(uint32_t I) const {
65+
return Callsites.find(I) != Callsites.end();
66+
}
67+
68+
const CallTargetMapTy &callsite(uint32_t I) const {
69+
return Callsites.find(I)->second;
70+
}
71+
void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
72+
};
73+
74+
class PGOCtxProfileReader final {
75+
BitstreamCursor &Cursor;
76+
Expected<BitstreamEntry> advance();
77+
Error readMetadata();
78+
Error wrongValue(const Twine &);
79+
Error unsupported(const Twine &);
80+
81+
Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
82+
readContext(bool ExpectIndex);
83+
bool canReadContext();
84+
85+
public:
86+
PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
87+
88+
Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
89+
};
90+
} // namespace llvm
91+
#endif

‎llvm/include/llvm/ProfileData/PGOCtxProfWriter.h‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// This file declares a utility for writing a contextual profile to bitstream.
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
14+
#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
15+
16+
#include "llvm/Bitstream/BitstreamWriter.h"
17+
#include "llvm/ProfileData/CtxInstrContextNode.h"
18+
19+
namespace llvm {
20+
enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
21+
22+
enum PGOCtxProfileBlockIDs {
23+
ProfileMetadataBlockID = 100,
24+
ContextNodeBlockID = ProfileMetadataBlockID + 1
25+
};
26+
27+
/// Write one or more ContextNodes to the provided raw_fd_stream.
28+
/// The caller must destroy the PGOCtxProfileWriter object before closing the
29+
/// stream.
30+
/// The design allows serializing a bunch of contexts embedded in some other
31+
/// file. The overall format is:
32+
///
33+
/// [... other data written to the stream...]
34+
/// SubBlock(ProfileMetadataBlockID)
35+
/// Version
36+
/// SubBlock(ContextNodeBlockID)
37+
/// [RECORDS]
38+
/// SubBlock(ContextNodeBlockID)
39+
/// [RECORDS]
40+
/// [... more SubBlocks]
41+
/// EndBlock
42+
/// EndBlock
43+
///
44+
/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
45+
/// for Version, which is just for metadata). All contexts will have Guid and
46+
/// Counters, and all but the roots have CalleeIndex. The order in which the
47+
/// records appear does not matter, but they must precede any subcontexts,
48+
/// because that helps keep the reader code simpler.
49+
///
50+
/// Subblock containment captures the context->subcontext relationship. The
51+
/// "next()" relationship in the raw profile, between call targets of indirect
52+
/// calls, are just modeled as peer subblocks where the callee index is the
53+
/// same.
54+
///
55+
/// Versioning: the writer may produce additional records not known by the
56+
/// reader. The version number indicates a more structural change.
57+
/// The current version, in particular, is set up to expect optional extensions
58+
/// like value profiling - which would appear as additional records. For
59+
/// example, value profiling would produce a new record with a new record ID,
60+
/// containing the profiled values (much like the counters)
61+
class PGOCtxProfileWriter final {
62+
SmallVector<char, 1 << 20> Buff;
63+
BitstreamWriter Writer;
64+
65+
void writeCounters(const ctx_profile::ContextNode &Node);
66+
void writeImpl(std::optional<uint32_t> CallerIndex,
67+
const ctx_profile::ContextNode &Node);
68+
69+
public:
70+
PGOCtxProfileWriter(raw_fd_stream &Out,
71+
std::optional<unsigned> VersionOverride = std::nullopt)
72+
: Writer(Buff, &Out, 0) {
73+
Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
74+
CodeLen);
75+
const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
76+
Writer.EmitRecord(PGOCtxProfileRecords::Version,
77+
SmallVector<unsigned, 1>({Version}));
78+
}
79+
80+
~PGOCtxProfileWriter() { Writer.ExitBlock(); }
81+
82+
void write(const ctx_profile::ContextNode &);
83+
84+
// constants used in writing which a reader may find useful.
85+
static constexpr unsigned CodeLen = 2;
86+
static constexpr uint32_t CurrentVersion = 1;
87+
static constexpr unsigned VBREncodingBits = 6;
88+
};
89+
90+
} // namespace llvm
91+
#endif

‎llvm/lib/ProfileData/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMProfileData
77
ItaniumManglingCanonicalizer.cpp
88
MemProf.cpp
99
MemProfReader.cpp
10+
PGOCtxProfReader.cpp
11+
PGOCtxProfWriter.cpp
1012
ProfileSummaryBuilder.cpp
1113
SampleProf.cpp
1214
SampleProfReader.cpp

‎llvm/lib/ProfileData/PGOCtxProfReader.cpp‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Read a contextual profile into a datastructure suitable for maintenance
10+
// throughout IPO
11+
//
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "llvm/ProfileData/PGOCtxProfReader.h"
15+
#include "llvm/Bitstream/BitCodeEnums.h"
16+
#include "llvm/Bitstream/BitstreamReader.h"
17+
#include "llvm/ProfileData/PGOCtxProfWriter.h"
18+
#include "llvm/Support/Errc.h"
19+
#include "llvm/Support/Error.h"
20+
21+
using namespace llvm;
22+
23+
#define EXPECT_OR_RET(LHS, RHS) \
24+
auto LHS = RHS; \
25+
if (!LHS) \
26+
return LHS.takeError();
27+
28+
#define RET_ON_ERR(EXPR) \
29+
if (auto Err = (EXPR)) \
30+
return Err;
31+
32+
Expected<PGOContextualProfile &>
33+
PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
34+
SmallVectorImpl<uint64_t> &&Counters) {
35+
auto I = Callsites[Index].insert(
36+
{G, PGOContextualProfile(G, std::move(Counters))});
37+
if (!I.second)
38+
return make_error<StringError>(llvm::errc::invalid_argument,
39+
"Duplicate GUID for same callsite.");
40+
return I.first->second;
41+
}
42+
43+
void PGOContextualProfile::getContainedGuids(
44+
DenseSet<GlobalValue::GUID> &Guids) const {
45+
Guids.insert(GUID);
46+
for (const auto &[_, Callsite] : Callsites)
47+
for (const auto &[_, Callee] : Callsite)
48+
Callee.getContainedGuids(Guids);
49+
}
50+
51+
Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
52+
return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
53+
}
54+
55+
Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
56+
return make_error<StringError>(llvm::errc::invalid_argument, Msg);
57+
}
58+
59+
Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
60+
return make_error<StringError>(llvm::errc::not_supported, Msg);
61+
}
62+
63+
bool PGOCtxProfileReader::canReadContext() {
64+
auto Blk = advance();
65+
if (!Blk) {
66+
consumeError(Blk.takeError());
67+
return false;
68+
}
69+
return Blk->Kind == BitstreamEntry::SubBlock &&
70+
Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
71+
}
72+
73+
Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
74+
PGOCtxProfileReader::readContext(bool ExpectIndex) {
75+
RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
76+
77+
std::optional<ctx_profile::GUID> Guid;
78+
std::optional<SmallVector<uint64_t, 16>> Counters;
79+
std::optional<uint32_t> CallsiteIndex;
80+
81+
SmallVector<uint64_t, 1> RecordValues;
82+
83+
// We don't prescribe the order in which the records come in, and we are ok
84+
// if other unsupported records appear. We seek in the current subblock until
85+
// we get all we know.
86+
while (!Guid || !Counters || (ExpectIndex && !CallsiteIndex)) {
87+
RecordValues.clear();
88+
EXPECT_OR_RET(Entry, advance());
89+
if (Entry->Kind != BitstreamEntry::Record)
90+
return unsupported(
91+
"Expected records before encountering more subcontexts");
92+
EXPECT_OR_RET(ReadRecord,
93+
Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
94+
switch (*ReadRecord) {
95+
case PGOCtxProfileRecords::Guid: {
96+
if (RecordValues.size() != 1)
97+
return wrongValue("The GUID record should have exactly one value");
98+
Guid = RecordValues[0];
99+
break;
100+
}
101+
case PGOCtxProfileRecords::Counters:
102+
Counters = std::move(RecordValues);
103+
if (Counters->empty())
104+
return wrongValue("Empty counters. At least the entry counter (one "
105+
"value) was expected");
106+
break;
107+
case PGOCtxProfileRecords::CalleeIndex: {
108+
if (!ExpectIndex)
109+
return wrongValue("The root context should not have a callee index");
110+
if (RecordValues.size() != 1)
111+
return wrongValue("The callee index should have exactly one value");
112+
CallsiteIndex = RecordValues[0];
113+
break;
114+
}
115+
default:
116+
// OK if we see records we do not understand.
117+
break;
118+
}
119+
}
120+
121+
PGOContextualProfile Ret(*Guid, std::move(*Counters));
122+
123+
while (canReadContext()) {
124+
EXPECT_OR_RET(SC, readContext(true));
125+
if (!Ret.callsites()[*SC->first]
126+
.insert({SC->second.guid(), std::move(SC->second)})
127+
.second)
128+
return wrongValue("Duplicate");
129+
}
130+
return std::make_pair(CallsiteIndex, std::move(Ret));
131+
}
132+
133+
Error PGOCtxProfileReader::readMetadata() {
134+
EXPECT_OR_RET(Blk, advance());
135+
if (Blk->Kind != BitstreamEntry::SubBlock)
136+
return unsupported("Expected Version record");
137+
RET_ON_ERR(
138+
Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
139+
EXPECT_OR_RET(MData, advance());
140+
if (MData->Kind != BitstreamEntry::Record)
141+
return unsupported("Expected Version record");
142+
143+
SmallVector<uint64_t, 1> Ver;
144+
EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
145+
if (*Code != PGOCtxProfileRecords::Version)
146+
return unsupported("Expected Version record");
147+
if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
148+
return unsupported("Version " + std::to_string(*Code) +
149+
" is higher than supported version " +
150+
std::to_string(PGOCtxProfileWriter::CurrentVersion));
151+
return Error::success();
152+
}
153+
154+
Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
155+
PGOCtxProfileReader::loadContexts() {
156+
std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
157+
RET_ON_ERR(readMetadata());
158+
while (canReadContext()) {
159+
EXPECT_OR_RET(E, readContext(false));
160+
auto Key = E->second.guid();
161+
if (!Ret.insert({Key, std::move(E->second)}).second)
162+
return wrongValue("Duplicate roots");
163+
}
164+
return Ret;
165+
}

0 commit comments

Comments
 (0)