Skip to content

Commit cc66f6c

Browse files
victoria-mcgrathguptask
victoria-mcgrath
authored and
guptask
committed
Fixed total cache size in CacheMemoryStats (#38)
Return a sum of sizes of each tier instead of just 1st tier's size.
1 parent 414ab78 commit cc66f6c

File tree

2 files changed

+162
-86
lines changed

2 files changed

+162
-86
lines changed

‎cachelib/allocator/CacheAllocator-inl.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3834,7 +3834,10 @@ GlobalCacheStats CacheAllocator<CacheTrait>::getGlobalCacheStats() const {
38343834

38353835
template <typename CacheTrait>
38363836
CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
3837-
const auto totalCacheSize = allocator_[currentTier()]->getMemorySize();
3837+
size_t totalCacheSize = 0;
3838+
for(auto& allocator: allocator_) {
3839+
totalCacheSize += allocator->getMemorySize();
3840+
}
38383841

38393842
auto addSize = [this](size_t a, PoolId pid) {
38403843
return a + allocator_[currentTier()]->getPool(pid).getPoolSize();

‎cachelib/allocator/tests/MemoryTiersTest.cpp

Lines changed: 158 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
*/
1616

1717
#include <numeric>
18+
1819
#include "cachelib/allocator/CacheAllocator.h"
1920
#include "cachelib/allocator/tests/TestBase.h"
2021

2122
namespace facebook {
2223
namespace cachelib {
2324
namespace tests {
2425

25-
2626
using LruAllocatorConfig = CacheAllocatorConfig<LruAllocator>;
2727
using LruMemoryTierConfigs = LruAllocatorConfig::MemoryTierConfigs;
2828
using Strings = std::vector<std::string>;
@@ -34,64 +34,96 @@ const std::string defaultCacheDir{"/var/metadataDir"};
3434
const std::string defaultPmemPath{"/dev/shm/p1"};
3535
const std::string defaultDaxPath{"/dev/dax0.0"};
3636

37+
const size_t metaDataSize = 4194304;
38+
constexpr size_t MB = 1024ULL * 1024ULL;
39+
constexpr size_t GB = MB * 1024ULL;
40+
3741
template <typename Allocator>
38-
class MemoryTiersTest: public AllocatorTest<Allocator> {
39-
public:
40-
void basicCheck(
41-
LruAllocatorConfig& actualConfig,
42-
const Strings& expectedPaths = {defaultPmemPath},
43-
size_t expectedTotalCacheSize = defaultTotalCacheSize,
44-
const std::string& expectedCacheDir = defaultCacheDir) {
45-
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
46-
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
47-
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
48-
auto configs = actualConfig.getMemoryTierConfigs();
49-
50-
size_t sum_sizes = std::accumulate(configs.begin(), configs.end(), 0,
51-
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getSize();});
52-
size_t sum_ratios = std::accumulate(configs.begin(), configs.end(), 0,
53-
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getRatio();});
54-
55-
size_t partition_size = 0;
56-
if (sum_ratios) {
57-
partition_size = actualConfig.getCacheSize() / sum_ratios;
58-
/* Sum of sizes can be lower due to rounding down to partition_size. */
59-
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
60-
}
42+
class MemoryTiersTest : public AllocatorTest<Allocator> {
43+
public:
44+
void basicCheck(LruAllocatorConfig& actualConfig,
45+
const Strings& expectedPaths = {defaultPmemPath},
46+
size_t expectedTotalCacheSize = defaultTotalCacheSize,
47+
const std::string& expectedCacheDir = defaultCacheDir) {
48+
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
49+
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
50+
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
51+
auto configs = actualConfig.getMemoryTierConfigs();
6152

62-
for(auto i = 0; i < configs.size(); ++i) {
63-
auto &opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
64-
EXPECT_EQ(opt.path, expectedPaths[i]);
65-
EXPECT_GT(configs[i].getSize(), 0);
66-
if (configs[i].getRatio() && (i < configs.size() - 1)) {
67-
EXPECT_EQ(configs[i].getSize(), partition_size * configs[i].getRatio());
68-
}
69-
}
53+
size_t sum_sizes = std::accumulate(
54+
configs.begin(), configs.end(), 0,
55+
[](const size_t i, const MemoryTierCacheConfig& config) {
56+
return i + config.getSize();
57+
});
58+
size_t sum_ratios = std::accumulate(
59+
configs.begin(), configs.end(), 0,
60+
[](const size_t i, const MemoryTierCacheConfig& config) {
61+
return i + config.getRatio();
62+
});
63+
64+
size_t partition_size = 0;
65+
if (sum_ratios) {
66+
partition_size = actualConfig.getCacheSize() / sum_ratios;
67+
/* Sum of sizes can be lower due to rounding down to partition_size. */
68+
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
7069
}
7170

72-
LruAllocatorConfig createTestCacheConfig(
73-
const Strings& tierPaths = {defaultPmemPath},
74-
const SizePairs& sizePairs = {std::make_tuple(1 /* ratio */, 0 /* size */)},
75-
bool setPosixForShm = true,
76-
size_t cacheSize = defaultTotalCacheSize,
77-
const std::string& cacheDir = defaultCacheDir) {
78-
LruAllocatorConfig cfg;
79-
cfg.setCacheSize(cacheSize)
80-
.enableCachePersistence(cacheDir);
81-
82-
if (setPosixForShm)
83-
cfg.usePosixForShm();
84-
85-
LruMemoryTierConfigs tierConfigs;
86-
tierConfigs.reserve(tierPaths.size());
87-
for(auto i = 0; i < tierPaths.size(); ++i) {
88-
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
89-
.setRatio(std::get<0>(sizePairs[i]))
90-
.setSize(std::get<1>(sizePairs[i])));
71+
for (auto i = 0; i < configs.size(); ++i) {
72+
auto& opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
73+
EXPECT_EQ(opt.path, expectedPaths[i]);
74+
EXPECT_GT(configs[i].getSize(), 0);
75+
if (configs[i].getRatio() && (i < configs.size() - 1)) {
76+
EXPECT_EQ(configs[i].getSize(), partition_size * configs[i].getRatio());
9177
}
92-
cfg.configureMemoryTiers(tierConfigs);
93-
return cfg;
9478
}
79+
}
80+
81+
LruAllocatorConfig createTestCacheConfig(
82+
const Strings& tierPaths = {defaultPmemPath},
83+
const SizePairs& sizePairs = {std::make_tuple(1 /* ratio */,
84+
0 /* size */)},
85+
bool setPosixForShm = true,
86+
size_t cacheSize = defaultTotalCacheSize,
87+
const std::string& cacheDir = defaultCacheDir) {
88+
LruAllocatorConfig cfg;
89+
cfg.setCacheSize(cacheSize).enableCachePersistence(cacheDir);
90+
91+
if (setPosixForShm)
92+
cfg.usePosixForShm();
93+
94+
LruMemoryTierConfigs tierConfigs;
95+
tierConfigs.reserve(tierPaths.size());
96+
for (auto i = 0; i < tierPaths.size(); ++i) {
97+
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
98+
.setRatio(std::get<0>(sizePairs[i]))
99+
.setSize(std::get<1>(sizePairs[i])));
100+
}
101+
cfg.configureMemoryTiers(tierConfigs);
102+
return cfg;
103+
}
104+
105+
LruAllocatorConfig createTieredCacheConfig(size_t totalCacheSize,
106+
size_t numTiers = 2) {
107+
LruAllocatorConfig tieredCacheConfig{};
108+
std::vector<MemoryTierCacheConfig> configs;
109+
for (auto i = 1; i <= numTiers; ++i) {
110+
configs.push_back(MemoryTierCacheConfig::fromFile(
111+
folly::sformat("/tmp/tier{}-{}", i, ::getpid()))
112+
.setRatio(1));
113+
}
114+
tieredCacheConfig.setCacheSize(totalCacheSize)
115+
.enableCachePersistence(
116+
folly::sformat("/tmp/multi-tier-test/{}", ::getpid()))
117+
.usePosixForShm()
118+
.configureMemoryTiers(configs);
119+
return tieredCacheConfig;
120+
}
121+
122+
LruAllocatorConfig createDramCacheConfig(size_t totalCacheSize) {
123+
LruAllocatorConfig dramConfig{};
124+
dramConfig.setCacheSize(totalCacheSize);
125+
return dramConfig;
126+
}
95127
};
96128

97129
using LruMemoryTiersTest = MemoryTiersTest<LruAllocator>;
@@ -107,77 +139,118 @@ TEST_F(LruMemoryTiersTest, TestValid1TierDaxRatioConfig) {
107139
}
108140

109141
TEST_F(LruMemoryTiersTest, TestValid1TierDaxSizeConfig) {
110-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath},
111-
{std::make_tuple(0, defaultTotalCacheSize)},
112-
/* setPosixShm */ true,
113-
/* cacheSize */ 0);
142+
LruAllocatorConfig cfg =
143+
createTestCacheConfig({defaultDaxPath},
144+
{std::make_tuple(0, defaultTotalCacheSize)},
145+
/* setPosixShm */ true,
146+
/* cacheSize */ 0);
114147
basicCheck(cfg, {defaultDaxPath});
115148

116149
// Setting size after conifguringMemoryTiers with sizes is not allowed.
117-
EXPECT_THROW(cfg.setCacheSize(defaultTotalCacheSize + 1), std::invalid_argument);
150+
EXPECT_THROW(cfg.setCacheSize(defaultTotalCacheSize + 1),
151+
std::invalid_argument);
118152
}
119153

120154
TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemConfig) {
121-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
122-
{std::make_tuple(1, 0), std::make_tuple(1, 0)});
155+
LruAllocatorConfig cfg =
156+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
157+
{std::make_tuple(1, 0), std::make_tuple(1, 0)});
123158
basicCheck(cfg, {defaultDaxPath, defaultPmemPath});
124159
}
125160

126161
TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemRatioConfig) {
127-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
128-
{std::make_tuple(5, 0), std::make_tuple(2, 0)});
162+
LruAllocatorConfig cfg =
163+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
164+
{std::make_tuple(5, 0), std::make_tuple(2, 0)});
129165
basicCheck(cfg, {defaultDaxPath, defaultPmemPath});
130166
}
131167

132168
TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemSizeConfig) {
133169
size_t size_1 = 4321, size_2 = 1234;
134-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
135-
{std::make_tuple(0, size_1), std::make_tuple(0, size_2)},
136-
true, 0);
170+
LruAllocatorConfig cfg = createTestCacheConfig(
171+
{defaultDaxPath, defaultPmemPath},
172+
{std::make_tuple(0, size_1), std::make_tuple(0, size_2)}, true, 0);
137173
basicCheck(cfg, {defaultDaxPath, defaultPmemPath}, size_1 + size_2);
138174

139175
// Setting size after conifguringMemoryTiers with sizes is not allowed.
140176
EXPECT_THROW(cfg.setCacheSize(size_1 + size_2 + 1), std::invalid_argument);
141177
}
142178

143179
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigPosixShmNotSet) {
144-
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
145-
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
146-
/* setPosixShm */ false);
180+
LruAllocatorConfig cfg =
181+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
182+
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
183+
/* setPosixShm */ false);
147184
}
148185

149186
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigNumberOfPartitionsTooLarge) {
150187
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
151-
{std::make_tuple(defaultTotalCacheSize, 0), std::make_tuple(1, 0)}).validate(),
188+
{std::make_tuple(defaultTotalCacheSize, 0),
189+
std::make_tuple(1, 0)})
190+
.validate(),
152191
std::invalid_argument);
153192
}
154193

155194
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesAndRatiosMixed) {
156-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
157-
{std::make_tuple(1, 0), std::make_tuple(1, 1)}),
158-
std::invalid_argument);
159-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
160-
{std::make_tuple(1, 1), std::make_tuple(0, 1)}),
161-
std::invalid_argument);
195+
EXPECT_THROW(
196+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
197+
{std::make_tuple(1, 0), std::make_tuple(1, 1)}),
198+
std::invalid_argument);
199+
EXPECT_THROW(
200+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
201+
{std::make_tuple(1, 1), std::make_tuple(0, 1)}),
202+
std::invalid_argument);
162203
}
163204

164205
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesAndRatioNotSet) {
165-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
166-
{std::make_tuple(1, 0), std::make_tuple(0, 0)}),
167-
std::invalid_argument);
206+
EXPECT_THROW(
207+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
208+
{std::make_tuple(1, 0), std::make_tuple(0, 0)}),
209+
std::invalid_argument);
168210
}
169211

170212
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatiosCacheSizeNotSet) {
171-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
172-
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
173-
/* setPosixShm */ true, /* cacheSize */ 0).validate(),
174-
std::invalid_argument);
213+
EXPECT_THROW(
214+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
215+
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
216+
/* setPosixShm */ true, /* cacheSize */ 0)
217+
.validate(),
218+
std::invalid_argument);
175219
}
176220

177221
TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) {
178-
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
179-
{std::make_tuple(0, 1), std::make_tuple(0, 1)}),
180-
std::invalid_argument);
222+
EXPECT_THROW(
223+
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
224+
{std::make_tuple(0, 1), std::make_tuple(0, 1)}),
225+
std::invalid_argument);
226+
}
227+
228+
TEST_F(LruMemoryTiersTest, TestTieredCacheSize) {
229+
size_t totalSizes[] = {50 * MB, 77 * MB, 100 * MB, 101 * MB + MB / 2,
230+
1 * GB, 4 * GB, 8 * GB, 9 * GB};
231+
size_t numTiers[] = {2, 3, 4};
232+
233+
auto getCacheSize = [&](size_t cacheSize, size_t tiers) {
234+
std::unique_ptr<LruAllocator> alloc;
235+
if (tiers < 2) {
236+
alloc = std::unique_ptr<LruAllocator>(
237+
new LruAllocator(createDramCacheConfig(cacheSize)));
238+
} else {
239+
alloc = std::unique_ptr<LruAllocator>(
240+
new LruAllocator(LruAllocator::SharedMemNew,
241+
createTieredCacheConfig(cacheSize, tiers)));
242+
}
243+
return alloc->getCacheMemoryStats().cacheSize;
244+
};
245+
246+
for (auto totalSize : totalSizes) {
247+
auto dramCacheSize = getCacheSize(totalSize, 1);
248+
for (auto n : numTiers) {
249+
auto tieredCacheSize = getCacheSize(totalSize, n);
250+
EXPECT_GT(dramCacheSize, tieredCacheSize);
251+
EXPECT_GE(metaDataSize * n * 2, dramCacheSize - tieredCacheSize);
252+
}
253+
}
181254
}
182255

183256
} // namespace tests

0 commit comments

Comments
 (0)