Skip to content

Fixed total cache size in CacheMemoryStats #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 8, 2022
Merged

Fixed total cache size in CacheMemoryStats #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c879923
Fixed total cache size in stats
victoria-mcgrath Feb 3, 2022
6cadd4b
Added tests for cache size changes
victoria-mcgrath Feb 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion cachelib/allocator/CacheAllocator-inl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3610,7 +3610,10 @@ GlobalCacheStats CacheAllocator<CacheTrait>::getGlobalCacheStats() const {

template <typename CacheTrait>
CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
const auto totalCacheSize = allocator_[currentTier()]->getMemorySize();
size_t totalCacheSize = 0;
for(auto& allocator: allocator_) {
totalCacheSize += allocator->getMemorySize();
}

auto addSize = [this](size_t a, PoolId pid) {
return a + allocator_[currentTier()]->getPool(pid).getPoolSize();
Expand Down
243 changes: 158 additions & 85 deletions cachelib/allocator/tests/MemoryTiersTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@
*/

#include <numeric>

#include "cachelib/allocator/CacheAllocator.h"
#include "cachelib/allocator/tests/TestBase.h"

namespace facebook {
namespace cachelib {
namespace tests {


using LruAllocatorConfig = CacheAllocatorConfig<LruAllocator>;
using LruMemoryTierConfigs = LruAllocatorConfig::MemoryTierConfigs;
using Strings = std::vector<std::string>;
Expand All @@ -34,64 +34,96 @@ const std::string defaultCacheDir{"/var/metadataDir"};
const std::string defaultPmemPath{"/dev/shm/p1"};
const std::string defaultDaxPath{"/dev/dax0.0"};

const size_t metaDataSize = 4194304;
constexpr size_t MB = 1024ULL * 1024ULL;
constexpr size_t GB = MB * 1024ULL;

template <typename Allocator>
class MemoryTiersTest: public AllocatorTest<Allocator> {
public:
void basicCheck(
LruAllocatorConfig& actualConfig,
const Strings& expectedPaths = {defaultPmemPath},
size_t expectedTotalCacheSize = defaultTotalCacheSize,
const std::string& expectedCacheDir = defaultCacheDir) {
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
auto configs = actualConfig.getMemoryTierConfigs();

size_t sum_sizes = std::accumulate(configs.begin(), configs.end(), 0,
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getSize();});
size_t sum_ratios = std::accumulate(configs.begin(), configs.end(), 0,
[](const size_t i, const MemoryTierCacheConfig& config) { return i + config.getRatio();});

size_t partition_size = 0;
if (sum_ratios) {
partition_size = actualConfig.getCacheSize() / sum_ratios;
/* Sum of sizes can be lower due to rounding down to partition_size. */
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
}
class MemoryTiersTest : public AllocatorTest<Allocator> {
public:
void basicCheck(LruAllocatorConfig& actualConfig,
const Strings& expectedPaths = {defaultPmemPath},
size_t expectedTotalCacheSize = defaultTotalCacheSize,
const std::string& expectedCacheDir = defaultCacheDir) {
EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
EXPECT_EQ(actualConfig.getMemoryTierConfigs().size(), expectedPaths.size());
EXPECT_EQ(actualConfig.getCacheDir(), expectedCacheDir);
auto configs = actualConfig.getMemoryTierConfigs();

for(auto i = 0; i < configs.size(); ++i) {
auto &opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
EXPECT_EQ(opt.path, expectedPaths[i]);
EXPECT_GT(configs[i].getSize(), 0);
if (configs[i].getRatio() && (i < configs.size() - 1)) {
EXPECT_EQ(configs[i].getSize(), partition_size * configs[i].getRatio());
}
}
size_t sum_sizes = std::accumulate(
configs.begin(), configs.end(), 0,
[](const size_t i, const MemoryTierCacheConfig& config) {
return i + config.getSize();
});
size_t sum_ratios = std::accumulate(
configs.begin(), configs.end(), 0,
[](const size_t i, const MemoryTierCacheConfig& config) {
return i + config.getRatio();
});

size_t partition_size = 0;
if (sum_ratios) {
partition_size = actualConfig.getCacheSize() / sum_ratios;
/* Sum of sizes can be lower due to rounding down to partition_size. */
EXPECT_GE(sum_sizes, expectedTotalCacheSize - partition_size);
}

LruAllocatorConfig createTestCacheConfig(
const Strings& tierPaths = {defaultPmemPath},
const SizePairs& sizePairs = {std::make_tuple(1 /* ratio */, 0 /* size */)},
bool setPosixForShm = true,
size_t cacheSize = defaultTotalCacheSize,
const std::string& cacheDir = defaultCacheDir) {
LruAllocatorConfig cfg;
cfg.setCacheSize(cacheSize)
.enableCachePersistence(cacheDir);

if (setPosixForShm)
cfg.usePosixForShm();

LruMemoryTierConfigs tierConfigs;
tierConfigs.reserve(tierPaths.size());
for(auto i = 0; i < tierPaths.size(); ++i) {
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
.setRatio(std::get<0>(sizePairs[i]))
.setSize(std::get<1>(sizePairs[i])));
for (auto i = 0; i < configs.size(); ++i) {
auto& opt = std::get<FileShmSegmentOpts>(configs[i].getShmTypeOpts());
EXPECT_EQ(opt.path, expectedPaths[i]);
EXPECT_GT(configs[i].getSize(), 0);
if (configs[i].getRatio() && (i < configs.size() - 1)) {
EXPECT_EQ(configs[i].getSize(), partition_size * configs[i].getRatio());
}
cfg.configureMemoryTiers(tierConfigs);
return cfg;
}
}

LruAllocatorConfig createTestCacheConfig(
const Strings& tierPaths = {defaultPmemPath},
const SizePairs& sizePairs = {std::make_tuple(1 /* ratio */,
0 /* size */)},
bool setPosixForShm = true,
size_t cacheSize = defaultTotalCacheSize,
const std::string& cacheDir = defaultCacheDir) {
LruAllocatorConfig cfg;
cfg.setCacheSize(cacheSize).enableCachePersistence(cacheDir);

if (setPosixForShm)
cfg.usePosixForShm();

LruMemoryTierConfigs tierConfigs;
tierConfigs.reserve(tierPaths.size());
for (auto i = 0; i < tierPaths.size(); ++i) {
tierConfigs.push_back(MemoryTierCacheConfig::fromFile(tierPaths[i])
.setRatio(std::get<0>(sizePairs[i]))
.setSize(std::get<1>(sizePairs[i])));
}
cfg.configureMemoryTiers(tierConfigs);
return cfg;
}

LruAllocatorConfig createTieredCacheConfig(size_t totalCacheSize,
size_t numTiers = 2) {
LruAllocatorConfig tieredCacheConfig{};
std::vector<MemoryTierCacheConfig> configs;
for (auto i = 1; i <= numTiers; ++i) {
configs.push_back(MemoryTierCacheConfig::fromFile(
folly::sformat("/tmp/tier{}-{}", i, ::getpid()))
.setRatio(1));
}
tieredCacheConfig.setCacheSize(totalCacheSize)
.enableCachePersistence(
folly::sformat("/tmp/multi-tier-test/{}", ::getpid()))
.usePosixForShm()
.configureMemoryTiers(configs);
return tieredCacheConfig;
}

LruAllocatorConfig createDramCacheConfig(size_t totalCacheSize) {
LruAllocatorConfig dramConfig{};
dramConfig.setCacheSize(totalCacheSize);
return dramConfig;
}
};

using LruMemoryTiersTest = MemoryTiersTest<LruAllocator>;
Expand All @@ -107,77 +139,118 @@ TEST_F(LruMemoryTiersTest, TestValid1TierDaxRatioConfig) {
}

TEST_F(LruMemoryTiersTest, TestValid1TierDaxSizeConfig) {
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath},
{std::make_tuple(0, defaultTotalCacheSize)},
/* setPosixShm */ true,
/* cacheSize */ 0);
LruAllocatorConfig cfg =
createTestCacheConfig({defaultDaxPath},
{std::make_tuple(0, defaultTotalCacheSize)},
/* setPosixShm */ true,
/* cacheSize */ 0);
basicCheck(cfg, {defaultDaxPath});

// Setting size after conifguringMemoryTiers with sizes is not allowed.
EXPECT_THROW(cfg.setCacheSize(defaultTotalCacheSize + 1), std::invalid_argument);
EXPECT_THROW(cfg.setCacheSize(defaultTotalCacheSize + 1),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemConfig) {
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)});
LruAllocatorConfig cfg =
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)});
basicCheck(cfg, {defaultDaxPath, defaultPmemPath});
}

TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemRatioConfig) {
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(5, 0), std::make_tuple(2, 0)});
LruAllocatorConfig cfg =
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(5, 0), std::make_tuple(2, 0)});
basicCheck(cfg, {defaultDaxPath, defaultPmemPath});
}

TEST_F(LruMemoryTiersTest, TestValid2TierDaxPmemSizeConfig) {
size_t size_1 = 4321, size_2 = 1234;
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(0, size_1), std::make_tuple(0, size_2)},
true, 0);
LruAllocatorConfig cfg = createTestCacheConfig(
{defaultDaxPath, defaultPmemPath},
{std::make_tuple(0, size_1), std::make_tuple(0, size_2)}, true, 0);
basicCheck(cfg, {defaultDaxPath, defaultPmemPath}, size_1 + size_2);

// Setting size after conifguringMemoryTiers with sizes is not allowed.
EXPECT_THROW(cfg.setCacheSize(size_1 + size_2 + 1), std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigPosixShmNotSet) {
LruAllocatorConfig cfg = createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
/* setPosixShm */ false);
LruAllocatorConfig cfg =
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
/* setPosixShm */ false);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigNumberOfPartitionsTooLarge) {
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(defaultTotalCacheSize, 0), std::make_tuple(1, 0)}).validate(),
{std::make_tuple(defaultTotalCacheSize, 0),
std::make_tuple(1, 0)})
.validate(),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesAndRatiosMixed) {
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 1)}),
std::invalid_argument);
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 1), std::make_tuple(0, 1)}),
std::invalid_argument);
EXPECT_THROW(
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 1)}),
std::invalid_argument);
EXPECT_THROW(
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 1), std::make_tuple(0, 1)}),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesAndRatioNotSet) {
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(0, 0)}),
std::invalid_argument);
EXPECT_THROW(
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(0, 0)}),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatiosCacheSizeNotSet) {
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
/* setPosixShm */ true, /* cacheSize */ 0).validate(),
std::invalid_argument);
EXPECT_THROW(
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(1, 0), std::make_tuple(1, 0)},
/* setPosixShm */ true, /* cacheSize */ 0)
.validate(),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) {
EXPECT_THROW(createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(0, 1), std::make_tuple(0, 1)}),
std::invalid_argument);
EXPECT_THROW(
createTestCacheConfig({defaultDaxPath, defaultPmemPath},
{std::make_tuple(0, 1), std::make_tuple(0, 1)}),
std::invalid_argument);
}

TEST_F(LruMemoryTiersTest, TestTieredCacheSize) {
size_t totalSizes[] = {50 * MB, 77 * MB, 100 * MB, 101 * MB + MB / 2,
1 * GB, 4 * GB, 8 * GB, 9 * GB};
size_t numTiers[] = {2, 3, 4};

auto getCacheSize = [&](size_t cacheSize, size_t tiers) {
std::unique_ptr<LruAllocator> alloc;
if (tiers < 2) {
alloc = std::unique_ptr<LruAllocator>(
new LruAllocator(createDramCacheConfig(cacheSize)));
} else {
alloc = std::unique_ptr<LruAllocator>(
new LruAllocator(LruAllocator::SharedMemNew,
createTieredCacheConfig(cacheSize, tiers)));
}
return alloc->getCacheMemoryStats().cacheSize;
};

for (auto totalSize : totalSizes) {
auto dramCacheSize = getCacheSize(totalSize, 1);
for (auto n : numTiers) {
auto tieredCacheSize = getCacheSize(totalSize, n);
EXPECT_GT(dramCacheSize, tieredCacheSize);
EXPECT_GE(metaDataSize * n * 2, dramCacheSize - tieredCacheSize);
}
}
}

} // namespace tests
Expand Down