Publish changes from innersouce

cachelib/allocator/CMakeLists.txt

@@ -109,6 +109,7 @@ if (BUILD_TESTS)
   add_test (tests/ChainedHashTest.cpp)
   add_test (tests/AllocatorResizeTypeTest.cpp)
   add_test (tests/AllocatorHitStatsTypeTest.cpp)
+  add_test (tests/MemoryTiersTest.cpp)
   add_test (tests/MultiAllocatorTest.cpp)
   add_test (tests/NvmAdmissionPolicyTest.cpp)
   add_test (nvmcache/tests/NvmItemTests.cpp)

cachelib/allocator/CacheAllocator-inl.h

@@ -68,7 +68,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.accessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                              false, config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -79,7 +80,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.chainedItemAccessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                              false, config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -89,7 +91,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
       nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(),
                      config_.isNvmCacheTruncateAllocSizeEnabled()} {
   initCommon(false);
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+    PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -107,13 +110,15 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
       accessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.accessConfig,
-          shmManager_->attachShm(detail::kShmHashTableName),
+          shmManager_->attachShm(detail::kShmHashTableName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemAccessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.chainedItemAccessConfig,
-          shmManager_->attachShm(detail::kShmChainedItemHashTableName),
+          shmManager_->attachShm(detail::kShmChainedItemHashTableName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemLocks_(config_.chainedItemsLockPower,
@@ -130,7 +135,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
   // We will create a new info shm segment on shutDown(). If we don't remove
   // this info shm segment here and the new info shm segment's size is larger
   // than this one, creating new one will fail.
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+    PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -148,6 +154,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::createNewMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       getAllocatorConfig(config_),
       shmManager_
@@ -162,6 +169,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::restoreMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       deserializer_->deserialize<MemoryAllocator::SerializationType>(),
       shmManager_
@@ -265,7 +273,8 @@ void CacheAllocator<CacheTrait>::initWorkers() {
 
 template <typename CacheTrait>
 std::unique_ptr<Deserializer> CacheAllocator<CacheTrait>::createDeserializer() {
-  auto infoAddr = shmManager_->attachShm(detail::kShmInfoName);
+  auto infoAddr = shmManager_->attachShm(detail::kShmInfoName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm));
   return std::make_unique<Deserializer>(
       reinterpret_cast<uint8_t*>(infoAddr.addr),
       reinterpret_cast<uint8_t*>(infoAddr.addr) + infoAddr.size);
@@ -3041,8 +3050,11 @@ void CacheAllocator<CacheTrait>::saveRamCache() {
   std::unique_ptr<folly::IOBuf> ioBuf = serializedBuf.move();
   ioBuf->coalesce();
 
-  void* infoAddr =
-      shmManager_->createShm(detail::kShmInfoName, ioBuf->length()).addr;
+  ShmSegmentOpts opts;
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
+
+  void* infoAddr = shmManager_->createShm(detail::kShmInfoName, ioBuf->length(),
+      nullptr, opts).addr;
   Serializer serializer(reinterpret_cast<uint8_t*>(infoAddr),
                         reinterpret_cast<uint8_t*>(infoAddr) + ioBuf->length());
   serializer.writeToBuffer(std::move(ioBuf));
@@ -3386,7 +3398,7 @@ bool CacheAllocator<CacheTrait>::stopReaper(std::chrono::seconds timeout) {
 
 template <typename CacheTrait>
 bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
-    const std::string& cacheDir, bool posix) {
+  const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector<CacheMemoryTierConfig>& config */) {
   if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) {
     try {
       // cache dir exists. clean up only if there are no other processes
@@ -3405,6 +3417,12 @@ bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
     ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix);
     ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName,
                              posix);
+
+    // TODO(SHM_FILE): try to nuke segments of differente types (which require
+    // extra info)
+    // for (auto &tier : config) {
+    //   ShmManager::removeByName(cacheDir, tierShmName, config_.memoryTiers[i].opts);
+    // }
   }
   return true;
 }

cachelib/allocator/CacheAllocator.h

@@ -1035,7 +1035,8 @@ class CacheAllocator : public CacheBase {
   // returns true if there was no error in trying to cleanup the segment
   // because another process was attached. False if the user tried to clean up
   // and the cache was actually attached.
-  static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix);
+  static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix
+    /*TODO: const std::vector<CacheMemoryTierConfig>& config = {} */);
 
   // gives a relative offset to a pointer within the cache.
   uint64_t getItemPtrAsOffset(const void* ptr);

cachelib/allocator/CacheAllocatorConfig.h

@@ -25,6 +25,7 @@
 #include <string>
 
 #include "cachelib/allocator/Cache.h"
+#include "cachelib/allocator/MemoryTierCacheConfig.h"
 #include "cachelib/allocator/MM2Q.h"
 #include "cachelib/allocator/MemoryMonitor.h"
 #include "cachelib/allocator/NvmAdmissionPolicy.h"
@@ -49,6 +50,7 @@ class CacheAllocatorConfig {
   using NvmCacheDeviceEncryptor = typename CacheT::NvmCacheT::DeviceEncryptor;
   using MoveCb = typename CacheT::MoveCb;
   using NvmCacheConfig = typename CacheT::NvmCacheT::Config;
+  using MemoryTierConfigs = std::vector<MemoryTierCacheConfig>;
   using Key = typename CacheT::Key;
   using EventTrackerSharedPtr = std::shared_ptr<typename CacheT::EventTracker>;
   using Item = typename CacheT::Item;
@@ -186,14 +188,23 @@ class CacheAllocatorConfig {
   // This allows cache to be persisted across restarts. One example use case is
   // to preserve the cache when releasing a new version of your service. Refer
   // to our user guide for how to set up cache persistence.
+  // TODO: get rid of baseAddr or if set make sure all mapping are adjacent?
+  // We can also make baseAddr a per-tier configuration
   CacheAllocatorConfig& enableCachePersistence(std::string directory,
                                                void* baseAddr = nullptr);
 
-  // uses posix shm segments instead of the default sys-v shm segments.
-  // @throw std::invalid_argument if called without enabling
-  // cachePersistence()
+  // Uses posix shm segments instead of the default sys-v shm
+  // segments. @throw std::invalid_argument if called without enabling
+  // cachePersistence().
   CacheAllocatorConfig& usePosixForShm();
 
+  // Configures cache memory tiers. Accepts vector of MemoryTierCacheConfig.
+  // Each vector element describes configuration for a single memory cache tier.
+  CacheAllocatorConfig& configureMemoryTiers(const MemoryTierConfigs& configs);
+
+  // Return reference to MemoryTierCacheConfigs.
+  const MemoryTierConfigs& getMemoryTierConfigs();
+
   // This turns on a background worker that periodically scans through the
   // access container and look for expired items and remove them.
   CacheAllocatorConfig& enableItemReaperInBackground(
@@ -541,6 +552,9 @@ class CacheAllocatorConfig {
   // cache.
   uint64_t nvmAdmissionMinTTL{0};
 
+  // Configuration for memory tiers.
+  MemoryTierConfigs memoryTierConfigs;
+
   friend CacheT;
 
  private:
@@ -801,6 +815,74 @@ CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::enableItemReaperInBackground(
   return *this;
 }
 
+template <typename T>
+CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::configureMemoryTiers(
+      const MemoryTierConfigs& config) {
+  memoryTierConfigs = config;
+  size_t sum_ratios = 0;
+  size_t sum_sizes = 0;
+
+  for (auto tier_config: memoryTierConfigs) {
+    auto tier_size = tier_config.getSize();
+    auto tier_ratio = tier_config.getRatio();
+    if ((!tier_size and !tier_ratio) || (tier_size and tier_ratio)) {
+      throw std::invalid_argument(
+        "For each memory tier either size or ratio must be set.");
+    }
+    sum_ratios += tier_ratio;
+    sum_sizes += tier_size;
+  }
+
+  if (sum_ratios) {
+    if (!getCacheSize()) {
+      throw std::invalid_argument(
+          "Total cache size must be specified when size ratios are \
+          used to specify memory tier sizes.");
+    } else {
+      if (getCacheSize() < sum_ratios) {
+        throw std::invalid_argument(
+          "Sum of all tier size ratios is greater than total cache size.");
+      }
+      // Convert ratios to sizes
+      sum_sizes = 0;
+      size_t partition_size = getCacheSize() / sum_ratios;
+      for (auto& tier_config: memoryTierConfigs) {
+        tier_config.setSize(partition_size * tier_config.getRatio());
+        sum_sizes += tier_config.getSize();
+      }
+      if (getCacheSize() != sum_sizes) {
+        // Adjust capacity of the last tier to account for rounding error
+        memoryTierConfigs.back().setSize(memoryTierConfigs.back().getSize() + \
+                                         (getCacheSize() - sum_sizes));
+        sum_sizes = getCacheSize();
+      }
+    }
+  } else if (sum_sizes) {
+    if (getCacheSize() && sum_sizes != getCacheSize()) {
+      throw std::invalid_argument(
+          "Sum of tier sizes doesn't match total cache size. \
+          Setting of cache total size is not required when per-tier \
+          sizes are specified - it is calculated as sum of tier sizes.");
+    }
+  } else {
+    throw std::invalid_argument(
+      "Either sum of all memory tiers sizes or sum of all ratios \
+      must be greater than 0.");
+  }
+
+  if (sum_sizes && !getCacheSize()) {
+    setCacheSize(sum_sizes);
+  }
+
+  return *this;
+}
+
+//const std::vector<MemoryTierCacheConfig>& CacheAllocatorConfig<T>::getMemoryTierConfigs() {
+template <typename T>
+const typename CacheAllocatorConfig<T>::MemoryTierConfigs& CacheAllocatorConfig<T>::getMemoryTierConfigs() {
+  return memoryTierConfigs;
+}
+
 template <typename T>
 CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::disableCacheEviction() {
   disableEviction = true;
@@ -970,7 +1052,7 @@ std::map<std::string, std::string> CacheAllocatorConfig<T>::serialize() const {
 
   configMap["size"] = std::to_string(size);
   configMap["cacheDir"] = cacheDir;
-  configMap["posixShm"] = usePosixShm ? "set" : "empty";
+  configMap["posixShm"] = isUsingPosixShm() ? "set" : "empty";
 
   configMap["defaultAllocSizes"] = "";
   // Stringify std::set

cachelib/allocator/MemoryTierCacheConfig.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace facebook {
+namespace cachelib {
+class MemoryTierCacheConfig {
+public:
+  // Creates instance of MemoryTierCacheConfig for file-backed memory.
+  // @param path to file which CacheLib will use to map memory from.
+  // TODO: add fromDirectory, fromAnonymousMemory
+  static MemoryTierCacheConfig fromFile(const std::string& _file) {
+    MemoryTierCacheConfig config;
+    config.path = _file;
+    return config;
+  }
+
+  // Specifies size of this memory tier. Sizes of tiers  must be specified by
+  // either setting size explicitly or using ratio, mixing of the two is not supported.
+  MemoryTierCacheConfig& setSize(size_t _size) {
+    size = _size;
+    return *this;
+  }
+
+  // Specifies ratio of this memory tier to other tiers. Absolute size
+  // of each tier can be calculated as:
+  // cacheSize * tierRatio / Sum of ratios for all tiers; the difference
+  // between total cache size and sum of all tier sizes resulted from
+  // round off error is accounted for when calculating the last tier's
+  // size to make the totals equal.
+  MemoryTierCacheConfig& setRatio(double _ratio) {
+    ratio = _ratio;
+    return *this;
+  }
+
+  size_t getRatio() const noexcept { return ratio; }
+
+  size_t getSize() const noexcept { return size; }
+
+  const std::string& getPath() const noexcept { return path; }
+
+  bool isFileBacked() const {
+    return  !path.empty();
+  }
+
+  // Size of this memory tiers
+  size_t size{0};
+
+  // Ratio is a number of parts of the total cache size to be allocated for this tier.
+  // E.g. if X is a total cache size, Yi are ratios specified for memory tiers,
+  // then size of the i-th tier Xi = (X / (Y1 + Y2)) * Yi and X = sum(Xi)
+  size_t ratio{0};
+
+  // Path to file for file system-backed memory tier
+  // TODO: consider using variant<file, directory, NUMA> to support different
+  // memory sources
+  std::string path;
+
+private:
+  MemoryTierCacheConfig() = default;
+};
+} // namespace cachelib
+} // namespace facebook

cachelib/allocator/TempShmMapping.cpp

@@ -34,7 +34,8 @@ TempShmMapping::TempShmMapping(size_t size)
 TempShmMapping::~TempShmMapping() {
   try {
     if (addr_) {
-      shmManager_->removeShm(detail::kTempShmCacheName.str());
+      shmManager_->removeShm(detail::kTempShmCacheName.str(),
+        PosixSysVSegmentOpts(false /* posix */));
     }
     if (shmManager_) {
       shmManager_.reset();
@@ -77,7 +78,8 @@ void* TempShmMapping::createShmMapping(ShmManager& shmManager,
     return shmAddr;
   } catch (...) {
     if (shmAddr) {
-      shmManager.removeShm(detail::kTempShmCacheName.str());
+      shmManager.removeShm(detail::kTempShmCacheName.str(),
+        PosixSysVSegmentOpts(false /* posix */));
     } else {
       munmap(addr, size);
     }

cachelib/allocator/memory/tests/SlabAllocatorTest.cpp

@@ -584,7 +584,7 @@ TEST_F(SlabAllocatorTest, AdviseRelease) {
   shmName += std::to_string(::getpid());
   shmManager.createShm(shmName, allocSize, memory);
 
-  SCOPE_EXIT { shmManager.removeShm(shmName); };
+  SCOPE_EXIT { shmManager.removeShm(shmName, PosixSysVSegmentOpts(false)); };
 
   memory = util::align(Slab::kSize, size, memory, allocSize);
 
@@ -714,7 +714,7 @@ TEST_F(SlabAllocatorTest, AdviseSaveRestore) {
   ShmManager shmManager(cacheDir, false /* posix */);
   shmManager.createShm(shmName, allocSize, memory);
 
-  SCOPE_EXIT { shmManager.removeShm(shmName); };
+  SCOPE_EXIT { shmManager.removeShm(shmName, PosixSysVSegmentOpts(false)); };
 
   {
     SlabAllocator s(memory, size, config);