Add option to insert items to first free tier (#87)

instead of always inserting to topmost tier

Author: igchor

cachelib/allocator/CacheAllocator-inl.h

@@ -424,7 +424,8 @@ CacheAllocator<CacheTrait>::allocateInternalTier(TierId tid,
                                                  uint32_t size,
                                                  uint32_t creationTime,
                                                  uint32_t expiryTime,
-                                                 bool fromBgThread) {
+                                                 bool fromBgThread,
+                                                 bool evict) {
   util::LatencyTracker tracker{stats().allocateLatency_};
 
   SCOPE_FAIL { stats_.invalidAllocs.inc(); };
@@ -445,7 +446,9 @@ CacheAllocator<CacheTrait>::allocateInternalTier(TierId tid,
     backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp();
   }
 
-  if (memory == nullptr) {
+  if (memory == nullptr && !evict) {
+    return {};
+  } else if (memory == nullptr) {
     memory = findEviction(tid, pid, cid);
   }
 
@@ -495,7 +498,8 @@ CacheAllocator<CacheTrait>::allocateInternal(PoolId pid,
                                              bool fromBgThread) {
   auto tid = 0; /* TODO: consult admission policy */
   for(TierId tid = 0; tid < getNumTiers(); ++tid) {
-    auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromBgThread);
+    bool evict = !config_.insertToFirstFreeTier || tid == getNumTiers() - 1;
+    auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromBgThread, evict);
     if (handle) return handle;
   }
   return {};
@@ -1829,13 +1833,17 @@ CacheAllocator<CacheTrait>::tryEvictToNextMemoryTier(
 
   TierId nextTier = tid; // TODO - calculate this based on some admission policy
   while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers
+    // always evict item from the nextTier to make room for new item
+    bool evict = true;
+
     // allocateInternal might trigger another eviction
     auto newItemHdl = allocateInternalTier(nextTier, pid,
                      item.getKey(),
                      item.getSize(),
                      item.getCreationTime(),
                      item.getExpiryTime(),
-                     fromBgThread);
+                     fromBgThread,
+                     evict);
 
     if (newItemHdl) {
       XDCHECK_EQ(newItemHdl->getSize(), item.getSize());
@@ -1871,13 +1879,17 @@ CacheAllocator<CacheTrait>::tryPromoteToNextMemoryTier(
     auto toPromoteTier = nextTier - 1;
     --nextTier;
 
+    // always evict item from the toPromoteTier to make room for new item
+    bool evict = true;
+
     // allocateInternal might trigger another eviction
     auto newItemHdl = allocateInternalTier(toPromoteTier, pid,
                      item.getKey(),
                      item.getSize(),
                      item.getCreationTime(),
                      item.getExpiryTime(),
-                     fromBgThread);
+                     fromBgThread,
+                     true);
 
     if (newItemHdl) {
       XDCHECK_EQ(newItemHdl->getSize(), item.getSize());
@@ -3251,6 +3263,8 @@ CacheAllocator<CacheTrait>::allocateNewItemForOldItem(const Item& oldItem) {
 
   const auto allocInfo =
       allocator_[getTierId(oldItem)]->getAllocInfo(static_cast<const void*>(&oldItem));
+    
+  bool evict = !config_.insertToFirstFreeTier || getTierId(oldItem) == getNumTiers() - 1;
 
   // Set up the destination for the move. Since oldItem would have the moving
   // bit set, it won't be picked for eviction.
@@ -3260,7 +3274,8 @@ CacheAllocator<CacheTrait>::allocateNewItemForOldItem(const Item& oldItem) {
                                          oldItem.getSize(),
                                          oldItem.getCreationTime(),
                                          oldItem.getExpiryTime(),
-                                         false);
+                                         false,
+                                         evict);
   if (!newItemHdl) {
     return {};
   }

cachelib/allocator/CacheAllocator.h

@@ -1521,13 +1521,19 @@ class CacheAllocator : public CacheBase {
   // For description see allocateInternal.
   //
   // @param tid id a memory tier
+  // @param fromBgThread whether this function was called from a bg
+  //        thread - this is used to decide whether bg thread should
+  //        be waken in case there is no free memory
+  // @param evict whether to evict an item from tier tid in case there
+  //        is not enough memory
   WriteHandle allocateInternalTier(TierId tid,
                                    PoolId id,
                                    Key key,
                                    uint32_t size,
                                    uint32_t creationTime,
                                    uint32_t expiryTime,
-                                   bool fromBgThread);
+                                   bool fromBgThread,
+                                   bool evict);
 
   // Allocate a chained item
   //

cachelib/allocator/CacheAllocatorConfig.h

@@ -311,6 +311,9 @@ class CacheAllocatorConfig {
   // Library team if you find yourself customizing this.
   CacheAllocatorConfig& setThrottlerConfig(util::Throttler::Config config);
 
+  // Insert items to first free memory tier
+  CacheAllocatorConfig& enableInsertToFirstFreeTier();
+
   // Passes in a callback to initialize an event tracker when the allocator
   // starts
   CacheAllocatorConfig& setEventTracker(EventTrackerSharedPtr&&);
@@ -527,6 +530,11 @@ class CacheAllocatorConfig {
   // ABOVE are the config for various cache workers
   //
 
+  // if turned off, always insert new elements to topmost memory tier.
+  // if turned on, insert new element to first free memory tier or evict memory
+  // from the bottom one if memory cache is full
+  bool insertToFirstFreeTier = false;
+
   // the number of tries to search for an item to evict
   // 0 means it's infinite
   unsigned int evictionSearchTries{50};
@@ -662,6 +670,12 @@ class CacheAllocatorConfig {
       {MemoryTierCacheConfig::fromShm().setRatio(1)}};
 };
 
+template <typename T>
+CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::enableInsertToFirstFreeTier() {
+  insertToFirstFreeTier = true;
+  return *this;
+}
+
 template <typename T>
 CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::setCacheName(
     const std::string& _cacheName) {
@@ -1241,6 +1255,7 @@ std::map<std::string, std::string> CacheAllocatorConfig<T>::serialize() const {
   configMap["nvmAdmissionMinTTL"] = std::to_string(nvmAdmissionMinTTL);
   configMap["delayCacheWorkersStart"] =
       delayCacheWorkersStart ? "true" : "false";
+  configMap["insertToFirstFreeTier"] = std::to_string(insertToFirstFreeTier);
   mergeWithPrefix(configMap, throttleConfig.serialize(), "throttleConfig");
   mergeWithPrefix(configMap,
                   chainedItemAccessConfig.serialize(),

cachelib/cachebench/cache/Cache-inl.h

@@ -104,6 +104,8 @@ Cache<Allocator>::Cache(const CacheConfig& config,
     allocatorConfig_.configureMemoryTiers(config_.memoryTierConfigs);
   }
 
+  allocatorConfig_.insertToFirstFreeTier = config_.insertToFirstFreeTier;
+
   auto cleanupGuard = folly::makeGuard([&] {
     if (!nvmCacheFilePath_.empty()) {
       util::removePath(nvmCacheFilePath_);

cachelib/cachebench/util/CacheConfig.cpp

@@ -49,6 +49,8 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, tryLockUpdate);
   JSONSetVal(configJson, lruIpSpec);
   JSONSetVal(configJson, useCombinedLockForIterators);
+  
+  JSONSetVal(configJson, insertToFirstFreeTier);
 
   JSONSetVal(configJson, lru2qHotPct);
   JSONSetVal(configJson, lru2qColdPct);

cachelib/cachebench/util/CacheConfig.h

@@ -97,6 +97,8 @@ struct CacheConfig : public JSONConfig {
   bool lruUpdateOnRead{true};
   bool tryLockUpdate{false};
   bool useCombinedLockForIterators{true};
+  
+  bool insertToFirstFreeTier{false};
 
   // LRU param
   uint64_t lruIpSpec{0};