Meta rebase

.github/workflows/build-cachelib-centos.yml → .github/workflows/build-cachelib-centos-long.yml

@@ -1,7 +1,8 @@
 name: build-cachelib-centos-latest
 on:
   schedule:
-     - cron:  '30 5 * * 1,4'
+    - cron:  '0 7 * * *'
+
 jobs:
   build-cachelib-centos8-latest:
     name: "CentOS/latest - Build CacheLib with all dependencies"
@@ -33,3 +34,6 @@ jobs:
         uses: actions/checkout@v2
       - name: "build CacheLib using build script"
         run: ./contrib/build.sh -j -v -T
+      - name: "run tests"
+        timeout-minutes: 60
+        run: cd opt/cachelib/tests && ../../../run_tests.sh long

.github/workflows/build-cachelib-debian.yml

@@ -1,7 +1,8 @@
 name: build-cachelib-debian-10
 on:
   schedule:
-     - cron:  '30 5 * * 2,6'
+    - cron:  '30 5 * * 0,3'
+
 jobs:
   build-cachelib-debian-10:
     name: "Debian/Buster - Build CacheLib with all dependencies"
@@ -37,3 +38,6 @@ jobs:
         uses: actions/checkout@v2
       - name: "build CacheLib using build script"
         run: ./contrib/build.sh -j -v -T
+      - name: "run tests"
+        timeout-minutes: 60
+        run: cd opt/cachelib/tests && ../../../run_tests.sh

.github/workflows/build-cachelib-docker.yml

@@ -0,0 +1,49 @@
+name: build-cachelib-docker
+on:
+  push:
+  pull_request:
+
+jobs:
+  build-cachelib-docker:
+    name: "CentOS/latest - Build CacheLib with all dependencies"
+    runs-on: ubuntu-latest
+    env:
+      REPO:           cachelib
+      GITHUB_REPO:    pmem/CacheLib
+      CONTAINER_REG:  ghcr.io/pmem/cachelib
+      CONTAINER_REG_USER:   ${{ secrets.GH_CR_USER }}
+      CONTAINER_REG_PASS:   ${{ secrets.GH_CR_PAT }}
+      FORCE_IMAGE_ACTION:   ${{ secrets.FORCE_IMAGE_ACTION }}
+      HOST_WORKDIR:         ${{ github.workspace }}
+      WORKDIR:              docker
+      IMG_VER:              devel
+    strategy:
+      matrix:
+        CONFIG: ["OS=centos OS_VER=8streams PUSH_IMAGE=1"]
+    steps:
+      - name: "System Information"
+        run: |
+          echo === uname ===
+          uname -a
+          echo === /etc/os-release ===
+          cat /etc/os-release
+          echo === df -hl ===
+          df -hl
+          echo === free -h ===
+          free -h
+          echo === top ===
+          top -b -n1 -1 -Eg || timeout 1 top -b -n1
+          echo === env ===
+          env
+          echo === gcc -v ===
+          gcc -v
+      - name: "checkout sources"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Pull the image or rebuild and push it
+        run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh $FORCE_IMAGE_ACTION
+
+      - name: Run the build
+        run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build.sh

.github/workflows/clang-format-check.yml

@@ -1,6 +1,6 @@
 # From: https://github.com/marketplace/actions/clang-format-check#multiple-paths
 name: clang-format Check
-on: [pull_request]
+on: []
 jobs:
   formatting-check:
     name: Formatting Check
@@ -13,7 +13,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Run clang-format style check for C/C++ programs.
-      uses: jidicula/clang-format-action@v3.4.0
+      uses: jidicula/clang-format-action@v4.6.2
       with:
-        clang-format-version: '11'
+        clang-format-version: '13'
         check-path: ${{ matrix.path }}

.packit.yaml

@@ -0,0 +1,25 @@
+# See the documentation for more information:
+# https://packit.dev/docs/configuration
+
+specfile_path: cachelib.spec
+
+upstream_package_name: CacheLib
+downstream_package_name: cachelib
+
+actions:
+  fix-spec-file:
+  - bash -c "sed -i cachelib.spec -e \"s/%global commit.*/%global commit $(git rev-parse HEAD)/\""
+  - bash -c "sed -i cachelib.spec -e \"s/%global date.*/%global date $(git show -s --date=format:'%Y%m%d' --format=%cd)/\""
+  create-archive:
+  - bash -c "COMMIT=$(git rev-parse HEAD); curl -ORL https://github.com/facebook/CacheLib/archive/${COMMIT}/cachelib-${COMMIT}.tar.gz; echo cachelib-${COMMIT}.tar.gz"
+  post-upstream-clone: "bash -c \"rm -rf cachelib-dist-git; git clone -b packit https://pagure.io/meta/cachelib.git cachelib-dist-git && mv cachelib-dist-git/cachelib*.{spec,patch} .\""
+
+jobs:
+- job: copr_build
+  trigger: pull_request
+  metadata:
+    targets:
+    - fedora-rawhide-aarch64
+    - fedora-rawhide-x86_64
+    - fedora-35-aarch64
+    - fedora-35-x86_64

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## V17
+In this version, `CacheAllocator::ItemHandle` is removed. Updating to this version will cause compilation error if `ItemHandle` is still used.
+
 ## V16
 
 This version is incompatible with versions below 15. Downgrading from this version directly to a version below 15 will require the cache to be dropped. If you need to downgrade from this version, please make sure you downgrade to version 15 first to avoid dropping the cache.
@@ -8,7 +11,7 @@ This version is incompatible with versions below 15. Downgrading from this versi
 
 This version is incompatible with any previous versions.
 
-Updating to this version may cause compliation error because:
+Updating to this version may cause compilation error because:
 - The following APIs are removed:
 1. CacheAllocator::allocatePermanent_deprecated.
 

cachelib/CMakeLists.txt

@@ -17,7 +17,7 @@
 # refer to the root source directory of the project as ${HELLO_SOURCE_DIR} and
 # to the root binary directory of the project as ${HELLO_BINARY_DIR}.
 
-cmake_minimum_required (VERSION 3.19)
+cmake_minimum_required (VERSION 3.12)
 
 ## TODO: get version from variable
 project (CacheLib VERSION 0.1)

cachelib/allocator/CCacheAllocator.cpp

@@ -30,12 +30,12 @@ CCacheAllocator::CCacheAllocator(MemoryAllocator& allocator, PoolId poolId)
 CCacheAllocator::CCacheAllocator(MemoryAllocator& allocator,
                                  PoolId poolId,
                                  const SerializationType& object)
-    : CCacheAllocatorBase(*object.ccMetadata_ref()),
+    : CCacheAllocatorBase(*object.ccMetadata()),
       allocator_(allocator),
       poolId_(poolId),
       currentChunksIndex_(0) {
   auto& currentChunks = chunks_[currentChunksIndex_];
-  for (auto chunk : *object.chunks_ref()) {
+  for (auto chunk : *object.chunks()) {
     currentChunks.push_back(allocator_.unCompress(CompressedPtr(chunk)));
   }
 }
@@ -93,11 +93,11 @@ size_t CCacheAllocator::resize() {
 
 CCacheAllocator::SerializationType CCacheAllocator::saveState() {
   CCacheAllocator::SerializationType object;
-  *object.ccMetadata_ref() = ccType_.saveState();
+  *object.ccMetadata() = ccType_.saveState();
 
   std::lock_guard<std::mutex> guard(resizeLock_);
   for (auto chunk : getCurrentChunks()) {
-    object.chunks_ref()->push_back(allocator_.compress(chunk).saveState());
+    object.chunks()->push_back(allocator_.compress(chunk).saveState());
   }
   return object;
 }

cachelib/allocator/CCacheManager.cpp

@@ -24,7 +24,7 @@ CCacheManager::CCacheManager(const SerializationType& object,
     : memoryAllocator_(memoryAllocator) {
   std::lock_guard<std::mutex> guard(lock_);
 
-  for (const auto& allocator : *object.allocators_ref()) {
+  for (const auto& allocator : *object.allocators()) {
     auto id = memoryAllocator_.getPoolId(allocator.first);
     allocators_.emplace(
         std::piecewise_construct,
@@ -81,8 +81,7 @@ CCacheManager::SerializationType CCacheManager::saveState() {
 
   SerializationType object;
   for (auto& allocator : allocators_) {
-    object.allocators_ref()->emplace(allocator.first,
-                                     allocator.second.saveState());
+    object.allocators()->emplace(allocator.first, allocator.second.saveState());
   }
   return object;
 }

cachelib/allocator/CMakeLists.txt

@@ -77,14 +77,16 @@ install(TARGETS cachelib_allocator
         DESTINATION ${LIB_INSTALL_DIR} )
 
 if (BUILD_TESTS)
-  add_library (allocator_test_support
+  add_library (allocator_test_support OBJECT
     ${DATASTRUCT_TESTS_THRIFT_FILES}
     ./nvmcache/tests/NvmTestBase.cpp
     ./memory/tests/TestBase.cpp
+    ../common/TestUtils.cpp
     )
   add_dependencies(allocator_test_support thrift_generated_files)
   target_link_libraries (allocator_test_support PUBLIC
     cachelib_allocator
+    common_test_utils
     glog::glog
     gflags
     GTest::gtest
@@ -116,8 +118,11 @@ if (BUILD_TESTS)
   add_test (tests/ChainedHashTest.cpp)
   add_test (tests/AllocatorResizeTypeTest.cpp)
   add_test (tests/AllocatorHitStatsTypeTest.cpp)
+  add_test (tests/AllocatorMemoryTiersTest.cpp)
+  add_test (tests/MemoryTiersTest.cpp)
   add_test (tests/MultiAllocatorTest.cpp)
   add_test (tests/NvmAdmissionPolicyTest.cpp)
+  add_test (tests/CacheAllocatorConfigTest.cpp)
   add_test (nvmcache/tests/NvmItemTests.cpp)
   add_test (nvmcache/tests/InFlightPutsTest.cpp)
   add_test (nvmcache/tests/TombStoneTests.cpp)

cachelib/allocator/Cache.cpp

@@ -23,6 +23,12 @@
 namespace facebook {
 namespace cachelib {
 
+CacheBase::CacheBase(unsigned numTiers): numTiers_(numTiers) {}
+
+unsigned CacheBase::getNumTiers() const {
+  return numTiers_;
+}
+
 void CacheBase::setRebalanceStrategy(
     PoolId pid, std::shared_ptr<RebalanceStrategy> strategy) {
   std::unique_lock<std::mutex> l(lock_);