Skip to content

Commit 5130a65

Browse files
igchorbyrnedj
igchor
authored and
byrnedj
committed
Run centos and debian workflows on push and PR
Run tests on CI Run long tests (navy/bench) every day on CI Run CI on prebuild docker image Run only centos build on CI Update docker file used in CI Centos8 is EOL Disable failing clang-format-check Add extra param to build-package.sh Add scripts for rebuilding/pushing docker images Taken from: pmem/dev-utils-kit@30794c3 Extend CI to rebuild docker automatically Update build-cachelib-docker.yml Do not use shallow clone to make sure Docker rebuild logic works correctly. Added required packages to install Intel ittapi Update CI to use intel/CacheLib repo (#17) Add multi-tier navy benchmark and run it on CI - fix navy multi-tier config for NUMA bindings added code coverage support in CacheLib Adding libdml to CentOS docker image (#53) --------------------------------------------- one large commit for all CI and code coverage see above for the change history.
1 parent 44cc9f2 commit 5130a65

19 files changed

+730
-7
lines changed

‎.github/workflows/build-cachelib-centos-long.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: build-cachelib-centos-latest
2+
on:
3+
schedule:
4+
- cron: '0 7 * * *'
5+
6+
jobs:
7+
build-cachelib-centos8-latest:
8+
name: "CentOS/latest - Build CacheLib with all dependencies"
9+
runs-on: ubuntu-latest
10+
# Docker container image name
11+
container: "centos:latest"
12+
steps:
13+
- name: "update packages"
14+
run: dnf upgrade -y
15+
- name: "install sudo,git"
16+
run: dnf install -y sudo git cmake gcc
17+
- name: "System Information"
18+
run: |
19+
echo === uname ===
20+
uname -a
21+
echo === /etc/os-release ===
22+
cat /etc/os-release
23+
echo === df -hl ===
24+
df -hl
25+
echo === free -h ===
26+
free -h
27+
echo === top ===
28+
top -b -n1 -1 -Eg || timeout 1 top -b -n1
29+
echo === env ===
30+
env
31+
echo === gcc -v ===
32+
gcc -v
33+
- name: "checkout sources"
34+
uses: actions/checkout@v2
35+
- name: "build CacheLib using build script"
36+
run: ./contrib/build.sh -j -v -T
37+
- name: "run tests"
38+
timeout-minutes: 60
39+
run: cd opt/cachelib/tests && ../../../run_tests.sh long

‎.github/workflows/build-cachelib-debian.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: build-cachelib-debian-10
2+
on:
3+
schedule:
4+
- cron: '30 5 * * 0,3'
5+
6+
jobs:
7+
build-cachelib-debian-10:
8+
name: "Debian/Buster - Build CacheLib with all dependencies"
9+
runs-on: ubuntu-latest
10+
# Docker container image name
11+
container: "debian:buster-slim"
12+
steps:
13+
- name: "update packages"
14+
run: apt-get update
15+
- name: "upgrade packages"
16+
run: apt-get -y upgrade
17+
- name: "install sudo,git"
18+
run: apt-get install -y sudo git procps
19+
- name: "System Information"
20+
run: |
21+
echo === uname ===
22+
uname -a
23+
echo === /etc/os-release ===
24+
cat /etc/os-release
25+
echo === df -hl ===
26+
df -hl
27+
echo === free -h ===
28+
free -h
29+
echo === top ===
30+
top -b -n1 -1 -Eg || timeout 1 top -b -n1 ; true
31+
echo === env ===
32+
env
33+
echo === cc -v ===
34+
cc -v || true
35+
echo === g++ -v ===
36+
g++ - || true
37+
- name: "checkout sources"
38+
uses: actions/checkout@v2
39+
- name: "build CacheLib using build script"
40+
run: ./contrib/build.sh -j -v -T
41+
- name: "run tests"
42+
timeout-minutes: 60
43+
run: cd opt/cachelib/tests && ../../../run_tests.sh

‎.github/workflows/build-cachelib-docker.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: build-cachelib-docker
2+
on:
3+
push:
4+
pull_request:
5+
6+
jobs:
7+
build-cachelib-docker:
8+
name: "CentOS/latest - Build CacheLib with all dependencies"
9+
runs-on: ubuntu-latest
10+
env:
11+
REPO: cachelib
12+
GITHUB_REPO: intel/CacheLib
13+
CONTAINER_REG: ghcr.io/pmem/cachelib
14+
CONTAINER_REG_USER: ${{ secrets.GH_CR_USER }}
15+
CONTAINER_REG_PASS: ${{ secrets.GH_CR_PAT }}
16+
FORCE_IMAGE_ACTION: ${{ secrets.FORCE_IMAGE_ACTION }}
17+
HOST_WORKDIR: ${{ github.workspace }}
18+
WORKDIR: docker
19+
IMG_VER: devel
20+
strategy:
21+
matrix:
22+
CONFIG: ["OS=centos OS_VER=8streams PUSH_IMAGE=1"]
23+
steps:
24+
- name: "System Information"
25+
run: |
26+
echo === uname ===
27+
uname -a
28+
echo === /etc/os-release ===
29+
cat /etc/os-release
30+
echo === df -hl ===
31+
df -hl
32+
echo === free -h ===
33+
free -h
34+
echo === top ===
35+
top -b -n1 -1 -Eg || timeout 1 top -b -n1
36+
echo === env ===
37+
env
38+
echo === gcc -v ===
39+
gcc -v
40+
- name: "checkout sources"
41+
uses: actions/checkout@v2
42+
with:
43+
fetch-depth: 0
44+
45+
- name: Pull the image or rebuild and push it
46+
run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh $FORCE_IMAGE_ACTION
47+
48+
- name: Run the build
49+
run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build.sh

‎.github/workflows/clang-format-check.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# From: https://github.com/marketplace/actions/clang-format-check#multiple-paths
22
name: clang-format Check
3-
on: [pull_request]
3+
on: []
44
jobs:
55
formatting-check:
66
name: Formatting Check

‎cachelib/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ set(CMAKE_MODULE_PATH
8585
set(CMAKE_CXX_STANDARD 17)
8686
set(CMAKE_CXX_STANDARD_REQUIRED True)
8787

88+
if(COVERAGE_ENABLED)
89+
# Add code coverage
90+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage")
91+
endif()
92+
8893
# include(fb_cxx_flags)
8994
message(STATUS "Update CXXFLAGS: ${CMAKE_CXX_FLAGS}")
9095

‎cachelib/cachebench/test_configs/consistency/navy-multi-tier.json

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"cache_config" : {
3+
"cacheSizeMB" : 300,
4+
"poolRebalanceIntervalSec" : 1,
5+
"moveOnSlabRelease" : true,
6+
7+
"cacheDir": "/tmp/mem-tier2",
8+
"memoryTiers" : [
9+
{
10+
"ratio": 1,
11+
"memBindNodes": 0
12+
},
13+
{
14+
"ratio": 1,
15+
"memBindNodes": 0
16+
}
17+
],
18+
19+
"numPools" : 2,
20+
"poolSizes" : [0.5, 0.5],
21+
"allocFactor" : 2.0,
22+
"nvmCacheSizeMB" : 1024
23+
},
24+
"test_config" :
25+
{
26+
27+
"checkConsistency" : true,
28+
29+
"numOps" : 60000,
30+
"numThreads" : 20,
31+
"numKeys" : 200000,
32+
33+
34+
"keySizeRange" : [1, 8, 64],
35+
"keySizeRangeProbability" : [0.5, 0.5],
36+
37+
"valSizeRange" : [256, 1024, 4096, 8192],
38+
"valSizeRangeProbability" : [0.2, 0.7, 0.1],
39+
40+
"chainedItemLengthRange" : [1, 2, 4, 32],
41+
"chainedItemLengthRangeProbability" : [0.8, 0.18, 0.02],
42+
43+
"chainedItemValSizeRange" : [1, 128, 256, 1024, 4096, 20480],
44+
"chainedItemValSizeRangeProbability" : [0.1, 0.1, 0.2, 0.3, 0.3],
45+
46+
"getRatio" : 0.8,
47+
"setRatio" : 0.1,
48+
"delRatio" : 0.0,
49+
"addChainedRatio" : 0.05,
50+
"keyPoolDistribution": [0.5, 0.5],
51+
"opPoolDistribution" : [0.5, 0.5]
52+
}
53+
54+
}

‎cachelib/cachebench/test_configs/consistency/navy.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414

1515
"checkConsistency" : true,
1616

17-
"numOps" : 30000000,
18-
"numThreads" : 40,
17+
"numOps" : 600000,
18+
"numThreads" : 20,
1919
"numKeys" : 200000,
2020

2121

‎contrib/build-package.sh

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,8 @@ build_tests=
7878
show_help=
7979
many_jobs=
8080
verbose=
81-
PREFIX="$PWD/opt/cachelib/"
82-
83-
while getopts :BSdhijtvp: param
81+
install_path=
82+
while getopts :BSdhijtvI: param
8483
do
8584
case $param in
8685
i) install=yes ;;
@@ -91,7 +90,7 @@ do
9190
v) verbose=yes ;;
9291
j) many_jobs=yes ;;
9392
t) build_tests=yes ;;
94-
p) PREFIX=$OPTARG ;;
93+
I) install_path=${OPTARG} ; install=yes ;;
9594
?) die "unknown option. See -h for help."
9695
esac
9796
done
@@ -160,6 +159,7 @@ case "$1" in
160159
REPODIR=cachelib/external/$NAME
161160
SRCDIR=$REPODIR
162161
external_git_clone=yes
162+
external_git_tag=8.0.1
163163
cmake_custom_params="-DBUILD_SHARED_LIBS=ON"
164164
if test "$build_tests" = "yes" ; then
165165
cmake_custom_params="$cmake_custom_params -DFMT_TEST=YES"
@@ -276,6 +276,7 @@ test -d cachelib || die "expected 'cachelib' directory not found in $PWD"
276276

277277

278278
# After ensuring we are in the correct directory, set the installation prefix"
279+
PREFIX=${install_path:-"$PWD/opt/cachelib/"}
279280
CMAKE_PARAMS="$CMAKE_PARAMS -DCMAKE_INSTALL_PREFIX=$PREFIX"
280281
CMAKE_PREFIX_PATH="$PREFIX/lib/cmake:$PREFIX/lib64/cmake:$PREFIX/lib:$PREFIX/lib64:$PREFIX:${CMAKE_PREFIX_PATH:-}"
281282
export CMAKE_PREFIX_PATH

‎docker/build.sh

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
# Copyright 2022, Intel Corporation
4+
5+
#
6+
# build.sh - runs a Docker container from a Docker image with environment
7+
# prepared for running CacheLib builds and tests. It uses Docker image
8+
# tagged as described in ./images/build-image.sh.
9+
#
10+
# Notes:
11+
# - set env var 'HOST_WORKDIR' to where the root of this project is on the host machine,
12+
# - set env var 'OS' and 'OS_VER' properly to a system/Docker you want to build this
13+
# repo on (for proper values take a look at the list of Dockerfiles at the
14+
# utils/docker/images directory in this repo), e.g. OS=ubuntu, OS_VER=20.04,
15+
# - set env var 'CONTAINER_REG' to container registry address
16+
# [and possibly user/org name, and package name], e.g. "<CR_addr>/pmem/CacheLib",
17+
# - set env var 'DNS_SERVER' if you use one,
18+
# - set env var 'COMMAND' to execute specific command within Docker container or
19+
# env var 'TYPE' to pick command based on one of the predefined types of build (see below).
20+
#
21+
22+
set -e
23+
24+
source $(dirname ${0})/set-ci-vars.sh
25+
IMG_VER=${IMG_VER:-devel}
26+
TAG="${OS}-${OS_VER}-${IMG_VER}"
27+
IMAGE_NAME=${CONTAINER_REG}:${TAG}
28+
CONTAINER_NAME=CacheLib-${OS}-${OS_VER}
29+
WORKDIR=/CacheLib # working dir within Docker container
30+
SCRIPTSDIR=${WORKDIR}/docker
31+
32+
if [[ -z "${OS}" || -z "${OS_VER}" ]]; then
33+
echo "ERROR: The variables OS and OS_VER have to be set " \
34+
"(e.g. OS=fedora, OS_VER=32)."
35+
exit 1
36+
fi
37+
38+
if [[ -z "${HOST_WORKDIR}" ]]; then
39+
echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \
40+
"the root of this project on the host machine."
41+
exit 1
42+
fi
43+
44+
if [[ -z "${CONTAINER_REG}" ]]; then
45+
echo "ERROR: CONTAINER_REG environment variable is not set " \
46+
"(e.g. \"<registry_addr>/<org_name>/<package_name>\")."
47+
exit 1
48+
fi
49+
50+
# Set command to execute in the Docker container
51+
COMMAND="./run-build.sh";
52+
echo "COMMAND to execute within Docker container: ${COMMAND}"
53+
54+
if [ -n "${DNS_SERVER}" ]; then DOCKER_OPTS="${DOCKER_OPTS} --dns=${DNS_SERVER}"; fi
55+
56+
# Check if we are running on a CI (Travis or GitHub Actions)
57+
[ -n "${GITHUB_ACTIONS}" -o -n "${TRAVIS}" ] && CI_RUN="YES" || CI_RUN="NO"
58+
59+
# Do not allocate a pseudo-TTY if we are running on GitHub Actions
60+
[ ! "${GITHUB_ACTIONS}" ] && DOCKER_OPTS="${DOCKER_OPTS} --tty=true"
61+
62+
63+
echo "Running build using Docker image: ${IMAGE_NAME}"
64+
65+
# Run a container with
66+
# - environment variables set (--env)
67+
# - host directory containing source mounted (-v)
68+
# - working directory set (-w)
69+
docker run --privileged=true --name=${CONTAINER_NAME} -i \
70+
${DOCKER_OPTS} \
71+
--env http_proxy=${http_proxy} \
72+
--env https_proxy=${https_proxy} \
73+
--env TERM=xterm-256color \
74+
--env WORKDIR=${WORKDIR} \
75+
--env SCRIPTSDIR=${SCRIPTSDIR} \
76+
--env GITHUB_REPO=${GITHUB_REPO} \
77+
--env CI_RUN=${CI_RUN} \
78+
--env TRAVIS=${TRAVIS} \
79+
--env GITHUB_ACTIONS=${GITHUB_ACTIONS} \
80+
--env CI_COMMIT=${CI_COMMIT} \
81+
--env CI_COMMIT_RANGE=${CI_COMMIT_RANGE} \
82+
--env CI_BRANCH=${CI_BRANCH} \
83+
--env CI_EVENT_TYPE=${CI_EVENT_TYPE} \
84+
--env CI_REPO_SLUG=${CI_REPO_SLUG} \
85+
--env DOC_UPDATE_GITHUB_TOKEN=${DOC_UPDATE_GITHUB_TOKEN} \
86+
--env DOC_UPDATE_BOT_NAME=${DOC_UPDATE_BOT_NAME} \
87+
--env DOC_REPO_OWNER=${DOC_REPO_OWNER} \
88+
--env COVERITY_SCAN_TOKEN=${COVERITY_SCAN_TOKEN} \
89+
--env COVERITY_SCAN_NOTIFICATION_EMAIL=${COVERITY_SCAN_NOTIFICATION_EMAIL} \
90+
--env TEST_TIMEOUT=${TEST_TIMEOUT} \
91+
--env TZ='Europe/Warsaw' \
92+
--shm-size=4G \
93+
-v ${HOST_WORKDIR}:${WORKDIR} \
94+
-v /etc/localtime:/etc/localtime \
95+
-w ${SCRIPTSDIR} \
96+
${IMAGE_NAME} ${COMMAND}
97+

‎docker/images/build-image.sh

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
# Copyright 2016-2021, Intel Corporation
4+
#
5+
# build-image.sh - prepares a Docker image with <OS>-based environment for
6+
# testing (or dev) purpose, tagged with ${CONTAINER_REG}:${OS}-${OS_VER}-${IMG_VER},
7+
# according to the ${OS}-${OS_VER}.Dockerfile file located in the same directory.
8+
# IMG_VER is a version of Docker image (it usually relates to project's release tag)
9+
# and it defaults to "devel".
10+
#
11+
12+
set -e
13+
IMG_VER=${IMG_VER:-devel}
14+
TAG="${OS}-${OS_VER}-${IMG_VER}"
15+
16+
if [[ -z "${OS}" || -z "${OS_VER}" ]]; then
17+
echo "ERROR: The variables OS and OS_VER have to be set " \
18+
"(e.g. OS=fedora, OS_VER=34)."
19+
exit 1
20+
fi
21+
22+
if [[ -z "${CONTAINER_REG}" ]]; then
23+
echo "ERROR: CONTAINER_REG environment variable is not set " \
24+
"(e.g. \"<registry_addr>/<org_name>/<package_name>\")."
25+
exit 1
26+
fi
27+
28+
echo "Check if the file ${OS}-${OS_VER}.Dockerfile exists"
29+
if [[ ! -f "${OS}-${OS_VER}.Dockerfile" ]]; then
30+
echo "Error: ${OS}-${OS_VER}.Dockerfile does not exist."
31+
exit 1
32+
fi
33+
34+
echo "Build a Docker image tagged with: ${CONTAINER_REG}:${TAG}"
35+
docker build -t ${CONTAINER_REG}:${TAG} \
36+
--build-arg http_proxy=$http_proxy \
37+
--build-arg https_proxy=$https_proxy \
38+
-f ${OS}-${OS_VER}.Dockerfile .

0 commit comments

Comments
 (0)