Skip to content

Set up github actions CI #617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rtjohnso merged 7 commits into from
Feb 28, 2024
Merged

Set up github actions CI #617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3db457b
Create run-tests.yml
rtjohnso Feb 20, 2024
5ea003b
Update run-tests.yml
rtjohnso Feb 20, 2024
ceeb0c7
add testing workflow
rtjohnso Feb 21, 2024
2a33d81
test commit
rtjohnso Feb 21, 2024
30e5cee
better job names
rtjohnso Feb 21, 2024
cfb9fd0
Merge pull request #1 from rtjohnso/rtjohnso/workflow
rtjohnso Feb 21, 2024
6a879d0
Fix bugs in CI and discovered by CI
rtjohnso Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .github/workflows/run-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@

name: Splinter Tests

on:
pull_request:

jobs:

format-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-13 clang-format-13
- name: format-check
run: ./format-check.sh

run-test-matrix:
strategy:
matrix:
compiler_mode: [gcc, clang, gcc-asan, clang-msan]
build_mode: [release, debug]
target: [all] # Just compile most configs
exclude:
# Don't do a compile job on these, since we will do run-tests on them below
- compiler_mode: gcc-asan
build_mode: release
target: all
- compiler_mode: clang-msan
build_mode: release
target: all
- compiler_mode: clang
build_mode: debug
target: all
include:
# Compile and run tests on these configs
- compiler_mode: gcc-asan
build_mode: release
target: run-tests
- compiler_mode: clang-msan
build_mode: release
sanitizer: msan
target: run-tests
- compiler_mode: clang
build_mode: debug
target: run-tests
name: ${{ matrix.target == 'all' && 'Compile' || 'Run' }} ${{ matrix.compiler_mode }} ${{ matrix.build_mode }} ${{ matrix.target }}
runs-on: ubuntu-latest
env:
CC: ${{ startsWith(matrix.compiler_mode, 'gcc') && 'gcc' || 'clang' }}
LD: ${{ startsWith(matrix.compiler_mode, 'gcc') && 'gcc' || 'clang' }}
BUILD_MODE: ${{ matrix.build_mode }}
BUILD_ASAN: ${{ endsWith(matrix.compiler_mode, 'asan') && 1 || 0 }}
BUILD_MSAN: ${{ endsWith(matrix.compiler_mode, 'msan') && 1 || 0 }}
INCLUDE_SLOW_TESTS: true
steps:
- name: Maximize build space
uses: easimon/maximize-build-space@master
with:
root-reserve-mb: 512
swap-size-mb: 1
remove-dotnet: true
remove-android: true
remove-haskell: true
remove-codeql: true
remove-docker-images: false
- uses: actions/checkout@v4
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-13 clang-format-13
- name: make ${{ matrix.target }}
run: make ${{ matrix.target }}
19 changes: 7 additions & 12 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ INCLUDE = -I $(INCDIR) -I $(SRCDIR) -I $(SRCDIR)/platform_$(PLATFORM) -I $(TESTS

# use += here, so that extra flags can be provided via the environment

CFLAGS += -D_GNU_SOURCE -ggdb3 -Wall -pthread -Wfatal-errors -Werror -Wvla
CFLAGS += -D_GNU_SOURCE -ggdb -Wall -pthread -Wfatal-errors -Werror -Wvla
CFLAGS += -DXXH_STATIC_LINKING_ONLY -fPIC
CFLAGS += -DSPLINTERDB_PLATFORM_DIR=$(PLATFORM_DIR)

Expand All @@ -87,7 +87,7 @@ ifeq ($(cpu_arch),x86_64)
CFLAGS += -march=native
endif

LDFLAGS += -ggdb3 -pthread
LDFLAGS += -ggdb -pthread

LIBS = -lm -lpthread -laio -lxxhash
DEPFLAGS = -MMD -MP
Expand Down Expand Up @@ -444,8 +444,7 @@ $(BINDIR)/$(UNITDIR)/splinterdb_stress_test: $(COMMON_TESTOBJ)
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(LIBDIR)/libsplinterdb.so

$(BINDIR)/$(UNITDIR)/writable_buffer_test: $(UTIL_SYS) \
$(COMMON_TESTOBJ) \
$(BINDIR)/$(UNITDIR)/writable_buffer_test: $(COMMON_TESTOBJ) \
$(COMMON_UNIT_TESTOBJ) \
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(LIBDIR)/libsplinterdb.so
Expand All @@ -455,14 +454,12 @@ $(BINDIR)/$(UNITDIR)/limitations_test: $(COMMON_TESTOBJ)
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(LIBDIR)/libsplinterdb.so

$(BINDIR)/$(UNITDIR)/config_parse_test: $(UTIL_SYS) \
$(COMMON_TESTOBJ) \
$(BINDIR)/$(UNITDIR)/config_parse_test: $(COMMON_TESTOBJ) \
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(COMMON_UNIT_TESTOBJ) \
$(LIBDIR)/libsplinterdb.so

$(BINDIR)/$(UNITDIR)/task_system_test: $(UTIL_SYS) \
$(COMMON_TESTOBJ) \
$(BINDIR)/$(UNITDIR)/task_system_test: $(COMMON_TESTOBJ) \
$(COMMON_UNIT_TESTOBJ) \
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(LIBDIR)/libsplinterdb.so
Expand All @@ -472,8 +469,7 @@ $(BINDIR)/$(UNITDIR)/platform_apis_test: $(UTIL_SYS) \
$(OBJDIR)/$(TESTS_DIR)/config.o \
$(PLATFORM_SYS)

$(BINDIR)/$(UNITDIR)/splinter_shmem_test: $(UTIL_SYS) \
$(COMMON_UNIT_TESTOBJ) \
$(BINDIR)/$(UNITDIR)/splinter_shmem_test: $(COMMON_UNIT_TESTOBJ) \
$(LIBDIR)/libsplinterdb.so

$(BINDIR)/$(UNITDIR)/splinter_ipc_test: $(UTIL_SYS) \
Expand All @@ -485,8 +481,7 @@ $(BINDIR)/$(UNITDIR)/splinterdb_forked_child_test: $(OBJDIR)/$(TESTS_DIR)/config
$(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \
$(LIBDIR)/libsplinterdb.so

$(BINDIR)/$(UNITDIR)/large_inserts_stress_test: $(UTIL_SYS) \
$(OBJDIR)/$(TESTS_DIR)/config.o \
$(BINDIR)/$(UNITDIR)/large_inserts_stress_test: $(OBJDIR)/$(TESTS_DIR)/config.o \
$(COMMON_UNIT_TESTOBJ) \
$(LIBDIR)/libsplinterdb.so

Expand Down
1 change: 1 addition & 0 deletions src/routing_filter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,7 @@ routing_filter_add(cache *cc,
uint32 *dst_fp = fp_buffer;
uint32 index_bucket_start = old_index_no * index_size;
if (old_index_count != 0) {
platform_assert(old_index_count <= ROUTING_FPS_PER_PAGE);
PackedArray_unpack((uint32 *)old_block_start,
0,
old_src_fp,
Expand Down
8 changes: 7 additions & 1 deletion src/splinterdb.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,14 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN
// => Caller did not setup a platform-heap on entry.
debug_assert(kvs_cfg->heap_id == NULL);

platform_free(use_this_heap_id, kvs);
if (kvs) {
platform_free(use_this_heap_id, kvs);
}
platform_heap_destroy(&use_this_heap_id);
} else {
if (kvs) {
platform_free(use_this_heap_id, kvs);
}
}

return platform_status_to_int(status);
Expand Down
28 changes: 27 additions & 1 deletion test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,9 +606,12 @@ function run_fast_unit_tests() {
# shellcheck disable=SC2086
"$BINDIR"/unit/task_system_test $Use_shmem

rm splinterdb_unit_tests_db

echo " "
# shellcheck disable=SC2086
"$BINDIR"/driver_test io_apis_test $Use_shmem
rm db
}

# ##################################################################
Expand All @@ -635,6 +638,7 @@ function run_slower_unit_tests() {
# in the program to cough-up an error.
# shellcheck disable=SC2086
run_with_timing "${msg}" "$BINDIR"/unit/splinter_test ${Use_shmem} test_inserts
rm db

# Use fewer rows for this case, to keep elapsed times of MSAN runs reasonable.
msg="Splinter lookups test ${use_msg}"
Expand All @@ -643,13 +647,15 @@ function run_slower_unit_tests() {
# shellcheck disable=SC2086
run_with_timing "${msg}" \
"$BINDIR"/unit/splinter_test ${Use_shmem} --num-inserts ${num_rows} test_lookups
rm db

unset VERBOSE

msg="Splinter print diagnostics test ${use_msg}"
# shellcheck disable=SC2086
run_with_timing "${msg}" \
"$BINDIR"/unit/splinter_test ${Use_shmem} test_splinter_print_diags
rm db

# Test runs w/ default of 1M rows for --num-inserts
n_mills=1
Expand All @@ -667,6 +673,7 @@ function run_slower_unit_tests() {
# shellcheck disable=SC2086
run_with_timing "${msg}" \
"$BINDIR"/unit/large_inserts_stress_test ${Use_shmem} --num-inserts ${num_rows}
rm splinterdb_unit_tests_db

# Test runs w/ more inserts and enable bg-threads
n_mills=2
Expand All @@ -679,6 +686,7 @@ function run_slower_unit_tests() {
--num-inserts ${num_rows} \
--num-normal-bg-threads 4 \
--num-memtable-bg-threads 3
rm splinterdb_unit_tests_db
set -e
}

Expand All @@ -694,7 +702,8 @@ function run_slower_forked_process_tests() {

local msg="Splinter tests using default number of forked child processes"
run_with_timing "${msg}" "$BINDIR"/unit/splinterdb_forked_child_test

rm splinterdb_forked_child_test_db

# --------------------------------------------------------------------------
# Will be an interesting test to exercise, but ASAN job in CI failed with:
# TEST 4/4 splinterdb_forked_child:test_multiple_forked_process_doing_IOs OS-pid=1569, OS-tid=1569, Thread-ID=1, Assertion failed at src/trunk.c:5363:trunk_compact_bundle(): "height != 0".
Expand All @@ -720,6 +729,7 @@ function run_slower_forked_process_tests() {
--fork-child \
--num-inserts ${num_rows} \
test_seq_key_seq_values_inserts_forked
rm splinterdb_unit_tests_db
}

# ##################################################################
Expand All @@ -739,26 +749,30 @@ function run_splinter_functionality_tests() {
"$BINDIR"/driver_test splinter_test --functionality 1000000 100 \
$Use_shmem \
--key-size ${key_size} --seed "$SEED"
rm db

# shellcheck disable=SC2086
run_with_timing "Functionality test, with default key size${use_msg}" \
"$BINDIR"/driver_test splinter_test --functionality 1000000 100 \
$Use_shmem \
--seed "$SEED"
rm db

# shellcheck disable=SC2086
run_with_timing "Functionality test, default key size, with background threads${use_msg}" \
"$BINDIR"/driver_test splinter_test --functionality 1000000 100 \
$Use_shmem \
--num-normal-bg-threads 4 --num-memtable-bg-threads 2 \
--seed "$SEED"
rm db

max_key_size=102
# shellcheck disable=SC2086
run_with_timing "Functionality test, key size=maximum (${max_key_size} bytes)${use_msg}" \
"$BINDIR"/driver_test splinter_test --functionality 1000000 100 \
$Use_shmem \
--key-size ${max_key_size} --seed "$SEED"
rm db
}

# ##################################################################
Expand All @@ -784,6 +798,7 @@ function run_splinter_perf_tests() {
--num-inserts 10000 \
--cache-capacity-mib 512 \
--verbose-progress
rm db

# Re-run small perf test configuring background threads. This scenario
# validates that we can configure bg- and user-threads in one go.
Expand All @@ -797,6 +812,7 @@ function run_splinter_perf_tests() {
--cache-capacity-mib 512 \
--num-normal-bg-threads 1 \
--num-memtable-bg-threads 1
rm db

# shellcheck disable=SC2086
run_with_timing "Performance test${use_msg}" \
Expand All @@ -808,6 +824,7 @@ function run_splinter_perf_tests() {
--num-range-lookup-threads 0 \
--tree-size-gib 2 \
--cache-capacity-mib 512
rm db
}

# ##################################################################
Expand All @@ -824,23 +841,27 @@ function run_btree_tests() {
"$BINDIR"/driver_test btree_test --key-size ${key_size} \
$Use_shmem \
--seed "$SEED"
rm db

# shellcheck disable=SC2086
run_with_timing "BTree test, with default key size${use_msg}" \
"$BINDIR"/driver_test btree_test $Use_shmem --seed "$SEED"
rm db

key_size=100
# shellcheck disable=SC2086
run_with_timing "BTree test, key size=${key_size} bytes${use_msg}" \
"$BINDIR"/driver_test btree_test $Use_shmem \
--key-size ${key_size} --seed "$SEED"
rm db

# shellcheck disable=SC2086
run_with_timing "BTree Perf test${use_msg}" \
"$BINDIR"/driver_test btree_test --perf \
--cache-capacity-gib 4 \
--seed "$SEED" \
$Use_shmem
rm db
}

# ##################################################################
Expand All @@ -854,14 +875,17 @@ function run_other_driver_tests() {
# shellcheck disable=SC2086
run_with_timing "Cache test${use_msg}" \
"$BINDIR"/driver_test cache_test --seed "$SEED" $Use_shmem
rm db

# shellcheck disable=SC2086
run_with_timing "Log test${use_msg}" \
"$BINDIR"/driver_test log_test --seed "$SEED" $Use_shmem
rm db

# shellcheck disable=SC2086
run_with_timing "Filter test${use_msg}" \
"$BINDIR"/driver_test filter_test --seed "$SEED" $Use_shmem
rm db
}

# #######################################################################
Expand All @@ -882,12 +906,14 @@ function run_tests_with_shared_memory() {

# Run all the unit-tests first, to get basic coverage of shared-memory support.
run_with_timing "Fast unit tests using shared memory" "$BINDIR"/unit_test "--use-shmem"
rm splinterdb_unit_tests_db

# Additional case exercised while developing shared memory support for multi
# process execution to verify management of IO-contexts under forked processes
run_with_timing "IO APIs test using shared memory and forked child" \
"$BINDIR"/driver_test io_apis_test \
--use-shmem --fork-child
rm splinterdb_io_apis_test_db

Use_shmem="--use-shmem" run_slower_unit_tests
if [ -f "${UNIT_TESTS_DB_DEV}" ]; then rm "${UNIT_TESTS_DB_DEV}"; fi
Expand Down
2 changes: 2 additions & 0 deletions tests/unit/btree_stress_test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,8 @@ iterator_seek_tests(cache *cc,

btree_iterator_deinit(&dbiter);

platform_free(hid, keybuf);

return 1;
}

Expand Down