diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 570a3008645..bd80dca0d85 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,7 +1,7 @@ # Run the Bandit scanner for finding vulnerabilities in the python code. name: bandit -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5eac157c2e6..1624dd270ab 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,7 +3,7 @@ # This workflow does not use docker containers. name: CodeQL -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 1faa178053a..cc7a6c5b402 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -2,7 +2,7 @@ # checks basic builds with various compilers and executes all sets of tests. name: PMDK -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk @@ -26,16 +26,12 @@ jobs: SRC_CHECKERS: 0 strategy: matrix: - CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: [ + "N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1", "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1", "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug SRC_CHECKERS=1", - "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", - "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug", - "N=6 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", - "N=7 OS=fedora OS_VER=37 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", - "N=8 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PUSH_IMAGE=1", - "N=9 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", - "N=10 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" + "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", + "N=5 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug", ] steps: - name: Clone the git repo @@ -43,6 +39,9 @@ jobs: with: fetch-depth: 50 + - name: Get system information + run: ./$WORKDIR/get-system-info.sh + - name: Pull or rebuild the image run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 95e1cf6f7fd..46853ffb85b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,16 +30,22 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - CONFIG: ["OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: [ + "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", + "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", + "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", + "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug", + "OS=opensuse-leap OS_VER=15 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", "OS=rockylinux OS_VER=8 TEST_BUILD=debug", "OS=rockylinux OS_VER=8 TEST_BUILD=nondebug", "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", "OS=rockylinux OS_VER=9 TEST_BUILD=debug", "OS=rockylinux OS_VER=9 TEST_BUILD=nondebug", - "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1" + "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", + "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1", + "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", ] steps: - name: Clone the git repo @@ -47,6 +53,9 @@ jobs: with: fetch-depth: 50 + - name: Get system information + run: ./$WORKDIR/get-system-info.sh + - name: Pull or rebuild the image run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild diff --git a/.github/workflows/pkg_linux.yml b/.github/workflows/pkg_linux.yml deleted file mode 100644 index ee2a84d7d64..00000000000 --- a/.github/workflows/pkg_linux.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Run extra checks for making package. -name: PKG_linux - -on: [push, pull_request] - -jobs: - linux: - name: PKG_Linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - CONFIG: ["OS=ubuntu OS_VER=22.04", - "OS=rockylinux OS_VER=9", - "OS=opensuse-leap OS_VER=15" - ] - env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - VALGRIND: 1 - SRC_CHECKERS: 0 - CHECK_PKGS: 1 - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh - - # This step will create docker containers, build pmdk packages and test it. - - name: Build the CI and test packages - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh diff --git a/.github/workflows/ras_linux.yml b/.github/workflows/ras_linux.yml index 852e7b1b0a6..20102347daa 100644 --- a/.github/workflows/ras_linux.yml +++ b/.github/workflows/ras_linux.yml @@ -24,7 +24,7 @@ # utils/gha-runners/run-ras-linux.yml name: RAS_Linux -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: linux: diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 645ca915435..dfc4c3ea861 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -3,7 +3,7 @@ # This workflow is run on 'self-hosted' runners. name: PMDK_runners -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: linux: @@ -14,6 +14,14 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] + config: ['check TEST_BUILD=debug', + 'check TEST_BUILD=nondebug', + 'check TEST_BUILD=static-debug', + 'check TEST_BUILD=static-nondebug', + 'pycheck TEST_BUILD=debug', + 'pycheck TEST_BUILD=release', + 'pycheck TEST_BUILD=static_debug', + 'pycheck TEST_BUILD=static_release'] env: WORKDIR: utils/gha-runners @@ -31,4 +39,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: make check + run: make ${{ matrix.config }} diff --git a/.github/workflows/unittest_long.yml b/.github/workflows/unittest_long.yml index fc25542ee04..503259d4a7f 100644 --- a/.github/workflows/unittest_long.yml +++ b/.github/workflows/unittest_long.yml @@ -18,8 +18,15 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS -t long', - 'RUNTESTS.py -t long'] + config: ['RUNTESTS -t long -b debug', + 'RUNTESTS -t long -b nondebug', + 'RUNTESTS -t long -b static-debug', + 'RUNTESTS -t long -b static-nondebug', + 'RUNTESTS.py -t long -b debug', + 'RUNTESTS.py -t long -b release', + 'RUNTESTS.py -t long -b static_debug', + 'RUNTESTS.py -t long -b static_release'] + env: WORKDIR: utils/gha-runners diff --git a/.github/workflows/unittest_valgrind_py.yml b/.github/workflows/unittest_valgrind_py.yml index b9827681665..b9a6bca86c1 100644 --- a/.github/workflows/unittest_valgrind_py.yml +++ b/.github/workflows/unittest_valgrind_py.yml @@ -17,11 +17,10 @@ jobs: strategy: fail-fast: false matrix: + config: ['drd', 'pmemcheck', 'memcheck', 'helgrind'] + build: ['debug', 'release', 'static_debug', 'static_release'] os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS.py --force-enable memcheck', - 'RUNTESTS.py --force-enable pmemcheck', - 'RUNTESTS.py --force-enable drd', - 'RUNTESTS.py --force-enable helgrind'] + env: WORKDIR: utils/gha-runners @@ -39,4 +38,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: cd src/test/ && ./${{ matrix.config }} + run: cd src/test/ && ./RUNTESTS.py --force-enable ${{ matrix.config }} -b ${{ matrix.build }} diff --git a/.github/workflows/unittest_valgrind_sh.yml b/.github/workflows/unittest_valgrind_sh.yml index 2f8c590656d..ce885ecc173 100644 --- a/.github/workflows/unittest_valgrind_sh.yml +++ b/.github/workflows/unittest_valgrind_sh.yml @@ -17,11 +17,10 @@ jobs: strategy: fail-fast: false matrix: + config: ['-d', '-p', '-m', '-e'] + build: ['debug', 'nondebug', 'static-debug', 'static-nondebug'] os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS -m force-enable', - 'RUNTESTS -p force-enable', - 'RUNTESTS -d force-enable', - 'RUNTESTS -e force-enable'] + env: WORKDIR: utils/gha-runners @@ -39,4 +38,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: cd src/test/ && ./${{ matrix.config }} + run: cd src/test/ && ./RUNTESTS -b ${{ matrix.build }} ${{ matrix.config }} force-enable diff --git a/ChangeLog b/ChangeLog index f2156164b4b..d3dcceb549d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,33 @@ +Wed May 31 2023 Oksana Sałyk + + * Version 1.13.1 + + This release : + - Identifies and temporarily disables tests permanently or sporadically + failing in some configurations: + - ex_libpmem2 (#5593, #5635, #5638, #5641) + - obj_defrag_advanced (#5690) + - obj_sync (#5643, #5646) + - pmem2_future (#5596) + - pmem2_map (#5600) + - pmem2_map_prot (#5599) + - pmem2_mem_ext (#5640) + - pmem2_mover (partially fixed) (#5595, #5686, #5694) + - pmem2_vm_reservation (#5592) + + - Fixes failing tests: + - obj_tx_alloc_mt (#5680) + - pmem2_map_from_existing (#5594) + + - Temporarily disable pmem2_badblock test (#5650) due to missing infrastructure + (#5636, #5652) + + Other changes and notable bug fixes: + - pmem2: fix valgrind instrumentation in pmem2_map_from_exisiting (#5605) + - pmem2: remove data_mover memory leak (#5681, #5637) + - test: update permissions of DAX devices after clearing bad blocks (#5611) + - test: disable avx512f tests (pmem2_mem_ext) under Valgrind (#5658) + Wed Apr 26 2023 Łukasz Stolarczuk * Version 1.13.0 diff --git a/README.md b/README.md index d0da674b414..3faee656186 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ To build from source, clone this tree: For a stable version, checkout a [release tag](https://github.com/pmem/pmdk/releases) as follows. Otherwise skip this step to build the latest development release. ``` - $ git checkout tags/1.13.0 + $ git checkout tags/1.13.1 ``` Once the build system is setup, the Persistent Memory Development Kit is built using the `make` command at the top level: diff --git a/src/libpmem2/map_posix.c b/src/libpmem2/map_posix.c index e914d5251d0..073fe88ec0e 100644 --- a/src/libpmem2/map_posix.c +++ b/src/libpmem2/map_posix.c @@ -621,10 +621,11 @@ pmem2_map_delete(struct pmem2_map **map_ptr) if (ret) goto err_register_map; } - - if (!map->custom_vdm) - mover_delete(map->vdm); } + + if (!map->custom_vdm) + mover_delete(map->vdm); + Free(map); *map_ptr = NULL; diff --git a/src/libpmem2/mover.c b/src/libpmem2/mover.c index 25f5b12b490..e2dbadc1ea5 100644 --- a/src/libpmem2/mover.c +++ b/src/libpmem2/mover.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2021-2022, Intel Corporation */ +/* Copyright 2021-2023, Intel Corporation */ /* * mover.c -- default pmem2 data mover @@ -12,6 +12,7 @@ #include "out.h" #include "pmem2_utils.h" #include "util.h" +#include "alloc.h" #include #include #include @@ -194,6 +195,7 @@ mover_new(struct pmem2_map *map, struct vdm **vdm) goto membuf_failed; } + LOG(3, "dms %p", dms); return 0; membuf_failed: @@ -207,8 +209,9 @@ mover_new(struct pmem2_map *map, struct vdm **vdm) void mover_delete(struct vdm *dms) { + LOG(3, "dms %p", dms); membuf_delete(((struct data_mover *)dms)->membuf); - free((struct data_mover *)dms); + Free((struct data_mover *)dms); } /* diff --git a/src/test/RUNTESTS.py b/src/test/RUNTESTS.py index 1bc50e489ab..46742bae003 100755 --- a/src/test/RUNTESTS.py +++ b/src/test/RUNTESTS.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2022, Intel Corporation +# Copyright 2019-2023, Intel Corporation """Main script for unit tests execution. @@ -132,9 +132,14 @@ def run_tests(self): return ret def _test_failed(self, tc, ctx, fail): - self.msg.print('{}: {}FAILED{}\t({}/{})' + """Print message specific for failed test""" + if self.config.tm: + tm = '\t[{:06.3F} s]'.format(tc.elapsed) + else: + tm = '' + self.msg.print('{}: {}FAILED{}\t({}/{}) {}' .format(tc, futils.Color.RED, - futils.Color.END, tc.test_type, ctx)) + futils.Color.END, tc.test_type, ctx, tm)) self.msg.print(fail) if not self.config.keep_going: @@ -143,7 +148,7 @@ def _test_failed(self, tc, ctx, fail): def _test_passed(self, tc): """Print message specific for passed test""" if self.config.tm: - tm = '\t\t\t[{:06.3F} s]'.format(tc.elapsed) + tm = '\t[{:06.3F} s]'.format(tc.elapsed) else: tm = '' diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 5183cf91cef..1221f509d8b 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -4,6 +4,7 @@ # import futils import testframework as t +from testframework import granularity as g @t.require_build(['debug', 'release']) @@ -73,7 +74,10 @@ def run(self, ctx): ctx.exec(example_path, *args, stdout_file='out4.log') -class TEST5(EX_LIBPMEM2): +# XXX Disable the test execution under pmemcheck with g.PAGE until the issue +# https://github.com/pmem/pmdk/issues/5641 is fixed. +# additionall test TEST501 has been added to cover non-pmemcheck configs. +class EX_LIBPMEM2_TEST5(EX_LIBPMEM2): def run(self, ctx): example_path = futils.get_example_path(ctx, 'pmem2', 'unsafe_shutdown') @@ -82,14 +86,40 @@ def run(self, ctx): ctx.exec(example_path, "read", file_path, stdout_file='out5.log') +@g.require_granularity(g.CACHELINE, g.BYTE) # to be removed when fixed +@t.require_valgrind_enabled('pmemcheck') # to be removed when fixed +class TEST5(EX_LIBPMEM2_TEST5): + + pass + + +# XXX Disable the test execution with 'memcheck' until the issue: +# https://github.com/pmem/pmdk/issues/5635 is fixed. +# additionall test TEST501 has been added to cover non-pmemcheck configs. +# @t.require_valgrind_disabled('memcheck') # to be removed when fixed +# @t.require_valgrind_disabled('pmemcheck') +@t.require_valgrind_disabled('pmemcheck', 'memcheck') +class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed + + pass + + @t.windows_exclude +# XXX disable the test for `memcheck' and 'helgrind' +# until https://github.com/pmem/pmdk/issues/5638 is fixed. +# @t.require_valgrind_disabled('memcheck', 'helgrind') +# XXX disable the test for `drd' +# until https://github.com/pmem/pmdk/issues/5593 is fixed. +# @t.require_valgrind_disabled('drd') # This test case would require two VALGRIND_SET_CLEAN() calls # to be added to the "src/examples/libpmem2/ringbuf/ringbuf.c" # example (see https://github.com/pmem/pmdk/pull/5604) # in order to pass under pmemcheck, but examples # do not use valgrind macros on purpose (to avoid unnecessary # complication), so this test case just should not be run under pmemcheck. -@t.require_valgrind_disabled('pmemcheck') +# @t.require_valgrind_disabled('pmemcheck') +# XXX _disabled() can be used only once. +@t.require_valgrind_disabled('memcheck', 'drd', 'pmemcheck', 'helgrind') class TEST6(EX_LIBPMEM2): def run(self, ctx): diff --git a/src/test/obj_defrag_advanced/TESTS.py b/src/test/obj_defrag_advanced/TESTS.py index e4447bcd390..a33200c44dd 100755 --- a/src/test/obj_defrag_advanced/TESTS.py +++ b/src/test/obj_defrag_advanced/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -103,8 +103,12 @@ class TEST4(ObjDefragAdvancedMt): ncycles = 25 +# XXX disable the test for 'drd' +# until https://github.com/pmem/pmdk/issues/5690 is fixed. +# previousely the test has been disabled for other Valgrind options # This test last too long under helgrind/memcheck/pmemcheck -@t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck') +# @t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck') +@t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck', 'drd') class TEST5(ObjDefragAdvancedMt): max_nodes = 256 diff --git a/src/test/obj_sync/TEST2 b/src/test/obj_sync/DISABLE_TEST2 similarity index 65% rename from src/test/obj_sync/TEST2 rename to src/test/obj_sync/DISABLE_TEST2 index cda7a4dc13f..72aa1510d40 100755 --- a/src/test/obj_sync/TEST2 +++ b/src/test/obj_sync/DISABLE_TEST2 @@ -1,11 +1,15 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST2 -- unit test for PMEM-resident locks # +# XXX disable the test for 'helgrind' +# until https://github.com/pmem/pmdk/issues/5646 is fixed +# by adding the prefix DISABLE_ to the script name. + . ../unittest/unittest.sh require_test_type medium diff --git a/src/test/obj_sync/TEST5 b/src/test/obj_sync/DISABLE_TEST5 similarity index 66% rename from src/test/obj_sync/TEST5 rename to src/test/obj_sync/DISABLE_TEST5 index fc13b065c16..88fe36477e5 100755 --- a/src/test/obj_sync/TEST5 +++ b/src/test/obj_sync/DISABLE_TEST5 @@ -1,11 +1,15 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST5 -- unit test for PMEM-resident locks # +# XXX disable the test for 'drd' +# until https://github.com/pmem/pmdk/issues/5646 is fixed +# by adding the prefix DISABLE_ to the script name. + . ../unittest/unittest.sh require_test_type medium diff --git a/src/test/obj_sync/TEST6 b/src/test/obj_sync/DISABLE_TEST6 similarity index 65% rename from src/test/obj_sync/TEST6 rename to src/test/obj_sync/DISABLE_TEST6 index 8c11d2ad23f..22d3c80d247 100755 --- a/src/test/obj_sync/TEST6 +++ b/src/test/obj_sync/DISABLE_TEST6 @@ -1,11 +1,15 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST6 -- unit test for PMEM-resident locks # +# XXX disable the test for 'helgrind' +# until https://github.com/pmem/pmdk/issues/5646 is fixed +# by adding the prefix DISABLE_ to the script name. + . ../unittest/unittest.sh require_test_type medium diff --git a/src/test/obj_sync/TEST0 b/src/test/obj_sync/TEST0 index 824a32eae8f..570b86a6b8f 100755 --- a/src/test/obj_sync/TEST0 +++ b/src/test/obj_sync/TEST0 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST0 -- unit test for PMEM-resident locks @@ -12,6 +12,9 @@ require_test_type medium require_fs_type none require_build_type debug nondebug +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5643 is fixed. +configure_valgrind pmemcheck force-disable setup diff --git a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c index 9199a9ef418..fb34e4f5c0b 100644 --- a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c +++ b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c @@ -23,20 +23,8 @@ static unsigned Threads; static unsigned Ops_per_thread; static unsigned Tx_per_thread; -struct action { - struct pobj_action pact; - os_mutex_t lock; - os_cond_t cond; -}; - -struct root { - uint64_t offs[MAX_THREADS][MAX_OPS_PER_THREAD]; - struct action actions[MAX_THREADS][MAX_OPS_PER_THREAD]; -}; - struct worker_args { PMEMobjpool *pop; - struct root *r; unsigned idx; }; @@ -167,21 +155,11 @@ main(int argc, char *argv[]) UT_FATAL("!pmemobj_open"); } - PMEMoid oid = pmemobj_root(pop, sizeof(struct root)); - struct root *r = pmemobj_direct(oid); - UT_ASSERTne(r, NULL); - struct worker_args args[MAX_THREADS]; for (unsigned i = 0; i < Threads; ++i) { args[i].pop = pop; - args[i].r = r; args[i].idx = i; - for (unsigned j = 0; j < Ops_per_thread; ++j) { - struct action *a = &r->actions[i][j]; - util_mutex_init(&a->lock); - util_cond_init(&a->cond); - } } /* diff --git a/src/test/pmem2_badblock/TESTS.py b/src/test/pmem2_badblock/TESTS.py index d42f70aac08..5b57c1c475e 100755 --- a/src/test/pmem2_badblock/TESTS.py +++ b/src/test/pmem2_badblock/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2021, Intel Corporation +# Copyright 2021-2023, Intel Corporation # import os @@ -69,7 +69,8 @@ def run_test(self, ctx, filepath): bbTool.clear_all(filepath) -class TEST0(PMEM2_BADBLOCK_COUNT): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST0(PMEM2_BADBLOCK_COUNT): """ compares the number of bad blocks returned by pmem2 and ndctl on fsdax """ @@ -80,8 +81,9 @@ def run(self, ctx): self.run_test(ctx, filepath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST1(PMEM2_BADBLOCK_COUNT): +class DISABLE_TEST1(PMEM2_BADBLOCK_COUNT): """ compares the number of bad blocks returned by pmem2 and ndctl on devdax """ @@ -92,7 +94,8 @@ def run(self, ctx): self.run_test(ctx, ddpath) -class TEST2(PMEM2_BADBLOCK): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST2(PMEM2_BADBLOCK): """test mcsafe read operation with encountered badblock""" test_case = "test_pmem2_src_mcsafe_badblock_read" @@ -101,7 +104,8 @@ def run(self, ctx): self.run_test(ctx, filepath) -class TEST3(PMEM2_BADBLOCK): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST3(PMEM2_BADBLOCK): """test mcsafe write operation with encountered badblock""" test_case = "test_pmem2_src_mcsafe_badblock_write" @@ -110,8 +114,9 @@ def run(self, ctx): self.run_test(ctx, filepath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST4(PMEM2_BADBLOCK): +class DISABLE_TEST4(PMEM2_BADBLOCK): """test mcsafe read operation with encountered badblock on devdax""" test_case = "test_pmem2_src_mcsafe_badblock_read" @@ -120,8 +125,9 @@ def run(self, ctx): self.run_test(ctx, ddpath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST5(PMEM2_BADBLOCK): +class DISABLE_TEST5(PMEM2_BADBLOCK): """test mcsafe write operation with encountered badblock on devdax""" test_case = "test_pmem2_src_mcsafe_badblock_write" diff --git a/src/test/pmem2_future/TESTS.py b/src/test/pmem2_future/TESTS.py index 60b6a735878..237b8c3b9b7 100755 --- a/src/test/pmem2_future/TESTS.py +++ b/src/test/pmem2_future/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # import sys import testframework as t @@ -72,21 +72,33 @@ def run(self, ctx): F"Got {len(matches)} instead.") +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST0(PMEM2_FUTURE): size = 64 test_case = 'test_pmem2_future_mover' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST1(PMEM2_FUTURE): size = 4096 test_case = 'test_pmem2_future_mover' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST2(PMEM2_FUTURE): size = 64 test_case = 'test_pmem2_future_vdm' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST3(PMEM2_FUTURE): size = 4096 test_case = 'test_pmem2_future_vdm' diff --git a/src/test/pmem2_integration/TESTS.py b/src/test/pmem2_integration/TESTS.py index 2b6b3500a21..527fe027156 100755 --- a/src/test/pmem2_integration/TESTS.py +++ b/src/test/pmem2_integration/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2021, Intel Corporation +# Copyright 2019-2023, Intel Corporation # diff --git a/src/test/pmem2_map/TESTS.py b/src/test/pmem2_map/TESTS.py index 23427017781..9fd6a4b7d25 100755 --- a/src/test/pmem2_map/TESTS.py +++ b/src/test/pmem2_map/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2021, Intel Corporation +# Copyright 2019-2023, Intel Corporation # import os @@ -43,12 +43,18 @@ def run(self, ctx): ctx.exec('pmem2_map', self.test_case, dd.path) +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST0(PMEM2_MAP): """map a O_RDWR file""" test_case = "test_map_rdrw_file" with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST1(PMEM2_MAP_DEVDAX): """DevDax map a O_RDWR file""" test_case = "test_map_rdrw_file" @@ -67,11 +73,17 @@ class TEST3(PMEM2_MAP_DEVDAX): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST4(PMEM2_MAP): """map valid memory ranges""" test_case = "test_map_valid_ranges" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST5(PMEM2_MAP_DEVDAX): """DevDax map valid memory ranges""" test_case = "test_map_valid_ranges" @@ -102,21 +114,33 @@ class TEST10(PMEM2_MAP): test_case = "test_map_invalid_fd" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST11(PMEM2_MAP): """unmap valid pmem2 mapping""" test_case = "test_unmap_valid" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST12(PMEM2_MAP_DEVDAX): """DevDax unmap valid pmem2 mapping""" test_case = "test_unmap_valid" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST13(PMEM2_MAP): """unmap a pmem2 mapping with an invalid length""" test_case = "test_unmap_zero_length" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST14(PMEM2_MAP_DEVDAX): """DevDax unmap a pmem2 mapping with an invalid length""" test_case = "test_unmap_zero_length" @@ -124,11 +148,17 @@ class TEST14(PMEM2_MAP_DEVDAX): # UnmapViewOfFile does not care about the address alignment @t.windows_exclude +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST15(PMEM2_MAP): """unmap a pmem2 mapping with an unaligned address""" test_case = "test_unmap_unaligned_addr" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST16(PMEM2_MAP_DEVDAX): """DevDax unmap a pmem2 mapping with an unaligned address""" test_case = "test_unmap_unaligned_addr" @@ -177,6 +207,9 @@ class TEST23(PMEM2_MAP): filesize = 0 +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST24(PMEM2_MAP): """ map a file with PMEM2_SHARED sharing, changes in the mapping are visible @@ -186,6 +219,9 @@ class TEST24(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST25(PMEM2_MAP): """ map a file with PMEM2_PRIVATE sharing, changes in the mapping are not @@ -195,6 +231,9 @@ class TEST25(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST26(PMEM2_MAP): """ map a file with PMEM2_PRIVATE sharing, changes in the mapping are not @@ -204,6 +243,9 @@ class TEST26(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST27(PMEM2_MAP): """ map O_RDONLY file with PMEM2_PRIVATE sharing @@ -220,6 +262,9 @@ class TEST28(PMEM2_MAP_DEVDAX): @t.linux_only @t.require_architectures('x86_64') +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST29(PMEM2_MAP): """map alignment test for huge pages""" test_case = "test_map_huge_alignment" @@ -228,6 +273,9 @@ class TEST29(PMEM2_MAP): @t.linux_only @t.require_architectures('x86_64') +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST30(PMEM2_MAP): """map alignment test for small pages""" test_case = "test_map_huge_alignment" diff --git a/src/test/pmem2_map_from_existing/TESTS.py b/src/test/pmem2_map_from_existing/TESTS.py index 9d8e50dd41b..ab8044bbca2 100644 --- a/src/test/pmem2_map_from_existing/TESTS.py +++ b/src/test/pmem2_map_from_existing/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2021, Intel Corporation +# Copyright 2020-2023, Intel Corporation # diff --git a/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c b/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c index 3868f1da83e..7cb4371bc59 100644 --- a/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c +++ b/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020-2021, Intel Corporation */ +/* Copyright 2020-2023, Intel Corporation */ /* * pmem2_map_from_existing.c -- pmem2_map_from_existing unittests @@ -37,6 +37,7 @@ test_two_same_mappings(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } @@ -69,6 +70,7 @@ test_mapping_overlap_bottom(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } @@ -101,6 +103,7 @@ test_mapping_overlap_upper(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } diff --git a/src/test/pmem2_map_prot/TESTS.py b/src/test/pmem2_map_prot/TESTS.py index 0e945eaa5ef..fd84f515356 100644 --- a/src/test/pmem2_map_prot/TESTS.py +++ b/src/test/pmem2_map_prot/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -72,6 +72,8 @@ class TEST6(PMEM2_MAP_PROT): @t.require_architectures('x86_64') +# XXX disable the test until https://github.com/pmem/pmdk/issues/5599 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST7(PMEM2_MAP_PROT): """ READ|EXEC protection on file opened in read|write|exec mode; test runs diff --git a/src/test/pmem2_mem_ext/TESTS.py b/src/test/pmem2_mem_ext/TESTS.py index a750c61a0cd..3d0c55e1570 100755 --- a/src/test/pmem2_mem_ext/TESTS.py +++ b/src/test/pmem2_mem_ext/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2022, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -115,6 +115,14 @@ def setup(self, ctx): ret = tools.Tools(ctx.env, ctx.build).cpufd() self.check_arch(ctx.variant(), ret.returncode) + # XXX all tests with VARIANT_AVX512F are disabled under Valgrind + # until the issue https://github.com/pmem/pmdk/issues/5640 is fixed. + # "win32" `if`` is related to unknown `is not None` by Windows Python + if not sys.platform.startswith('win32') and ctx.valgrind is not None: + if ctx.valgrind.tool.name != "NONE": + if ctx.variant() == VARIANT_AVX512F: + raise futils.Skip("AVX512F unavailable under Valigrind") + def check_arch(self, variant, available_arch): if variant == VARIANT_MOVDIR64B: if available_arch < MOVDIR64B: diff --git a/src/test/pmem2_mover/TESTS.py b/src/test/pmem2_mover/TESTS.py index 90c358453e0..b7bd292f2b6 100644 --- a/src/test/pmem2_mover/TESTS.py +++ b/src/test/pmem2_mover/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # @@ -39,57 +39,132 @@ def run(self, ctx): ctx.exec('pmem2_mover', self.test_case, self.filepath, self.thread_num) +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST0(PMEM2_MOVER): """verify pmem2 mover memcpy functionality""" test_case = "test_mover_memcpy_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST1(PMEM2_MOVER): """verify pmem2 mover memmove functionality""" test_case = "test_mover_memmove_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST2(PMEM2_MOVER): """verify pmem2 mover memset functionality""" test_case = "test_mover_memset_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST3(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memcpy functionality""" test_case = "test_mover_memcpy_multithreaded" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST4(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memmove functionality""" test_case = "test_mover_memmove_multithreaded" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST5(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memset functionality""" test_case = "test_mover_memset_multithreaded" -class TEST6(PMEM2_MOVER_MT): +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST6(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memcpy functionality (Long)""" test_case = "test_mover_memcpy_multithreaded" -class TEST7(PMEM2_MOVER_MT): +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST6(PMEM2_MOVER_MT_TEST6): + pass + + +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') +@g.require_granularity(g.BYTE) +class TEST61(PMEM2_MOVER_MT_TEST6): + pass + + +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST7(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memmove functionality (Long)""" test_case = "test_mover_memmove_multithreaded" -class TEST8(PMEM2_MOVER_MT): +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST7(PMEM2_MOVER_MT_TEST7): + pass + + +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') +@g.require_granularity(g.BYTE) +class TEST71(PMEM2_MOVER_MT_TEST7): + pass + + +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST8(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memset functionality (Long)""" test_case = "test_mover_memset_multithreaded" +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST8(PMEM2_MOVER_MT_TEST8): + pass + + +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') +@g.require_granularity(g.BYTE) +class TEST81(PMEM2_MOVER_MT_TEST8): + pass + + +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST9(PMEM2_MOVER): """verify pmem2 mover functionality""" test_case = "test_miniasync_mover" diff --git a/src/test/pmem2_mover/pmem2_mover.c b/src/test/pmem2_mover/pmem2_mover.c index 45d9deb6d1d..3e953551054 100644 --- a/src/test/pmem2_mover/pmem2_mover.c +++ b/src/test/pmem2_mover/pmem2_mover.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2019-2022, Intel Corporation */ +/* Copyright 2019-2023, Intel Corporation */ /* * pmem2_mover.c -- pmem2 mover tests @@ -113,6 +113,7 @@ test_mover_memmove_basic(const struct test_case *tc, int argc, char *argv[]) if (memcmp(data + string_size / 2, buffer_data, string_size)) UT_FATAL("data should be equal"); + free(buffer_data); pmem2_map_delete(&map); pmem2_config_delete(&cfg); pmem2_source_delete(&src); @@ -162,6 +163,7 @@ test_mover_memset_basic(const struct test_case *tc, int argc, char *argv[]) if (memcmp(data, memset_result, array_size)) UT_FATAL("data should be equal"); + free(memset_result); pmem2_map_delete(&map); pmem2_config_delete(&cfg); pmem2_source_delete(&src); diff --git a/src/test/pmem2_vm_reservation/TESTS.py b/src/test/pmem2_vm_reservation/TESTS.py index f95befefdea..e5cdfa85638 100755 --- a/src/test/pmem2_vm_reservation/TESTS.py +++ b/src/test/pmem2_vm_reservation/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2021, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import os @@ -140,6 +140,9 @@ class TEST10(PMEM2_VM_RESERVATION): test_case = "test_vm_reserv_new_region_occupied_map_below" +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST11(PMEM2_VM_RESERVATION_DEVDAX): """ DevDax create a reservation in the region overlapping lower half of @@ -212,6 +215,9 @@ class TEST21(PMEM2_VM_RESERVATION): test_case = "test_vm_reserv_map_unmap_multiple_files" +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST22(PMEM2_VM_RESERVATION_DEVDAX): """ DevDax map multiple files to a vm reservation, then @@ -293,6 +299,9 @@ class TEST31(PMEM2_VM_RESERVATION_ASYNC): ops_per_thread = 1000 +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST32(PMEM2_VM_RESERVATION_ASYNC_DEVDAX): """ DevDax map and unmap asynchronously multiple times to the whole diff --git a/src/test/unittest/basetest.py b/src/test/unittest/basetest.py index c4f9cc8d6a6..eda966ef0f0 100644 --- a/src/test/unittest/basetest.py +++ b/src/test/unittest/basetest.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2020, Intel Corporation +# Copyright 2019-2023, Intel Corporation """Base tests class and its functionalities. @@ -179,6 +179,7 @@ def _execute(self, c): self.check(c) except futils.Fail: + self.elapsed = (datetime.now() - start_time).total_seconds() self._on_fail() raise @@ -191,6 +192,7 @@ def _execute(self, c): msg = '{}: {}TIMEOUT{}\t({})'.format(self, futils.Color.RED, futils.Color.END, self.ctx) + self.elapsed = (datetime.now() - start_time).total_seconds() raise futils.Fail(msg) else: diff --git a/utils/docker/build-CI.sh b/utils/docker/build-CI.sh index 4498d3dc574..ce2313536a4 100755 --- a/utils/docker/build-CI.sh +++ b/utils/docker/build-CI.sh @@ -38,7 +38,6 @@ if [[ $MAKE_PKG -eq 1 ]] ; then command="./run-build-package.sh"; fi if [[ $COVERAGE -eq 1 ]] ; then command="./run-coverage.sh"; fi if [[ $BANDIT -eq 1 ]] ; then command="./run-bandit.sh"; fi if [[ "$COVERITY" -eq 1 ]]; then command="./run-coverity.sh"; fi -if [[ $CHECK_PKGS -eq 1 ]] ; then command="./build-and-test-pmdk-packages.sh"; fi if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi if [[ -f $CI_FILE_SKIP_BUILD_PKG_CHECK ]]; then BUILD_PACKAGE_CHECK=n; else BUILD_PACKAGE_CHECK=y; fi diff --git a/utils/docker/build-and-test-pmdk-packages.sh b/utils/docker/build-and-test-pmdk-packages.sh deleted file mode 100755 index 224f54bc627..00000000000 --- a/utils/docker/build-and-test-pmdk-packages.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2023, Intel Corporation - -# build-and-test-pmdk-packages.sh - is called inside a Docker container; Script for building and testing pmdk packages. - -set -eo pipefail - -# Prepare build environment -# Adding 'PKG' parameter is required for PKG tests. -./prepare-for-build.sh PKG - -# build_pmdk_package -- build pmdk package depending on the Linux distribution. -function build_pmdk_package { - cd $WORKDIR - echo "********** make package **********" - date - if [ $OS = opensuse/leap ] || [ $OS = rockylinux/rockylinux ]; then - make rpm EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - elif [ $OS = ubuntu ]; then - make dpkg EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - fi -} - -# Build pmdk to create src/nondebug directory used by python scripts in tests. -function build_pmdk { - cd $WORKDIR - echo "********** make pmdk **********" - make EXTRA_CFLAGS=-DUSE_VALGRIND -} - -# test_built_packages -- test if the packages are built correctly. -function test_built_packages { - echo "********** test built packages **********" - python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) -} - -# install_packages -- install built packages. -function install_packages { - echo "********** install packages **********" - sudo python3 $SCRIPTSDIR/packages/install-packages.py -r $(pwd) -} - -# test_installed_packages -- test if the packages were installed correctly. -function test_installed_packages { - echo "********** test installed packages **********" - python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) -} - -function build_unittests { - echo "********** build unittests **********" - cd $WORKDIR/src/test && make clobber - make test -} - -function run_check_unittests { - echo "********** run tests **********" - cd $WORKDIR/src/test && ./RUNTESTS -t check -o 60m -} - -build_pmdk_package -build_pmdk -test_built_packages -install_packages -test_installed_packages -build_unittests -run_check_unittests diff --git a/utils/docker/get-system-info.sh b/utils/docker/get-system-info.sh new file mode 100755 index 00000000000..dde7230d213 --- /dev/null +++ b/utils/docker/get-system-info.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +# +# get-system-info.sh - Script for printing system info +# + +function system_info { + echo "********** system_info **********" + cat /etc/os-release | grep -oP "PRETTY_NAME=\K.*" + uname -r + echo "libndctl: $(pkg-config --modversion libndctl || echo 'libndctl not found')" + echo "valgrind: $(pkg-config --modversion valgrind || echo 'valgrind not found')" + echo "*************** installed-packages ***************" + # Instructions below will return some minor errors, as they are dependent on the Linux distribution. + zypper se --installed-only 2>/dev/null || true + apt list --installed 2>/dev/null || true + yum list installed 2>/dev/null || true + echo "**********/proc/cmdline**********" + cat /proc/cmdline + echo "**********/proc/modules**********" + cat /proc/modules + echo "**********/proc/cpuinfo**********" + cat /proc/cpuinfo + echo "**********/proc/meminfo**********" + cat /proc/meminfo + echo "**********/proc/swaps**********" + cat /proc/swaps + echo "**********/proc/version**********" + cat /proc/version + echo "**********check-updates**********" + # Instructions below will return some minor errors, as they are dependent on the Linux distribution. + zypper list-updates 2>/dev/null || true + apt-get update 2>/dev/null || true + apt upgrade --dry-run 2>/dev/null || true + dnf check-update 2>/dev/null || true + echo "**********list-enviroment**********" + env + echo "******list-build-system-versions*******" + gcc --version 2>/dev/null || true + clang --version 2>/dev/null || true + make --version 2>/dev/null || true +} + +# Call the function above to print system info. +system_info diff --git a/utils/docker/images/Dockerfile.fedora-37 b/utils/docker/images/Dockerfile.fedora-37 index cef6e817458..47f6e92d56e 100644 --- a/utils/docker/images/Dockerfile.fedora-37 +++ b/utils/docker/images/Dockerfile.fedora-37 @@ -55,6 +55,7 @@ ENV TESTS_DEPS "\ libunwind-devel \ ndctl \ openssh-server \ + python3 \ strace" # packaging dependencies @@ -70,6 +71,7 @@ ENV MISC_DEPS "\ lbzip2 \ man \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -77,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -92,6 +99,8 @@ RUN dnf update -y && dnf install -y \ && ./install-valgrind.sh fedora \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add and switch user ENV USER pmdkuser ENV USERPASS pmdkpass diff --git a/utils/docker/images/Dockerfile.rockylinux-8 b/utils/docker/images/Dockerfile.rockylinux-8 index 03a2daab5f8..b562f6a13ef 100644 --- a/utils/docker/images/Dockerfile.rockylinux-8 +++ b/utils/docker/images/Dockerfile.rockylinux-8 @@ -71,6 +71,7 @@ ENV MISC_DEPS "\ man \ python3 \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -78,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -96,6 +102,8 @@ RUN dnf update -y && dnf install -y epel-release \ && ./install-valgrind.sh rockylinux \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add ans switch user ENV USER pmdkuser ENV USERPASS pmdkpass diff --git a/utils/docker/images/Dockerfile.ubuntu-22.04 b/utils/docker/images/Dockerfile.ubuntu-22.04 index b60bd18766b..cbf2b8949d0 100644 --- a/utils/docker/images/Dockerfile.ubuntu-22.04 +++ b/utils/docker/images/Dockerfile.ubuntu-22.04 @@ -69,6 +69,8 @@ ENV COVERITY_DEPS ruby gcc g++ wget # miscellaneous dependencies (mostly for CI) ENV MISC_DEPS "clang \ + hub \ + rsync \ clang-format \ flake8 \ python3-pip \ diff --git a/utils/docker/images/install-libndctl.sh b/utils/docker/images/install-libndctl.sh deleted file mode 100755 index fc8cee3fb88..00000000000 --- a/utils/docker/images/install-libndctl.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2017-2019, Intel Corporation - -# -# install-libndctl.sh - installs libndctl -# - -set -e - -OS=$2 - -echo "==== clone ndctl repo ====" -git clone https://github.com/pmem/ndctl.git -cd ndctl -git checkout $1 - -if [ "$OS" = "fedora" ]; then - -echo "==== setup rpmbuild tree ====" -rpmdev-setuptree - -RPMDIR=$HOME/rpmbuild/ -VERSION=$(./git-version) -SPEC=./rhel/ndctl.spec - -echo "==== create source tarball =====" -git archive --format=tar --prefix="ndctl-${VERSION}/" HEAD | gzip > "$RPMDIR/SOURCES/ndctl-${VERSION}.tar.gz" - -echo "==== build ndctl ====" -./autogen.sh -./configure --disable-docs -make -j$(nproc) - -echo "==== build ndctl packages ====" -rpmbuild -ba $SPEC - -echo "==== install ndctl packages ====" -RPM_ARCH=$(uname -m) -rpm -i $RPMDIR/RPMS/$RPM_ARCH/*.rpm - -echo "==== cleanup ====" -rm -rf $RPMDIR - -else - -echo "==== build ndctl ====" -./autogen.sh -./configure --disable-docs -make -j$(nproc) - -echo "==== install ndctl ====" -make -j$(nproc) install - -echo "==== cleanup ====" - -fi - -cd .. -rm -rf ndctl diff --git a/utils/docker/packages/cleanup-packages.py b/utils/docker/packages/cleanup-packages.py deleted file mode 100644 index 8a2f8c2ffcc..00000000000 --- a/utils/docker/packages/cleanup-packages.py +++ /dev/null @@ -1,112 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which clean up environment system after -installation of rpm packages from PMDK library. -""" - -from os import listdir, path, linesep -import distro -from subprocess import check_output -from argparse import ArgumentParser -import re -import json -from pathlib import Path - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from rpm directory. - """ - version = '' - architecture = '' - os_distro=distro.id() - - # first check if there is a json file with version and architecture - pkg_version_file_path = Path(pmdk_path).joinpath("pkgVersion.json") - if pkg_version_file_path.exists() and pkg_version_file_path.is_file(): - with open(pkg_version_file_path) as pkg_version_file: - version_from_file = {} - try: - version_from_file = json.load(pkg_version_file) - version = version_from_file.get('PMDK_VERSION', '') - architecture = version_from_file.get('SYSTEM_ARCHITECTURE', '') - except: - pass - - # if cannot read values from json file, read them from rpms: - if version == '' or architecture == '': - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - for elem in listdir(pkg_directory): - if '.src.rpm' in elem: - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - for elem in listdir(pkg_directory): - if '.changes' in elem: - # looks for the version number of dpkg package in dpkg package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - - return version, architecture - - -def uninstall_rpm_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - rpm_to_uninstall = check_output( - 'rpm -qa | grep ' + PMDK_VERSION, cwd=pmdk_path, shell=True) - pkg_to_uninstall = rpm_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('rpm -e ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -def uninstall_dpkg_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - dpkg_to_uninstall = check_output( - 'dpkg-query --show | grep ' + PMDK_VERSION + ''' | awk '{print $1}' ''', cwd=pmdk_path, shell=True) - pkg_to_uninstall = dpkg_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('dpkg -r ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -if __name__ == '__main__': - parser = ArgumentParser( - description='''Clean up system environment after installation rpm - packages from PMDK library''') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - - os_distro=distro.id() - if os_distro != 'ubuntu': - uninstall_rpm_packages(args.pmdk_path) - else: - uninstall_dpkg_packages(args.pmdk_path) diff --git a/utils/docker/packages/install-packages.py b/utils/docker/packages/install-packages.py deleted file mode 100644 index b79b37cd2c3..00000000000 --- a/utils/docker/packages/install-packages.py +++ /dev/null @@ -1,188 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which install packages from PMDK library. -""" - -from os import listdir, path, linesep -from subprocess import check_output -from argparse import ArgumentParser -import distro -import re -import json - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from packages directory. - """ - os_distro=distro.id() - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - - version = '' - architecture = '' - for elem in listdir(pkg_directory): - if os_distro != 'ubuntu': - if '.src.rpm' in elem: - # looks for the version number of package in package name - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - elif os_distro == 'ubuntu': - if '.changes' in elem: - # looks for the version number of packages in package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - return version, architecture - - -def save_pkg_version(pkg_version_file_path): - values_to_save = { - 'PMDK_VERSION': PMDK_VERSION, - 'SYSTEM_ARCHITECTURE': SYSTEM_ARCHITECTURE - } - with open(pkg_version_file_path, 'w') as file: - json.dump(values_to_save, file, indent=2) - - -def get_built_rpm_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - packages = listdir(path_to_rpm_files) - return packages - - -def get_built_dpkg_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - packages ='' - for elem in listdir(path_to_dpkg_files): - if '.deb' in elem: - packages += elem + ' ' - - return packages - - -def install_rpm_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = ' '.join(get_built_rpm_packages(pmdk_path)) - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - check_output('rpm -i --nodeps ' + packages, - cwd=path_to_rpm_files, shell=True) - - -def install_dpkg_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = get_built_dpkg_packages(pmdk_path) - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - check_output('dpkg -i ' + packages, - cwd=path_to_dpkg_files, shell=True) - - -def get_names_of_rpm_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION, cwd=packages_path, shell=True) - delimiter = '-' - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '-')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_names_of_dpkg_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'dpkg') - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION + ' | grep .deb', cwd=packages_path, shell=True) - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '_')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_installed_packages(so_path, split_param, packages_path): - """ - Returns names of packages from PMDK library, which are installed. - """ - libraries = get_names_of_pkg_content_func() - installed_packages = [] - - for library in libraries: - if library == "pmempool" and check_output( - 'find /usr/bin/ -name ' + library, cwd=args.pmdk_path, - shell=True): - installed_packages.append(library) - elif library == "libpmemobj++" and check_output( - 'find /usr/include/' + library + ' -name *.hpp', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - elif library == "pmdk": - pass - elif check_output('find ' + so_path + ' -name ' + library + '.so', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - return installed_packages - - -if __name__ == '__main__': - parser = ArgumentParser( - description='Install packages from PMDK library') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - os_distro=distro.id() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - save_pkg_version(args.pmdk_path + "/pkgVersion.json") - if os_distro != 'ubuntu': - so_path = '/usr/lib64/' - split_param = '-' - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - install_cmd = 'rpm -i --nodeps ' - install_func = install_rpm_packages - get_names_of_pkg_content_func = get_names_of_rpm_content - else: - so_path = '/lib/x86_64-linux-gnu/' - split_param = '_' - packages_path = path.join(args.pmdk_path, 'dpkg') - install_cmd = 'dpkg -i ' - install_func = install_dpkg_packages - get_names_of_pkg_content_func = get_names_of_dpkg_content - - if not get_installed_packages(so_path, packages_path, get_names_of_pkg_content_func): - install_func(args.pmdk_path) - else: - print("PMDK library is still installed") diff --git a/utils/docker/prepare-for-build.sh b/utils/docker/prepare-for-build.sh index c74e65cb628..332605cd89d 100755 --- a/utils/docker/prepare-for-build.sh +++ b/utils/docker/prepare-for-build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2022, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # prepare-for-build.sh - is called inside a Docker container; prepares @@ -21,5 +21,5 @@ fi # should be preserved KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then - ./configure-tests.sh + ./configure-tests.sh $@ fi diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 4d8eda44342..c211441a524 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2021, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # run-build-package.sh - is called inside a Docker container; prepares @@ -17,27 +17,38 @@ git config user.email "test@package.com" git config user.name "test package" git tag -a 1.4.99 -m "1.4" HEAD~1 || true -# Build all and run tests -cd $WORKDIR +echo "## Build package (and run basic tests)" +pushd $WORKDIR export PCHECK_OPTS="-j2 BLACKLIST_FILE=${BLACKLIST_FILE}" make -j$(nproc) $PACKAGE_MANAGER -# Install packages +echo "## Build PMDK once more (clobber from packaging process cleared out some required files)" +make -j$(nproc) + +echo "## Test built packages" +[ "$NDCTL_ENABLE" == "n" ] && extra_params="--skip-daxio" || extra_params="" +python3 $SCRIPTSDIR/test_package/test-built-packages.py -r $(pwd) ${extra_params} + +echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then - cd $PACKAGE_MANAGER + pushd $PACKAGE_MANAGER echo $USERPASS | sudo -S dpkg --install *.deb else RPM_ARCH=$(uname -m) - cd $PACKAGE_MANAGER/$RPM_ARCH + pushd $PACKAGE_MANAGER/$RPM_ARCH echo $USERPASS | sudo -S rpm --install *.rpm fi +popd + +echo "## Test installed packages" +python3 $SCRIPTSDIR/test_package/test-packages-installation.py -r $(pwd) -# Compile and run standalone test -cd $WORKDIR/utils/docker/test_package +echo "## Compile and run standalone test" +pushd $SCRIPTSDIR/test_package make -j$(nproc) LIBPMEMOBJ_MIN_VERSION=1.4 ./test_package testfile1 -# Use pmreorder installed in the system +echo "## Use pmreorder installed in the system" pmreorder_version="$(pmreorder -v)" pmreorder_pattern="pmreorder\.py .+$" (echo "$pmreorder_version" | grep -Ev "$pmreorder_pattern") && echo "pmreorder version failed" && exit 1 @@ -45,3 +56,17 @@ pmreorder_pattern="pmreorder\.py .+$" touch testfile2 touch logfile1 pmreorder -p testfile2 -l logfile1 +popd + +echo "## Run tests (against PMDK installed in the system)" +pushd $WORKDIR/src/test +make -j$(nproc) clobber +make -j$(nproc) + +# Prepare test config once more. Now, with path to PMDK set in the OS +# (rather than in the git tree) - for testing packages installed in the system. +$SCRIPTSDIR/configure-tests.sh PKG +./RUNTESTS -t check +popd + +popd diff --git a/utils/docker/test_package/README b/utils/docker/test_package/README index de81a6a5b8e..e4015df0f78 100644 --- a/utils/docker/test_package/README +++ b/utils/docker/test_package/README @@ -2,5 +2,8 @@ Persistent Memory Development Kit This is utils/docker/test_package/README. -This directory contains simple application which uses libpmemobj. -It can be used to test whether libpmemobj was installed properly. +This directory contains: +- Simple application which uses libpmemobj. + It can be used to test whether libpmemobj was installed properly. +- Test python modules with various checks for PMDK packages. + Each module contains a detailed description of its contents. diff --git a/utils/docker/packages/test-build-packages.py b/utils/docker/test_package/test-built-packages.py similarity index 94% rename from utils/docker/packages/test-build-packages.py rename to utils/docker/test_package/test-built-packages.py index 82247028e2e..5d96c772009 100644 --- a/utils/docker/packages/test-build-packages.py +++ b/utils/docker/test_package/test-built-packages.py @@ -10,8 +10,12 @@ -if all required packages are built, -if built packages are consistent with names of libraries read from .so files and other elements (tools and "PMDK"). + Required arguments: -r the PMDK library root path. + +Optional arguments: +--skip-daxio to be set if daxio was not built (e.g., if NDCTL_ENABLE was set to 'n') """ from os import listdir, path, linesep @@ -148,10 +152,12 @@ def find_missing_packages(packages_path, pmdk_path, pmdk_debuginfo_package_name) 'pmempool': PACKAGES_INFO(basic=True, devel=False, debug=False, debuginfo=True, debug_debuginfo=False), 'pmreorder': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=False, debug_debuginfo=False), - 'daxio': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=True, debug_debuginfo=False) + debuginfo=False, debug_debuginfo=False) } + if not skip_daxio: + tools['daxio'] = PACKAGES_INFO(basic=True, devel=False, debug=False, + debuginfo=True, debug_debuginfo=False) + tools_packages = get_names_of_packages(tools) missing_tools_packages = [ elem for elem in tools_packages if elem not in built_packages] @@ -237,11 +243,17 @@ def test_completeness_of_name_of_libraries_and_tools(self): if __name__ == '__main__': path_argument = '-r' + daxio_build_argument = "--skip-daxio" + skip_daxio = False if '-h' in sys.argv or '--help' in sys.argv: print(__doc__) unittest.main() elif path_argument in sys.argv: pmdk_path = parse_argument(path_argument) + if daxio_build_argument in sys.argv: + skip_daxio = True + index = sys.argv.index(daxio_build_argument) + sys.argv.pop(index) if pmdk_path: PMDK_VERSION, SYSTEM_ARCHITECTURE =\ get_package_version_and_system_architecture(pmdk_path) diff --git a/utils/docker/packages/test-packages-installation.py b/utils/docker/test_package/test-packages-installation.py similarity index 100% rename from utils/docker/packages/test-packages-installation.py rename to utils/docker/test_package/test-packages-installation.py diff --git a/utils/gha-runners/build-pmdk.sh b/utils/gha-runners/build-pmdk.sh index 9d66b16e58d..467acbff43a 100755 --- a/utils/gha-runners/build-pmdk.sh +++ b/utils/gha-runners/build-pmdk.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # # build-pmdk.sh - Script for building pmdk project @@ -16,10 +16,10 @@ set -eo pipefail # function build_pmdk { echo "********** make pmdk **********" - cd ${PMDK_PATH} && make clean - cd ${PMDK_PATH} && make EXTRA_CFLAGS=-DUSE_VALGRIND + cd ${PMDK_PATH} && make -j$(nproc) clean + cd ${PMDK_PATH} && make -j$(nproc) EXTRA_CFLAGS=-DUSE_VALGRIND echo "********** make pmdk test **********" - cd ${PMDK_PATH}/ && make test + cd ${PMDK_PATH}/ && make -j$(nproc) test } build_pmdk diff --git a/utils/gha-runners/create-testconfig.sh b/utils/gha-runners/create-testconfig.sh index ad0ae5783f9..ed72a96951c 100755 --- a/utils/gha-runners/create-testconfig.sh +++ b/utils/gha-runners/create-testconfig.sh @@ -21,6 +21,7 @@ DEVICE_DAX_PATH=($(ndctl list -X | jq -r '.[].daxregion.devices[].chardev' | awk KEEP_GOING=y TEST_TIMEOUT=120m ENABLE_SUDO_TESTS=y +TM=1 EOL # Create config file for py tests. diff --git a/utils/gha-runners/get-system-info.sh b/utils/gha-runners/get-system-info.sh index ad061d1ce08..9a41e1c6427 100755 --- a/utils/gha-runners/get-system-info.sh +++ b/utils/gha-runners/get-system-info.sh @@ -51,6 +51,10 @@ function system_info { ls -la /sys/bus/nd/devices/ndbus*/region*/pfn*/resource ls -la /sys/bus/nd/devices/ndbus*/region*/namespace*/resource ls -la /sys/bus/nd/devices/region*/deep_flush + echo "******list-build-system-versions*******" + gcc --version 2>/dev/null || true + clang --version 2>/dev/null || true + make --version 2>/dev/null || true } # Call the function above to print system info.