Code Coverage Reporting (#334)

* Update lib/rocprofiler-sdk/counters/{tests,parser/tests}/CMakeLists.txt - use rocprofiler-static-library instead of rocprofiler-object-library * Update scripts/run-ci.py - support gcovr and pycobertura * Update CI workflow for code coverage - load/save cache for XML code coverage (via gcovr) - generate and write code coverage comment - archive code coverage HTML report - fix name for sanitizer jobs * Update CI workflow - tweaks to env for PATH and LD_LIBRARY_PATH * Add scripts/upload-image-to-github.py - script for saving images to orphan branches to be used in markdown links * Update CI workflow - fix upload artifact conflict - use upload-image-to-github.py * Update CI workflow - install extra packages for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - install more recent git - tweak package installs for wkhtmltopdf/wkhtmltoimage * Update CI workflow (code coverage) - remove duplicate --cap-add=SYS_PTRACE * Update CI and upload-image-to-github.py - print versions * Update upload-image-to-github.py - check exit code of some subprocesses * Update CI workflow - fix GITHUB_PATH ordering - fix LD_LIBRARY_PATH * Update CI workflow - fix code coverage cache keys (use SHAs) - copy .codecov to .codecov.ref if a cached .codecov exists * Update upload-image-to-github.py - Update git pull/push commands * Update upload-image-to-github.py - git fetch before pulling - git pull before committing * Update upload-image-to-github.py - git fetch after committing - git pull after committing * Update CI workflow - list files before cat * Update upload-image-to-github.py - output messages * Update CI workflow and upload-image-to-github.py - fix output directory path for script to work with CI workflow * Update CI workflow - finishing touches/fixes on the code coverage comment generation * Reproducible filenames * Update CI workflow - fix archive of code coverage data * Fix relative path of reproducible file loc * Update upload-image-to-github.py - change update method * rocprofiler-v2-internal -> rocprofiler-sdk-internal
ROCm · Jan 3, 2024 · c5e4580 · c5e4580
1 parent 3d539c1
commit c5e4580
Show file tree

Hide file tree

Showing 8 changed files with 356 additions and 25 deletions.
diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml
@@ -70,11 +70,11 @@ jobs:
     # vega20 machine only has 24 cpus available.
     # container:
     #   image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
 
     container:
       image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
 
     needs: get_latest_mainline_build_number
 
@@ -99,7 +99,7 @@ jobs:
     - name: List Files
       shell: bash
       run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
         for i in python python3 git cmake ctest; do which-realpath $i; done
         ls -la
 
@@ -167,7 +167,7 @@ jobs:
     - name: Archive production artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: artifacts
+        name: installers
         path: |
           ${{github.workspace}}/build/*.deb
           ${{github.workspace}}/build/*.rpm
@@ -199,33 +199,64 @@ jobs:
     # vega20 machine only has 24 cpus available.
     container:
       image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
 
     # container:
     #   image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
 
     needs: get_latest_mainline_build_number
 
     steps:
+    - name: Patch Git
+      timeout-minutes: 25
+      run: |
+        apt-get update
+        apt-get install -y software-properties-common
+        add-apt-repository -y ppa:git-core/ppa
+        apt-get update
+        apt-get install -y git
+
     - uses: actions/checkout@v4
       with:
         submodules: true
 
+    - name: Load Existing XML Code Coverage
+      if: github.event_name == 'pull_request'
+      id: load-coverage
+      uses: actions/cache@v3
+      with:
+        key: ${{ github.event.pull_request.base.sha }}-codecov
+        path: .codecov/**
+
+    - name: Copy Existing XML Code Coverage
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ -d .codecov ]; then cp -r .codecov .codecov.ref; fi
+
+    - name: Configure Env
+      shell: bash
+      run: |
+        echo "${PATH}:/usr/local/bin:${HOME}/.local/bin" >> $GITHUB_PATH
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:${HOME}/.local/lib" >> $GITHUB_ENV
+
     - name: List Files
       shell: bash
       run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        echo "PATH: ${PATH}"
+        echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
         for i in python python3 git cmake ctest; do which-realpath $i; done
         ls -la
 
     - name: Install requirements
       shell: bash
       run: |
         git config --global --add safe.directory '*'
-        apt-get install -y cmake libgtest-dev python3-pip
+        apt-get install -y cmake libgtest-dev python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig
         python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
+        python3 -m pip install pytest pycobertura
 
     - name: Configure, Build, and Test (Total Code Coverage)
       timeout-minutes: 30
@@ -273,6 +304,67 @@ jobs:
           -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
           -DPython3_EXECUTABLE=$(which python3)
 
+    - name: Save XML Code Coverage
+      id: save-coverage
+      uses: actions/cache/save@v3
+      with:
+        key: ${{ github.sha }}-codecov
+        path: |
+          .codecov/*.xml
+
+    - name: Generate Code Coverage Comment
+      if: github.event_name == 'pull_request'
+      timeout-minutes: 5
+      shell: bash
+      run: |
+        echo "PWD: ${PWD}"
+        ls -la
+
+        for i in "all" "tests" "samples"; do
+            wkhtmltoimage --enable-local-file-access --quality 85 .codecov/${i}.html .codecov/${i}.png
+        done
+        ls -la .codecov
+        which -a git
+        git --version
+
+        ./source/scripts/upload-image-to-github.py --bot --token ${{ github.token }} --files .codecov/{all,tests,samples}.png --output-dir .codecov --name pr-${{ github.event.pull_request.number }}
+
+        echo -e "\n${PWD}:"
+        ls -la .
+
+        echo -e "\n.codecov:"
+        ls -la .codecov
+
+        get-md-contents() { cat .codecov/${1}.png.md .codecov/${1}.md; }
+        cat << EOF > .codecov/report.md
+        # Code Coverage Report
+
+        ## Tests Only
+        $(get-md-contents tests)
+
+        ## Samples Only
+        $(get-md-contents samples)
+
+        ## Tests + Samples
+        $(get-md-contents all)
+
+        EOF
+
+    - name: Write Code Coverage Comment
+      if: github.event_name == 'pull_request'
+      timeout-minutes: 5
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        comment_tag: codecov-report
+        filePath: .codecov/report.md
+
+    - name: Archive Code Coverage Data
+      uses: actions/upload-artifact@v4
+      with:
+        name: code-coverage-details
+        path: |
+          ${{github.workspace}}/.codecov/*
+
     - name: Verify Test Labels
       timeout-minutes: 5
       shell: bash
@@ -356,7 +448,7 @@ jobs:
 
     container:
       image: compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-no-npi-hipclang:${{ needs.get_latest_mainline_build_number.outputs.LATEST_BUILD_NUMBER }}-${{ matrix.os }}-stg1
-      options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
+      options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
 
     needs: get_latest_mainline_build_number
 
@@ -368,7 +460,7 @@ jobs:
     - name: List Files
       shell: bash
       run: |
-        which-realpath() { echo "$1 resolves to $(realpath $(which $1))"; }
+        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
         for i in python python3 git cmake ctest; do which-realpath $i; done
         ls -la
 
@@ -385,7 +477,7 @@ jobs:
       shell: bash
       run:
         python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.sanitizer }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-${{ matrix.sanitizer }}
           --build-jobs 8
           --site ${{ matrix.runner }}
           --gpu-targets ${{ env.GPU_LIST }}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -28,7 +28,7 @@ project(
     LANGUAGES C CXX
     VERSION ${ROCPROFILER_VERSION}
     DESCRIPTION "ROCm GPU performance analysis SDK"
-    HOMEPAGE_URL "https://github.com/ROCm/rocprofiler-v2-internal")
+    HOMEPAGE_URL "https://github.com/ROCm/rocprofiler-sdk-internal")
 
 set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "core")
 

diff --git a/README.md b/README.md
@@ -1 +1 @@
-# rocprofiler-v2-internal
+# rocprofiler-sdk-internal
diff --git a/source/docs/index.md b/source/docs/index.md
@@ -1,4 +1,4 @@
-# Welcome to the [ROCprofiler](https://github.com/ROCm/rocprofiler-v2-internal) Documentation!
+# Welcome to the [ROCprofiler](https://github.com/ROCm/rocprofiler-sdk-internal) Documentation!
 
 ```eval_rst
 .. toctree::

diff --git a/source/lib/rocprofiler-sdk/counters/parser/tests/CMakeLists.txt b/source/lib/rocprofiler-sdk/counters/parser/tests/CMakeLists.txt
@@ -6,14 +6,12 @@ set(ROCPROFILER_LIB_PARSER_TEST_SOURCES "parser_test.cpp")
 
 add_executable(parser-test)
 
-target_sources(
-    parser-test PRIVATE ${ROCPROFILER_LIB_PARSER_TEST_SOURCES}
-                        $<TARGET_OBJECTS:rocprofiler::rocprofiler-object-library>)
+target_sources(parser-test PRIVATE ${ROCPROFILER_LIB_PARSER_TEST_SOURCES})
 
 target_link_libraries(
     parser-test
     PRIVATE rocprofiler::rocprofiler-common-library
-            rocprofiler::rocprofiler-object-library GTest::gtest GTest::gtest_main)
+            rocprofiler::rocprofiler-static-library GTest::gtest GTest::gtest_main)
 
 gtest_add_tests(
     TARGET parser-test

diff --git a/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt b/source/lib/rocprofiler-sdk/counters/tests/CMakeLists.txt
@@ -7,14 +7,12 @@ set(ROCPROFILER_LIB_COUNTER_TEST_SOURCES metrics_test.cpp evaluate_ast_test.cpp
 
 add_executable(counter-test)
 
-target_sources(
-    counter-test PRIVATE ${ROCPROFILER_LIB_COUNTER_TEST_SOURCES}
-                         $<TARGET_OBJECTS:rocprofiler::rocprofiler-object-library>)
+target_sources(counter-test PRIVATE ${ROCPROFILER_LIB_COUNTER_TEST_SOURCES})
 
 target_link_libraries(
     counter-test
     PRIVATE rocprofiler::rocprofiler-hip rocprofiler::rocprofiler-common-library
-            rocprofiler::rocprofiler-object-library GTest::gtest GTest::gtest_main)
+            rocprofiler::rocprofiler-static-library GTest::gtest GTest::gtest_main)
 
 gtest_add_tests(
     TARGET counter-test

diff --git a/source/scripts/run-ci.py b/source/scripts/run-ci.py
@@ -14,6 +14,7 @@
 # and default value for CTEST_SUBMIT_URL
 _PROJECT_NAME = "rocprofiler-v2-internal"
 _BASE_URL = "10.194.116.31/cdash"
+_GCOVR_GENERATE_CMD = None
 
 
 def which(cmd, require):
@@ -45,6 +46,7 @@ def generate_custom(args, cmake_args, ctest_args):
 
     GIT_CMD = which("git", require=True)
     GCOV_CMD = which("gcov", require=False)
+    GCOVR_CMD = which("gcovr", require=False)
     CMAKE_CMD = which("cmake", require=True)
     # CTEST_CMD = which("ctest", require=True)
 
@@ -91,14 +93,55 @@ def generate_custom(args, cmake_args, ctest_args):
         "external/.*",
         "samples/.*",
         "tests/.*",
+        ".*/external/.*",
+        ".*/samples/.*",
+        ".*/tests/.*",
         ".*/details/.*",
-        "*/counters/parser/.*",
+        ".*/counters/parser/.*",
     ]
     if args.coverage == "samples":
         codecov_exclude += [".*/lib/common/.*"]
 
     COVERAGE_EXCLUDE = ";".join(codecov_exclude)
 
+    if args.coverage and GCOVR_CMD:
+        global _GCOVR_GENERATE_CMD
+
+        codecov_dir = os.path.join(args.source_dir, ".codecov")
+        codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+        codecov_html = os.path.join(codecov_dir, f"{args.coverage}.html")
+
+        if not os.path.exists(codecov_dir):
+            os.makedirs(codecov_dir)
+
+        with open(os.path.join(codecov_dir, ".gitignore"), "w") as f:
+            f.write("/*\n")
+
+        gcovr_codecov_exclude = []
+        for itr in codecov_exclude:
+            gcovr_codecov_exclude += ["--exclude", f"{itr}"]
+
+        _GCOVR_GENERATE_CMD = (
+            [GCOVR_CMD]
+            + [
+                "--root",
+                f"{args.source_dir}",
+                "--exclude-unreachable-branches",
+                "--exclude-throw-branches",
+                "--gcov-ignore-parse-errors",
+                "--gcov-executable",
+                GCOV_CMD,
+                "-s",
+                "-p",
+                "--xml",
+                codecov_xml,
+                "--html-details",
+                codecov_html,
+            ]
+            + gcovr_codecov_exclude
+            + [args.source_dir]
+        )
+
     return f"""
         set(CTEST_PROJECT_NAME "{_PROJECT_NAME}")
         set(CTEST_NIGHTLY_START_TIME "05:00:00 UTC")
@@ -438,10 +481,12 @@ def run(*args, **kwargs):
         dashboard_args.append(f"{args.mode}{itr}")
 
     try:
-        ctest_args += ["--no-tests=error"]
         if not args.quiet and len(ctest_args) == 0:
             ctest_args = ["--output-on-failure", "-V"]
 
+        # always fail if no tests exist
+        ctest_args += ["--no-tests=error"]
+
         run(
             [CTEST_CMD]
             + dashboard_args
@@ -482,3 +527,29 @@ def run(*args, **kwargs):
                 with open(file, "r") as inpf:
                     fdata = inpf.read()
                     print(fdata)
+
+    if _GCOVR_GENERATE_CMD:
+        print("\n\n\n###### Generating Cobertura XML... ######")
+        print(
+            "###### GCOVR command: '{}'... ######\n".format(" ".join(_GCOVR_GENERATE_CMD))
+        )
+        with open("/dev/null", "w") as devnull:
+            run(_GCOVR_GENERATE_CMD, stderr=devnull)
+
+            codecov_dir = os.path.join(args.source_dir, ".codecov")
+            codecov_xml = os.path.join(codecov_dir, f"{args.coverage}.xml")
+            codecov_md = os.path.join(codecov_dir, f"{args.coverage}.md")
+
+            PYCOBERTURA_CMD = which("pycobertura", require=False)
+            if PYCOBERTURA_CMD:
+                run(
+                    [
+                        PYCOBERTURA_CMD,
+                        "show",
+                        "--format",
+                        "markdown",
+                        "--output",
+                        codecov_md,
+                        codecov_xml,
+                    ]
+                )
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		# rocprofiler-v2-internal
		# rocprofiler-sdk-internal