Skip to content

Bump lifting bits llvm for x86 (#1050) #328

Bump lifting bits llvm for x86 (#1050)

Bump lifting bits llvm for x86 (#1050) #328

name: Linux CI
env:
# "Source" is set in the vcpkg install step
VCPKG_BINARY_SOURCES: 'clear;nuget,Source,readwrite;nugettimeout,3601'
# Self-hosted runners can hopefully reuse already downloaded packages
VCPKG_USE_NUGET_CACHE: 1
VCPKG_DEFAULT_HOST_TRIPLET: 'x64-linux-rel'
on:
release:
types:
- published
pull_request:
paths-ignore:
- 'docker/**'
- '.github/**'
- '**.md'
- '!.github/workflows/vcpkg_ci_amd64*'
push:
paths-ignore:
- 'docker/**'
- '.github/**'
- '**.md'
- '!.github/workflows/vcpkg_ci_amd64*'
tags-ignore:
- 'v*'
branches:
- 'master'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
strategy:
fail-fast: false
matrix:
image:
# 'name' is Docker image name whereas 'os' is more generic
- { os: 'ubuntu', name: 'ubuntu-v2', tag: '22.04' }
llvm: [
'llvm-16',
'llvm-17',
'llvm-17[liftingbits-llvm]'
]
target_arch: [ 'x64', 'arm64' ]
container:
image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}
volumes:
- /:/gha-runner
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
name: ${{matrix.image.os}}-${{matrix.image.tag}} ${{matrix.llvm}} ${{matrix.target_arch}}
runs-on: ubuntu-latest # run the job on the newly created runner
steps:
- name: Free Disk Space
run: |
df -h
rm -rf /gha-runner/usr/local/lib/android
rm -rf /gha-runner/usr/local/share/boost
df -h
- name: Set Artifact Name
run: |
# Need to fix because paths with brackets cause issues
NAME="$(echo 'vcpkg_${{ matrix.image.os }}-${{ matrix.image.tag }}_${{ matrix.llvm }}_${{ matrix.target_arch == 'x64' && 'amd64' || matrix.target_arch }}' | tr '[' '-' | tr -d ']')"
echo "ARTIFACT_NAME=${NAME}" >> "$GITHUB_ENV"
- uses: actions/checkout@v4
with:
submodules: recursive
fetch-depth: 2
# Used to get commit message since PRs are on a merge commit
- name: Get commit message
shell: bash
run: |
git config --global --add safe.directory /__w/cxx-common/cxx-common
echo 'COMMIT_MESSAGE<<EOF' >> "$GITHUB_ENV"
if [[ '${{ github.event_name }}' == 'push' ]]; then
echo "$(git log --format=%B -n 1 HEAD)" >> "$GITHUB_ENV"
elif [[ '${{ github.event_name }}' == 'pull_request' ]]; then
echo "$(git log --format=%B -n 1 HEAD^2)" >> "$GITHUB_ENV"
fi
echo "EOF" >> "$GITHUB_ENV"
- name: Clear prior vcpkg directory
run: |
rm -rf "${{ github.workspace }}/vcpkg"
- name: Initialize vcpkg
shell: bash
run: |
{ read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1
git clone "${vcpkg_repo_url}"
git -C vcpkg checkout "${vcpkg_commit}"
export VCPKG_DISABLE_METRICS=1
./vcpkg/bootstrap-vcpkg.sh
echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV
- name: 'vcpkg install dependencies'
shell: 'bash'
run: |
export VCPKG_DISABLE_METRICS=1
# Setup NuGet authentication
mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" sources add \
-source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \
-storepasswordincleartext \
-name "Source" \
-username "${{ github.repository_owner }}" \
-password "${{ secrets.GITHUB_TOKEN }}" || true
mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" sources update \
-source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \
-storepasswordincleartext \
-name "Source" \
-username "${{ github.repository_owner }}" \
-password "${{ secrets.GITHUB_TOKEN }}"
mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" setapikey \
-source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \
"${{ secrets.GITHUB_TOKEN }}"
./build_dependencies.sh --release --target-arch ${{ matrix.target_arch }} --export-dir ./${{ env.ARTIFACT_NAME }} --clean-after-build --debug
rm -rf vcpkg/downloads
rm -rf ~/.nuget/packages
echo "Check space..."
df -h
echo "DONE."
./build_dependencies.sh --release --target-arch ${{ matrix.target_arch }} --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug
echo "VCPKG_ROOT=$(pwd)/${{ env.ARTIFACT_NAME }}" >> $GITHUB_ENV
echo "TARGET_TRIPLET=${{ matrix.target_arch }}-linux-rel" >> $GITHUB_ENV
- name: Check space
if: failure()
run: |
echo "## Space left"
df -h
echo ""
echo "## Storage in working directory"
du -hs *
echo ""
echo "## Storage in vcpkg"
du -hs vcpkg/*
echo ""
echo "## Storage in nuget"
du -hs ~/.nuget/*
- name: Upload CMake logs on error
if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ env.ARTIFACT_NAME }}_logs
path: ${{ github.workspace }}/vcpkg/buildtrees/**/*.log
- name: Cleanup
shell: 'bash'
run: |
echo "Space left"
df -h
echo "Cleaning up..."
rm -rf vcpkg || true
rm -rf ~/.nuget || true
echo "Space left"
df -h
- name: 'Export Packages'
if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
shell: 'bash'
run: |
apt-get update
apt-get install -y pixz
tar --use-compress-program pixz -cf "${{ env.ARTIFACT_NAME }}.tar.xz" ./${{ env.ARTIFACT_NAME }}
- name: Publish Release Assets
if: github.event.release
uses: softprops/action-gh-release@v1
with:
files: ${{ env.ARTIFACT_NAME }}.tar.xz
prerelease: ${{ contains(github.ref, 'pre') || contains(github.ref, 'rc') }}
token: ${{ secrets.RELEASE_TOKEN }}
- uses: actions/upload-artifact@v4
if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
with:
name: ${{ env.ARTIFACT_NAME }}.tar.xz
path: ${{ env.ARTIFACT_NAME }}.tar.xz
- name: Prepare ccache
id: ccache_prep
shell: bash
run: |
echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV
echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
echo "CMAKE_C_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV
echo "CMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV
echo "timestamp=$(date +"%Y-%m-%d-%H:%M:%S" --utc)" >> ${GITHUB_OUTPUT}
- name: ccache cache files
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.ccache
key: ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}-${{ steps.ccache_prep.outputs.timestamp }}
restore-keys: |
ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}-
- name: ccache Initial stats
shell: bash
run: |
ccache --show-stats
- name: 'Rellic build'
shell: 'bash'
working-directory: rellic
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
run: |
# Does not compile with gcc
export CC="$(which clang)"
export CXX="$(which clang++)"
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
"-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
-DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
"-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
-S . -B build
cmake --build build
cmake --build build --target install
- name: 'Rellic test'
shell: 'bash'
working-directory: rellic
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
run: |
cmake --build build --target test
- name: 'Remill dependencies'
shell: 'bash'
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
working-directory: remill
run: |
python3 -m pip install poetry
python3 -m pip install --user ./scripts/diff_tester_export_insns
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: 'Remill build'
shell: 'bash'
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
working-directory: remill
run: |
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
"-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
-DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
"-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
-S . -B build
cmake --build build
cmake --install build
cmake --build build --target test_dependencies
- name: 'Remill test'
shell: 'bash'
working-directory: remill/build
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
run: |
# Some tests fail on ubuntu 22.04
env CTEST_OUTPUT_ON_FAILURE=1 ctest . || true
- name: 'Anvill build'
shell: 'bash'
working-directory: anvill
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
run: |
# TODO: Does not support compilation with gcc
export CC="$(which clang)"
export CXX="$(which clang++)"
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
"-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
"-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
-DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
-DANVILL_ENABLE_TESTS=true \
-DANVILL_ENABLE_INSTALL=true \
-DANVILL_ENABLE_PYTHON3_LIBS=OFF \
"-Dremill_DIR=$(pwd)/../remill/install/lib/cmake/remill" \
"-Dsleigh_DIR=$(pwd)/../remill/install/lib/cmake/sleigh" \
-S . -B build
cmake --build build
cmake --install build
- name: 'Anvill test'
shell: 'bash'
working-directory: anvill
if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16')
run: |
cmake --build build --target test
- name: Cache cleanup and reporting
shell: 'bash'
run: |
rm -rf vcpkg/{buildtrees,installed,packages}
ccache --show-stats
df -h