This repository has been archived by the owner on Nov 1, 2024. It is now read-only.
TorchArrow Nightly Build and Test for Conda #1048
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: TorchArrow Nightly Build and Test for Conda | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| # Use the following to trigger test on a dev PR | |
| # pull_request: | |
| # branches: | |
| # - main | |
| # # For PR created by ghstack | |
| # - gh/*/*/base | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| linux-container: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - 3.7 | |
| - 3.8 | |
| - 3.9 | |
| steps: | |
| - name: Print CPU info | |
| run: | | |
| cat /proc/cpuinfo | |
| - name: Create Conda Env | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| activate-environment: conda_build_env | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| # Based on https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| run: | | |
| echo "::set-output name=timestamp::$(/bin/date -u "+%Y%m%d-%H:%M:%S")" | |
| shell: bash | |
| - name: Load ccache files | |
| uses: actions/cache@v2 | |
| with: | |
| path: .ccache | |
| key: ubuntu-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} | |
| restore-keys: | | |
| ubuntu-ccache- | |
| - name: Install dependencies with APT | |
| run: | | |
| sudo apt-get update | |
| sudo apt install -y g++ cmake ccache ninja-build checkinstall git \ | |
| libssl-dev libboost-all-dev libdouble-conversion-dev libgoogle-glog-dev \ | |
| libgflags-dev libevent-dev libre2-dev libfl-dev libbison-dev | |
| # Based on https://github.com/facebookincubator/velox/blob/99429407c3d524e07b32b8b19a03aa7382f819cf/.circleci/config.yml#L114-L116 | |
| - name: Configure ccache | |
| run: | | |
| echo "$GITHUB_WORKSPACE" | |
| CCACHE_DIR=$GITHUB_WORKSPACE/.ccache_root ccache -sz -M 1G | |
| CCACHE_DIR=$GITHUB_WORKSPACE/.ccache ccache -sz -M 1G | |
| - name: Build and install folly and fmt | |
| env: | |
| CMAKE_C_COMPILER_LAUNCHER: ccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: ccache | |
| CCACHE_DIR: $GITHUB_WORKSPACE/.ccache_root | |
| CPU_TARGET: sse | |
| # sudo doesn't preserve environment vairable; set it after sudo: https://stackoverflow.com/questions/8633461/how-to-keep-environment-variables-when-using-sudo/33183620#33183620 | |
| run: | | |
| sudo scripts/setup-ubuntu.sh | |
| - name: Conda Build | |
| shell: bash -l {0} | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| CPU_TARGET: sse | |
| run: | | |
| # source /opt/conda/etc/profile.d/conda.sh | |
| conda activate conda_build_env | |
| conda install -yq conda-build -c conda-forge | |
| ./packaging/build_conda.sh | |
| - name: Conda Upload | |
| shell: bash -l {0} | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} | |
| CONDA_CHANNEL: pytorch-nightly | |
| run: | | |
| conda install -yq -c anaconda anaconda-client | |
| anaconda -t "${CONDA_NIGHTLY_PYTORCHBOT_TOKEN}" upload ./conda-bld/linux-64/torcharrow-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force | |
| macos-container: | |
| runs-on: macos-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - 3.7 | |
| - 3.8 | |
| - 3.9 | |
| steps: | |
| - name: Create Conda Env | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| activate-environment: conda_build_env | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Install dependencies from Brew | |
| run: | | |
| brew install --formula ninja flex bison cmake ccache icu4c boost gflags glog libevent | |
| - name: Conda Build | |
| shell: bash -l {0} | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| MACOSX_DEPLOYMENT_TARGET: 10.15 | |
| CPU_TARGET: sse | |
| run: | | |
| conda activate conda_build_env | |
| conda install -yq conda-build -c conda-forge | |
| scripts/build_mac_dep.sh ranges_v3 fmt double_conversion folly re2 | |
| ./packaging/build_conda.sh | |
| - name: Fix Velox Dylib Paths | |
| shell: bash -l {0} | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| run: | | |
| cd conda-bld/osx-64 | |
| pkg_name=`ls ./torcharrow-* | sed -n -e 's/.*\(torcharrow.*\).tar.bz2/\1/p'` | |
| mkdir ./${pkg_name} | |
| tar -xf ./${pkg_name}.tar.bz2 -C ./${pkg_name} | |
| rm ./${pkg_name}.tar.bz2 | |
| cd ./${pkg_name} | |
| source ${GITHUB_WORKSPACE}/packaging/fix_conda_dylib_paths.sh | |
| conda_lib_folder=lib/python${PYTHON_VERSION}/site-packages/torcharrow | |
| so_name=`ls ${conda_lib_folder}/_torcharrow.* | sed -n -e 's/.*\(_torcharrow.*\.so\)/\1/p'` | |
| fix_velox_dylib_paths ${conda_lib_folder}/${so_name} | |
| otool -L ${conda_lib_folder}/${so_name} | |
| tar -cjf ../${pkg_name}.tar.bz2 * | |
| cd .. | |
| rm -rf ./${pkg_name} | |
| cd ../.. | |
| - name: Conda Upload | |
| shell: bash -l {0} | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} | |
| run: | | |
| conda install -yq anaconda-client | |
| anaconda -t "${CONDA_NIGHTLY_PYTORCHBOT_TOKEN}" upload ./conda-bld/osx-64/torcharrow-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force |