Update ci.yml to ubuntu-20.04 #1392
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI jobs | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - staging | |
| - r[0-9]+.* | |
| pull_request: | |
| branches: | |
| - master | |
| - r[0-9]+.* | |
| types: [opened, reopened, synchronize, labeled, unlabeled] | |
| env: | |
| STAGING_PROFILE_ID: 46f80d0729c92d | |
| NATIVE_BUILD_PROJECTS: tensorflow-core/tensorflow-core-generator,tensorflow-core/tensorflow-core-api | |
| GCP_CREDS: ${{ secrets.GCP_CREDS }} | |
| jobs: | |
| quick-build: | |
| if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI build') | |
| runs-on: ubuntu-20.04 | |
| container: centos:7 | |
| steps: | |
| - name: Install environment | |
| run: | | |
| yum -y update | |
| yum -y install centos-release-scl-rh epel-release | |
| yum -y install devtoolset-9 | |
| echo Downloading Maven | |
| curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz | |
| tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ | |
| ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '17' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Build project | |
| run: | | |
| source scl_source enable devtoolset-9 || true | |
| echo $JAVA_HOME | |
| mvn -version | |
| mvn clean install -Pdev,jdk17 -B -U -e -Dlint.skip=true | |
| - name: Run lint checks | |
| run: | | |
| mvn compiler:compile -Pdev,jdk17 -B -U -e | |
| check-format: | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-20.04 | |
| container: centos:7 | |
| steps: | |
| - name: Install environment | |
| run: | | |
| yum -y update | |
| yum -y install centos-release-scl-rh epel-release | |
| yum -y install devtoolset-9 | |
| echo Downloading Maven | |
| curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz | |
| tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ | |
| ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '17' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Build project | |
| run: | | |
| source scl_source enable devtoolset-9 || true | |
| echo $JAVA_HOME | |
| mvn -version | |
| mvn clean install -Pdev,jdk17 -B -U -e -Dlint.skip=true -Dmaven.test.skip=true | |
| - name: Run format checks | |
| run: | | |
| mvn spotless:check -Pdev,jdk17 -B -U -e | |
| prepare: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }} | |
| steps: | |
| - name: Create staging repository | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/r') | |
| id: staging | |
| run: | | |
| echo "Creating staging repository with profile $STAGING_PROFILE_ID" | |
| echo "<promoteRequest><data><description>Releasing TF Java - created by CI build</description></data></promoteRequest>" > request.xml | |
| curl -X POST -d @request.xml -s -o response.xml -u ${{ secrets.CI_DEPLOY_USERNAME }}:${{ secrets.CI_DEPLOY_PASSWORD }} -H "Content-Type:application/xml" \ | |
| https://oss.sonatype.org/service/local/staging/profiles/$STAGING_PROFILE_ID/start | |
| STAGING_REPOSITORY_ID=`awk -F'[<>]' '/stagedRepositoryId/{print $3}' response.xml` | |
| echo "Staging repository created: $STAGING_REPOSITORY_ID" | |
| echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID" | |
| linux-x86_64: | |
| if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') | |
| runs-on: ubuntu-20.04 | |
| container: centos:7 | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| ext: ["", -gpu] #, -mkl, -mkl-gpu] | |
| steps: | |
| - name: Install environment | |
| run: | | |
| echo Not updating glibc since CUDA fails with updated versions | |
| GLIBC="glibc glibc-common glibc-devel glibc-headers" | |
| yum --disablerepo updates -y install $GLIBC | |
| yum -x "$GLIBC" -y update | |
| yum -x "$GLIBC" -y install centos-release-scl-rh epel-release | |
| yum -x "$GLIBC" -y install devtoolset-9 rh-git218 patch perl-Data-Dumper python36-devel python36-numpy python36-pip python36-six | |
| echo Downloading Maven | |
| curl -L https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -o $HOME/apache-maven-3.6.3-bin.tar.gz | |
| tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/ | |
| ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn | |
| echo Downloading Bazel | |
| curl -L https://github.com/bazelbuild/bazel/releases/download/5.1.1/bazel-5.1.1-installer-linux-x86_64.sh -o bazel.sh --retry 10 | |
| bash bazel.sh | |
| if [[ "${{ matrix.ext }}" == *-gpu ]]; then | |
| echo Installing CUDA | |
| curl -L https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda-repo-rhel7-11-2-local-11.2.2_460.32.03-1.x86_64.rpm -o $HOME/cuda.rpm | |
| curl -L https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-11.2-linux-x64-v8.1.1.33.tgz -o $HOME/cudnn.tgz | |
| curl -L https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz -o $HOME/nccl.txz | |
| rpm -i $HOME/cuda.rpm | |
| pushd /var/cuda-repo-rhel7-11-2-local/; rpm -i --nodeps cuda*.rpm libc*.rpm libn*.rpm; rm *.rpm; popd | |
| ln -sf /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/libcuda.so | |
| ln -sf /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/local/cuda/lib64/libnvidia-ml.so | |
| tar hxvf $HOME/cudnn.tgz -C /usr/local/ | |
| tar hxvf $HOME/nccl.txz --strip-components=1 -C /usr/local/cuda/ | |
| mv /usr/local/cuda/lib/* /usr/local/cuda/lib64/ | |
| echo Removing downloaded archives and unused libraries to avoid running out of disk space | |
| rm -f $HOME/*.rpm $HOME/*.tgz $HOME/*.txz $HOME/*.tar.* | |
| rm -f $(find /usr/local/cuda/ -name '*.a' -and -not -name libcudart_static.a -and -not -name libcudadevrt.a) | |
| rm -rf /usr/local/cuda/doc* /usr/local/cuda/libnvvp* /usr/local/cuda/nsight* /usr/local/cuda/samples* | |
| fi | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Build project | |
| run: | | |
| source scl_source enable devtoolset-9 rh-git218 || true | |
| git --version | |
| gcc --version | |
| mvn -version | |
| bazel version | |
| df -h | |
| echo "Fixing HOME to /root (was '$HOME')" | |
| export HOME=/root | |
| mkdir -p $HOME/.m2 | |
| [[ "${{ github.event_name }}" == "push" ]] && MAVEN_PHASE=deploy || MAVEN_PHASE=install | |
| echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml | |
| if [[ "${{ github.event_name }}" == "push" && "${{ github.repository }}" == "tensorflow/java" ]]; then | |
| printf '%s\n' "${GCP_CREDS}" > $HOME/gcp_creds.json | |
| export BAZEL_CACHE="--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=true --google_credentials=$HOME/gcp_creds.json" | |
| else | |
| export BAZEL_CACHE="--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=false" | |
| fi | |
| echo Executing Maven $MAVEN_PHASE | |
| mvn clean $MAVEN_PHASE -B -U -e -Djavacpp.platform=linux-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -pl $NATIVE_BUILD_PROJECTS -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=$BAZEL_CACHE" | |
| df -h | |
| macosx-x86_64: | |
| if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') | |
| runs-on: macos-10.15 | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| ext: [""] # , -mkl] | |
| steps: | |
| - name: Install environment | |
| run: | | |
| python3 -m pip install numpy six | |
| echo Downloading Bazel | |
| curl -L https://github.com/bazelbuild/bazel/releases/download/5.1.1/bazel-5.1.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10 | |
| bash bazel.sh | |
| brew install libomp perl | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Build project | |
| run: | | |
| git --version | |
| clang --version | |
| mvn -version | |
| bazel version | |
| mkdir -p $HOME/.m2 | |
| [[ "${{ github.event_name }}" == "push" ]] && MAVEN_PHASE=deploy || MAVEN_PHASE=install | |
| echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml | |
| if [[ "${{ github.event_name }}" == "push" && "${{ github.repository }}" == "tensorflow/java" ]]; then | |
| printf '%s\n' "${GCP_CREDS}" > $HOME/gcp_creds.json | |
| export BAZEL_CACHE="--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=true --google_credentials=$HOME/gcp_creds.json" | |
| else | |
| export BAZEL_CACHE="--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=false" | |
| fi | |
| df -h | |
| echo Executing Maven $MAVEN_PHASE | |
| mvn clean $MAVEN_PHASE -B -U -e -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -pl $NATIVE_BUILD_PROJECTS -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=$BAZEL_CACHE" | |
| df -h | |
| windows-x86_64: | |
| if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build') | |
| runs-on: windows-2019 | |
| needs: prepare | |
| strategy: | |
| matrix: | |
| ext: ["", -gpu] #, -mkl, -mkl-gpu] | |
| steps: | |
| - name: Configure page file | |
| uses: al-cheb/configure-pagefile-action@v1.2 | |
| with: | |
| minimum-size: 8GB | |
| maximum-size: 16GB | |
| disk-root: "C:" | |
| - name: Install environment | |
| shell: cmd | |
| run: | | |
| set "PATH=C:\msys64\usr\bin;%PATH%" | |
| echo Removing broken stuff from WSL and MSYS2 | |
| rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe" | |
| python -m pip install numpy six | |
| echo Removing old versions of MSVC that interfere with Bazel | |
| bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;" | |
| echo Downloading Bazel | |
| mkdir C:\bazel | |
| curl.exe -L https://github.com/bazelbuild/bazel/releases/download/5.1.1/bazel-5.1.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10 | |
| set "EXT=${{ matrix.ext }}" | |
| if "%EXT:~-4%" == "-gpu" ( | |
| echo Removing some unused stuff to avoid running out of disk space | |
| rm.exe -Rf "C:/Program Files (x86)/Android" "C:/Program Files/dotnet" "%CONDA%" "%GOROOT_1_10_X64%" "%GOROOT_1_11_X64%" "%GOROOT_1_12_X64%" "%GOROOT_1_13_X64%" "C:\hostedtoolcache\windows\Ruby" "C:\Rust" | |
| echo Installing CUDA | |
| curl.exe -L https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda_11.2.2_461.33_win10.exe -o cuda.exe | |
| curl.exe -L https://developer.download.nvidia.com/compute/redist/cudnn/v8.1.1/cudnn-11.2-windows-x64-v8.1.1.33.zip -o cudnn.zip | |
| cuda.exe -s | |
| mkdir cuda | |
| unzip.exe cudnn.zip | |
| cp.exe -a cuda/include cuda/lib cuda/bin "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2/" | |
| ) | |
| echo %JAVA_HOME% | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Build project | |
| shell: cmd | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 | |
| set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2" | |
| set "CUDA_PATH_V11_2=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2" | |
| set "PATH=C:\msys64\usr\bin;C:\bazel;C:\Program Files\Git\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2\libnvvp;%PATH%" | |
| echo Shorten work paths to prevent Bazel from reaching MAX_PATH limit | |
| set "TEST_TMPDIR=C:\tmp" | |
| set "TMPDIR=C:\tmp" | |
| set "TEMP=C:\tmp" | |
| set "TMP=C:\tmp" | |
| mkdir C:\tmp | |
| bash --version | |
| git --version | |
| cl | |
| call mvn -version | |
| bazel version | |
| mkdir %USERPROFILE%\.m2 | |
| if "${{ github.event_name }}" == "push" (set MAVEN_PHASE=deploy) else (set MAVEN_PHASE=install) | |
| echo ^<settings^>^<servers^>^<server^>^<id^>ossrh^</id^>^<username^>${{ secrets.CI_DEPLOY_USERNAME }}^</username^>^<password^>${{ secrets.CI_DEPLOY_PASSWORD }}^</password^>^</server^>^</servers^>^</settings^> > %USERPROFILE%\.m2\settings.xml | |
| set "BAZEL_CACHE=--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=false" | |
| if "${{ github.event_name }}" == "push" ( | |
| if "${{ github.repository }}" == "tensorflow/java" ( | |
| printenv GCP_CREDS > %USERPROFILE%\gcp_creds.json | |
| set "BAZEL_CACHE=--remote_cache=https://storage.googleapis.com/tensorflow-sigs-jvm --remote_upload_local_results=true --google_credentials=%USERPROFILE%\gcp_creds.json" | |
| ) | |
| ) | |
| df -h | |
| wmic pagefile list /format:list | |
| set "SKIP_EXPORT=true" | |
| echo Executing Maven %MAVEN_PHASE% | |
| call mvn clean %MAVEN_PHASE% -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -pl %NATIVE_BUILD_PROJECTS% -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=%BAZEL_CACHE%" | |
| if ERRORLEVEL 1 exit /b | |
| df -h | |
| wmic pagefile list /format:list | |
| deploy: | |
| if: github.event_name == 'push' && contains(github.ref, 'master') | |
| needs: [linux-x86_64, macosx-x86_64, windows-x86_64] | |
| runs-on: ubuntu-18.04 | |
| steps: | |
| - name: Configure Java | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '11' | |
| - name: Checkout repository | |
| uses: actions/checkout@v1 | |
| - name: Deploy snapshot artifacts | |
| run: | | |
| echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > settings.xml | |
| bash deploy.sh |