Skip to content

Set up CI with Azure Pipelines #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Set up CI with Azure Pipelines #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf80745
Travis CI improvement
yongtang Feb 8, 2019
86816f9
PyPI.org Test
yongtang Feb 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 43 additions & 79 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,91 +1,55 @@
dist: xenial
dist: trusty
sudo: required
language: minimal

addons:
apt:
update: true

env:
- BAZEL_VERSION=0.20.0
git:
quiet: true

services:
- docker
- docker

before_script:
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh

matrix:
jobs:
include:
- name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 16.04 Python 2.7 Tests"
script:
- docker pull ubuntu:16.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 16.04 Python 3.5 Tests"
script:
- docker pull ubuntu:16.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 18.04 Python 2.7 Tests"
script:
- docker pull ubuntu:18.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 18.04 Python 3.6 Tests"
script:
- docker pull ubuntu:18.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
- name: "Ubuntu 16.04 Python 2.7 Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:16.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
- name: "Ubuntu 16.04 Python 3.5 Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:16.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
- name: "Ubuntu 18.04 Python 2.7 Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:18.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
- name: "Ubuntu 18.04 Python 3.6 Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:18.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
- name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:16.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
- name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests"
script:
- docker pull tensorflow/tensorflow:custom-op
- docker pull ubuntu:18.04
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
- stage: Build
script: echo Build Success
- stage: Test
script: echo Test Success
- stage: Nightly
script: echo PyPI.org

#matrix:
# include:
# - name: "Python 2.7 Build"
# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true TEST_UBUNTU1804=true
# - name: "Python 3.4 Build"
# env: PYTHON_VERSION=3.4 BUILD_IMAGE=tensorflow/tensorflow:custom-op
# - name: "Python 3.5 Build"
# env: PYTHON_VERSION=3.5 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true
# - name: "Python 3.6 Build"
# env: PYTHON_VERSION=3.6 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1804=true
# - name: "Python 2.7 Build R Ubuntu 16.04"
# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1604=true
# - name: "Python 2.7 Build R Ubuntu 18.04"
# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1804=true
# - name: "Python 2.7 Ubuntu 16.04 Build"
# env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
# - name: "Python 2.7 Ubuntu 18.04 Build"
# env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true
# - name: "Python 3.5 Ubuntu 16.04 Build"
# env: PYTHON_VERSION=3.5 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
# - name: "Python 3.6 Ubuntu 18.04 Build"
# env: PYTHON_VERSION=3.6 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true

#script:
#- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ${BUILD_IMAGE} bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION}
#- if [[ ! -z ${TEST_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
#- if [[ ! -z ${TEST_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
#- if [[ ! -z ${TEST_R_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi
#- if [[ ! -z ${TEST_R_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi

notifications:
email: false
11 changes: 11 additions & 0 deletions .travis/bazel.install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Install bazel
set -e -x

apt-get -y -qq update
apt-get -y -qq install unzip curl > /dev/null
BAZEL_OS=${1}
BAZEL_VERSION=${2}
curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
54 changes: 0 additions & 54 deletions .travis/python.configure.sh
View file
Open in desktop

This file was deleted.

100 changes: 100 additions & 0 deletions .travis/python.install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#/bin/bash
set -x -e

apt-get -y -qq update
apt-get -y -qq install lsb-core > /dev/null
if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then
apt-get -y -qq install libav-tools > /dev/null
if [[ ${1} == "2.7" ]]; then
echo "Python 2.7"
elif [[ ${1} == "3.4" ]]; then
rm -f /usr/local/bin/pip
ln -s /usr/local/bin/pip3.4 /usr/local/bin/pip
rm -f /usr/bin/python
ln -s /usr/bin/python3.4 /usr/bin/python
elif [[ ${1} == "3.5" ]]; then
curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh
sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh
sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh
bash install_python3.5_pip_packages.sh
rm -f install_python3.5_pip_packages.sh
rm -f /usr/bin/python
ln -s /usr/bin/python3.5 /usr/bin/python
rm -f /usr/local/bin/pip
ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip
elif [[ ${1} == "3.6" ]]; then
rm -f /usr/local/bin/pip3
curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh
sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh
sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh
sed -i 's/pip3 install/pip3 -q install/g' install_python3.6_pip_packages.sh
sed -i 's/tar xvf/tar xf/g' install_python3.6_pip_packages.sh
sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh
sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh
sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh
bash install_python3.6_pip_packages.sh
rm -f install_python3.6_pip_packages.sh make.log
rm -rf Python-3.6.1*
rm -f /usr/bin/python
ln -s /usr/local/bin/python3.6 /usr/bin/python
rm -f /usr/local/bin/pip
ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip
else
echo Python ${1} not supported!
exit 1
fi
elif [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then
if [[ ${1} == "2.7" ]]; then
echo "Python 2.7"
apt-get -y -qq install ffmpeg python-pip python3-pip patchelf > /dev/null
pip3 install -q auditwheel==1.5.0
# Pin wheel==0.31.1 to work around issue
# https://github.com/pypa/auditwheel/issues/102
pip3 install -q wheel==0.31.1
elif [[ ${1} == "3.5" ]]; then
echo "Python 3.5"
apt-get -y -qq install ffmpeg python3-pip patchelf > /dev/null
pip3 install -q auditwheel==1.5.0
# Pin wheel==0.31.1 to work around issue
# https://github.com/pypa/auditwheel/issues/102
pip3 install wheel==0.31.1
rm -f /usr/bin/python
ln -s /usr/bin/python3 /usr/bin/python
rm -f /usr/bin/pip
ln -s /usr/bin/pip3 /usr/bin/pip
else
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
exit 1
fi
elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then
if [[ ${1} == "2.7" ]]; then
echo "Python 2.7"
apt-get -y -qq install ffmpeg python-pip python3-pip python3-wheel patchelf > /dev/null
pip3 install -q auditwheel==1.5.0
# Pin wheel==0.31.1 to work around issue
# https://github.com/pypa/auditwheel/issues/102
pip3 install wheel==0.31.1
elif [[ ${1} == "3.6" ]]; then
echo "Python 3.6"
apt-get -y -qq install ffmpeg python3-pip python3-wheel patchelf > /dev/null
pip3 install -q auditwheel==1.5.0
# Pin wheel==0.31.1 to work around issue
# https://github.com/pypa/auditwheel/issues/102
pip3 install wheel==0.31.1
rm -f /usr/bin/python
ln -s /usr/bin/python3 /usr/bin/python
rm -f /usr/bin/pip
ln -s /usr/bin/pip3 /usr/bin/pip
else
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
exit 1
fi
else
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
exit 1
fi
python --version
pip --version
pip freeze
Loading