diff --git a/.buildkite/build_batfish.sh b/.buildkite/build_batfish.sh deleted file mode 100755 index 5d63061..0000000 --- a/.buildkite/build_batfish.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -source "$(dirname "${BASH_SOURCE[0]}")"/common_vars.sh -mkdir $ARTIFACT_DIR - -# Build Batfish -BF_DIR=$(mktemp -d) -git clone --depth=1 --branch=${BATFISH_GITHUB_BATFISH_REF} ${BATFISH_GITHUB_BATFISH_REPO} ${BF_DIR} -pushd ${BF_DIR} - BATFISH_TAG=$(git rev-parse --short HEAD) -popd -# Update version and build jar -mvn -f ${BF_DIR}/projects versions:set -DnewVersion=${BATFISH_VERSION_STRING} -mvn -f ${BF_DIR}/projects package - -# Copy artifacts -# 1. The jar -cp ${BF_DIR}/projects/allinone/target/allinone-bundle*.jar ${ARTIFACT_DIR}/allinone-bundle.jar - -# 2. Questions from Batfish -TMP_DIR=$(mktemp -d) -QUESTION_DIR=${TMP_DIR}/questions -mkdir -p ${QUESTION_DIR} -cp -r ${BF_DIR}/questions/{stable,experimental} ${QUESTION_DIR} -tar -czf ${ARTIFACT_DIR}/questions.tgz -C ${TMP_DIR} questions - -# 3. The tag for the Batfish image -echo ${BATFISH_TAG} > ${ARTIFACT_DIR}/batfish-tag.txt diff --git a/.buildkite/build_pybatfish.sh b/.buildkite/build_pybatfish.sh deleted file mode 100755 index 91a2564..0000000 --- a/.buildkite/build_pybatfish.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -source "$(dirname "${BASH_SOURCE[0]}")"/common_vars.sh -mkdir $ARTIFACT_DIR - -# Build Pybatfish -PYBF_DIR=$(mktemp -d) -git clone --depth=1 --branch=${BATFISH_GITHUB_PYBATFISH_REF} ${BATFISH_GITHUB_PYBATFISH_REPO} ${PYBF_DIR} -pushd ${PYBF_DIR} - PYBF_TAG=$(git rev-parse --short HEAD) - # Update version if specified - if [ "${BATFISH_VERSION_STRING:-}" != "" ]; then - # Sane in-place version replace: https://stackoverflow.com/a/22084103 - sed -i.bak -e "s/^__version__ = .*$/__version__ = \"${BATFISH_VERSION_STRING}\"/" pybatfish/__init__.py - rm -f pybatfish/__init__.py.bak - fi - python3 -m virtualenv .venv - source .venv/bin/activate - pip install 'setuptools==45.2.0' - python setup.py bdist_wheel - PYBF_VERSION=$(python setup.py --version) -popd - -# Copy artifacts -cp ${PYBF_DIR}/dist/pybatfish*.whl ${ARTIFACT_DIR} -tar -czf ${ARTIFACT_DIR}/pybatfish-tests.tgz -C ${PYBF_DIR} tests -tar -czf ${ARTIFACT_DIR}/pybatfish-notebooks.tgz -C ${PYBF_DIR} jupyter_notebooks -echo "${PYBF_TAG}" > ${ARTIFACT_DIR}/pybatfish-tag.txt -echo "${PYBF_VERSION}" > ${ARTIFACT_DIR}/pybatfish-version.txt diff --git a/.buildkite/common_vars.sh b/.buildkite/common_vars.sh deleted file mode 100755 index 258ffe7..0000000 --- a/.buildkite/common_vars.sh +++ /dev/null @@ -1,2 +0,0 @@ -ARTIFACT_DIR="artifacts" -TESTING_TAG="test" diff --git a/.buildkite/docker_build_allinone.sh b/.buildkite/docker_build_allinone.sh deleted file mode 100755 index 98373bc..0000000 --- a/.buildkite/docker_build_allinone.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Build Allinone Docker image and either push to Docker Hub or save an artifact -# Accepts optional arguments: -# 1. Batfish docker artifact filename: if specified, Batfish image is loaded -# from this artifact instead of pulling from Docker Hub -# 2. Allinone docker artifact filename: if specified, Allinone image is saved -# to this artifact instead of pushing to Docker Hub - -set -euxo pipefail - -BUILDKITE_DIR="$(dirname "${BASH_SOURCE[0]}")" -ABS_SOURCE_DIR="$(realpath "${BUILDKITE_DIR}/..")" -source ${BUILDKITE_DIR}/common_vars.sh -ASSET_DIR=./assets - -mkdir ${ARTIFACT_DIR} -mkdir ${ASSET_DIR} - -buildkite-agent artifact download ${ARTIFACT_DIR}/batfish-tag.txt ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish*.whl ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-notebooks.tgz ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-tag.txt ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-version.txt ${ARTIFACT_DIR} -if [ "${1-}" != "" ]; then - # Download and load the image artifact if an image file path is specified - buildkite-agent artifact download ${ARTIFACT_DIR}/$1 ${ARTIFACT_DIR} - docker load -i ${ARTIFACT_DIR}/$1 -fi - -PYBF_VERSION=$(cat ${ARTIFACT_DIR}/pybatfish-version.txt) -PYBF_TAG=$(cat ${ARTIFACT_DIR}/pybatfish-tag.txt) -BF_TAG=$(cat ${ARTIFACT_DIR}/batfish-tag.txt) -SHA_TAG="${BF_TAG}_${PYBF_TAG}" - -# Setup assets for the Batfish image -cp ${ARTIFACT_DIR}/pybatfish*.whl ${ASSET_DIR} -TEMP_DIR=$(mktemp -d) -tar xzf ${ARTIFACT_DIR}/pybatfish-notebooks.tgz -C ${TEMP_DIR} -cp -r ${TEMP_DIR}/jupyter_notebooks/ ${ASSET_DIR}/notebooks/ -# Script that starts Batfish + Jupyter server -cp ${ABS_SOURCE_DIR}/wrapper.sh ${ASSET_DIR} -cp ${ABS_SOURCE_DIR}/log4j2.yaml ${ASSET_DIR} - -docker build -f ${ABS_SOURCE_DIR}/allinone.dockerfile \ - -t batfish/allinone:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} \ - --label "org.batfish.allinone-tag=${SHA_TAG}" \ - --build-arg PYBATFISH_VERSION=${PYBF_VERSION} \ - --build-arg ASSETS=${ASSET_DIR} \ - --build-arg TAG=${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} . - -if [ "${2-}" == "" ]; then - # Upload the image to Docker Hub if no image file path is specified - docker push batfish/allinone:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} -else - # Upload the image as an artifact tar if an image file path is specified - docker save -o ${ARTIFACT_DIR}/$2 batfish/allinone:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} -fi diff --git a/.buildkite/docker_build_batfish.sh b/.buildkite/docker_build_batfish.sh deleted file mode 100755 index a2629e8..0000000 --- a/.buildkite/docker_build_batfish.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Build Batfish Docker image and either push to Docker Hub or save an artifact -# Accepts optional argument: -# 1. Batfish docker artifact filename: if specified, Batfish image is saved -# to this artifact instead of pushing to Docker Hub - -set -euxo pipefail - -BUILDKITE_DIR="$(dirname "${BASH_SOURCE[0]}")" -ABS_SOURCE_DIR="$(realpath "${BUILDKITE_DIR}/..")" -source ${BUILDKITE_DIR}/common_vars.sh -ASSET_DIR=./assets - -mkdir ${ARTIFACT_DIR} -mkdir ${ASSET_DIR} - -buildkite-agent artifact download ${ARTIFACT_DIR}/allinone-bundle.jar ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/batfish-tag.txt ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/questions.tgz ${ARTIFACT_DIR} - -BF_TAG=$(cat ${ARTIFACT_DIR}/batfish-tag.txt) - -# Setup assets for the Batfish image -tar xzf ${ARTIFACT_DIR}/questions.tgz -C ${ASSET_DIR} -cp ${ARTIFACT_DIR}/allinone-bundle.jar ${ASSET_DIR} -cp ${ABS_SOURCE_DIR}/log4j2.yaml ${ASSET_DIR} - -docker build -f ${ABS_SOURCE_DIR}/batfish.dockerfile \ - -t batfish/batfish:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} \ - --label "org.batfish.batfish-tag=${BF_TAG}" \ - --label "org.batfish.batfish-version=${BATFISH_VERSION_STRING-}" \ - --build-arg ASSETS=${ASSET_DIR} . - -if [ "${1-}" == "" ]; then - # Upload the image to Docker Hub if no image file path is specified - docker push batfish/batfish:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} -else - # Upload the image as an artifact tar if an image file path is specified - docker save -o ${ARTIFACT_DIR}/$1 batfish/batfish:${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER} -fi diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh deleted file mode 100755 index 5995db5..0000000 --- a/.buildkite/pipeline.sh +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -BUILDKITE_DIR="$(dirname "${BASH_SOURCE[0]}")" -source "${BUILDKITE_DIR}/common_vars.sh" - -BATFISH_UPLOAD_PIPELINE="f283deec-7e26-4f46-b8c6-b95b0cc1d974" - -BATFISH_DOCKER_CI_BASE_IMAGE="${BATFISH_DOCKER_CI_BASE_IMAGE:-batfish/ci-base:5d92c2f-14e42dd2de}" -BATFISH_GITHUB_BATFISH_REF="${BATFISH_GITHUB_BATFISH_REF:-master}" -BATFISH_GITHUB_BATFISH_REPO="${BATFISH_GITHUB_BATFISH_REPO:-https://github.com/batfish/batfish}" -BATFISH_GITHUB_PYBATFISH_REF="${BATFISH_GITHUB_PYBATFISH_REF:-master}" -if [ "${BUILDKITE_PIPELINE_ID}" == "${BATFISH_UPLOAD_PIPELINE}" ]; then - BATFISH_GITHUB_PYBATFISH_REPO="${BATFISH_GITHUB_PYBATFISH_REPO:-git@github.com:batfish/pybatfish.git}" -else - # Use SSL for precommit checks - BATFISH_GITHUB_PYBATFISH_REPO="${BATFISH_GITHUB_PYBATFISH_REPO:-https://github.com/batfish/pybatfish}" -fi -DOCKER_LOGIN_PLUGIN_VERSION="${DOCKER_LOGIN_PLUGIN_VERSION:-v2.0.1}" -DOCKER_LOGIN_PLUGIN_USERNAME="${DOCKER_LOGIN_PLUGIN_USERNAME:-batfishbuildkitebot}" - -ARTIFACTS_PLUGIN_VERSION="${ARTIFACTS_PLUGIN_VERSION:-v1.2.0}" -DOCKER_PLUGIN_VERSION="${DOCKER_PLUGIN_VERSION:-v3.3.0}" - -BATFISH_VERSION_STRING="${BATFISH_VERSION_STRING:-$(date +'%Y.%m.%d').${BUILDKITE_BUILD_NUMBER}}" - -# Test all Bf containers and Pybf wheels released in the last N days -BATFISH_MAX_TEST_ARTIFACT_AGE="${BATFISH_MAX_TEST_ARTIFACT_AGE:-90}" -# Minimum number of releases to test -BATFISH_MIN_RELEASE_TEST_COUNT=3 -PYBATFISH_MIN_RELEASE_TEST_COUNT=3 - -# Attributes common to all command steps -COMMON_STEP_ATTRIBUTES= -# Specify queue for upload builds -if [ "${BUILDKITE_PIPELINE_ID}" == "${BATFISH_UPLOAD_PIPELINE}" ]; then -COMMON_STEP_ATTRIBUTES=$(cat < :batfish: ${bf_tag}" - depends_on: pybf - if: pipeline.id == "${BATFISH_UPLOAD_PIPELINE}" - command: - - ".buildkite/test_batfish_container.sh" - env: - BATFISH_CONTAINER_TAG: ${bf_tag} - # Determines which Pybatfish integration tests are run - bf_version: ${bf_tag} - # Skip notebook ref tests - PYBATFISH_PYTEST_ARGS: '-k "not test_notebook_output"' -${COMMON_STEP_ATTRIBUTES} -EOF -fi -done <<< "${DATE_TAGS}" - -cat < :batfish: prod" - depends_on: pybf - if: pipeline.id == "${BATFISH_UPLOAD_PIPELINE}" - command: - - ".buildkite/test_batfish_container.sh" - env: - BATFISH_CONTAINER_TAG: latest - # Skip notebook ref tests - PYBATFISH_PYTEST_ARGS: '-k "not test_notebook_output"' -${COMMON_STEP_ATTRIBUTES} - - label: ":snake: dev <-> :batfish: dev" - depends_on: - - bf-upload - - pybf - if: pipeline.id == "${BATFISH_UPLOAD_PIPELINE}" - command: - - ".buildkite/test_batfish_container.sh" -${COMMON_STEP_ATTRIBUTES} -EOF - -# Get available Pybatfish versions from PyPI -python -m pip install --user 'requests==2.23.0' >/dev/null -PYBF_TAGS=$(python -c " -from datetime import datetime -import requests -releases = requests.get('https://pypi.python.org/pypi/pybatfish/json').json()['releases'].keys() -print('\n'.join(sorted(releases, key=lambda x: datetime.strptime(x, '%Y.%m.%d.%f'), reverse=True))) -") -# Keep track of how many versions we're testing -count=0 -while read pybf_tag; do -# Convert tag from YYYY.M.D to YYYY-M-D and just drop tags that do not start with four digits (need || true; to avoid erroring when regex doesn't match) -PARSED_TAG=$(echo ${pybf_tag} | { grep -o '[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1,2\}' || true; } | sed 's/\./-/g') -# Only consider tags that look like dates -if [[ "${PARSED_TAG}" != "" ]]; then -# Convert YYYY-M-D format into (comparable Unix time) timestamp -TAG_TIMESTAMP=$(date -d "${PARSED_TAG}" +"%s") -if (( count < PYBATFISH_MIN_RELEASE_TEST_COUNT )) || [[ ${MIN_TIMESTAMP} -le ${TAG_TIMESTAMP} ]]; then -count=$((count + 1)) -cat < :batfish: dev" - depends_on: bf-upload - if: pipeline.id == "${BATFISH_UPLOAD_PIPELINE}" - command: - - ".buildkite/test_batfish_container.sh" - env: - # Skip notebook ref tests - PYBATFISH_PYTEST_ARGS: '-k "not test_notebook_output"' - # Install specific version of Pybatfish from PyPI - PYBATFISH_VERSION: "pybatfish[dev]==${pybf_tag}" -${COMMON_STEP_ATTRIBUTES} -EOF -fi -fi -done <<< "${PYBF_TAGS}" - -cat < :batfish: dev" - depends_on: bf-upload - if: pipeline.id == "${BATFISH_UPLOAD_PIPELINE}" - command: - - ".buildkite/test_batfish_container.sh" - env: - # Skip notebook ref tests - PYBATFISH_PYTEST_ARGS: '-k "not test_notebook_output"' - # Install specific version of Pybatfish from PyPI - PYBATFISH_VERSION: "pybatfish[dev]" -${COMMON_STEP_ATTRIBUTES} -EOF - -cat <- - Perform manual testing. Instructions at - https://docs.google.com/document/d/15XWSdyHApnVbmZCg3FKpu6ree2HGDysmgYNFhbqTj1Q/ - Your build number is: ${BUILDKITE_BUILD_NUMBER} - Version string is: ${BATFISH_VERSION_STRING} - fields: - - select: "Artifact tag" - key: "release-tag" - default: "test" - options: - - label: "Public release" - value: "latest" - - label: "Test use only" - value: "test" - - select: "Artifact(s) to release" - key: "artifacts-to-release" - multiple: true - options: - - label: "Pybf" - value: "pybf" - - label: "Bf containers" - value: "bf" -EOF - -cat </dev/null - then - docker stop ${BATFISH_CONTAINER} - echo stopped Batfish container - fi - fi -} - -# Cleanup on exit -trap finish EXIT - -BUILDKITE_DIR="$(dirname "${BASH_SOURCE[0]}")" -ABS_SOURCE_DIR="$(realpath "${BUILDKITE_DIR}/..")" -source $BUILDKITE_DIR/common_vars.sh -mkdir $ARTIFACT_DIR - -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish*.whl ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-notebooks.tgz ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-tag.txt ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-tests.tgz ${ARTIFACT_DIR} -buildkite-agent artifact download ${ARTIFACT_DIR}/pybatfish-version.txt ${ARTIFACT_DIR} -if [ "${1-}" != "" ]; then - # Download and load the image artifact if an image file path is specified - buildkite-agent artifact download ${ARTIFACT_DIR}/$1 ${ARTIFACT_DIR} - docker load -i ${ARTIFACT_DIR}/$1 -fi - -# Use provided container tag, if applicable -BATFISH_CONTAINER_TAG="${BATFISH_CONTAINER_TAG:-${TESTING_TAG}-${BUILDKITE_BUILD_NUMBER}}" - -# Use host network so Batfish is accessible at localhost from inside test container -BATFISH_CONTAINER=$(docker run --memory=2560m -d --net=host batfish/batfish:${BATFISH_CONTAINER_TAG}) - -if [ "${bf_version-}" == "" ]; then - # Pull batfish version from container label if none is supplied via env var - BATFISH_VERSION=$(docker inspect -f '{{ index .Config.Labels "org.batfish.batfish-version" }}' ${BATFISH_CONTAINER}) -else - BATFISH_VERSION=${bf_version} -fi -echo "Using Batfish version: ${BATFISH_VERSION}" -# Run Pybatfish integration tests against Batfish container -docker run --net=host -v $(pwd)/${ARTIFACT_DIR}:/assets/ \ - -v $ABS_SOURCE_DIR/tests/test_batfish_container.sh:/test.sh \ - --env PYBATFISH_PYTEST_ARGS="${PYBATFISH_PYTEST_ARGS:-}" \ - --env bf_version="${BATFISH_VERSION}" \ - --env PYBATFISH_VERSION="${PYBATFISH_VERSION:-}" \ - --memory=768m \ - --entrypoint /bin/bash batfish/ci-base:latest /test.sh - -docker stop ${BATFISH_CONTAINER} diff --git a/build-infra/Dockerfile b/build-infra/Dockerfile deleted file mode 100644 index 67db2be..0000000 --- a/build-infra/Dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -# Setting the parent image to ubuntu -FROM ubuntu:20.04 -LABEL vendor="The Batfish Open Source Project" - -ARG GOOGLE_JAVA_FORMAT_VERSION=1.15.0 -ARG JACOCO_VERSION=0.8.2 -ARG MAVEN_VERSION=3.8.6 -ARG BAZELISK_VERSION=1.11.0 -ARG USERNAME=batfish -ARG UID=2000 -ARG TRIVY_VERSION=0.28.1 - -# Set a Working Dir -User root -WORKDIR /root/workdir - -# Installing Java, python -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ - apt-transport-https \ - binutils \ - ca-certificates \ - curl \ - gcc \ - git \ - jq \ - net-tools \ - openjdk-11-jdk \ - python3-distutils \ - rsync \ - software-properties-common \ - unzip \ - zip \ -&& apt-get upgrade -y \ -&& curl -L https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb -o trivy.deb \ -&& dpkg -i trivy.deb \ -&& rm trivy.deb \ -&& rm -rf /var/lib/apt/lists/* \ -&& apt-get clean - -# Install pip and virtualenv -RUN curl -L https://bootstrap.pypa.io/get-pip.py | python3 \ -&& python3 -m pip install 'virtualenv==20.0.5' - -# Install bazelisk -RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VERSION}/bazelisk-linux-amd64 -o bazelisk-linux-amd64 \ -&& install -m0755 bazelisk-linux-amd64 /usr/local/bin/bazelisk \ -&& ln -s /usr/local/bin/bazelisk /usr/local/bin/bazel \ -&& rm bazelisk-linux-amd64 - -##### Switch to non-root-user - -RUN groupadd -g ${UID} ${USERNAME} \ -&& useradd -m -r -u ${UID} -g ${USERNAME} ${USERNAME} \ -&& mkdir -p /home/${USERNAME}/workdir \ -&& chown -R ${USERNAME}:${USERNAME} /home/${USERNAME} -USER ${USERNAME} -WORKDIR /home/${USERNAME}/workdir - -# Create AWS-CLI virtualenv -RUN python3 -m virtualenv .venv-aws && . .venv-aws/bin/activate && python3 -m pip install 'awscli==1.18.5' && deactivate - -# Install maven -RUN curl -L https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip -o apache-maven-${MAVEN_VERSION}-bin.zip \ -&& unzip apache-maven-${MAVEN_VERSION}-bin.zip \ -&& rm -f apache-maven-${MAVEN_VERSION}-bin.zip - -# Setup JAVA_HOME -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/ - -# Add Maven to PATH -ENV PATH $PATH:/home/${USERNAME}/workdir/apache-maven-${MAVEN_VERSION}/bin:/home/${USERNAME}/bin - -# Fetch all the current Maven dependencies and bazel version -RUN git clone --depth=1 --branch=master https://github.com/batfish/batfish \ -&& cd batfish \ -&& mvn -f projects verify -DskipTests=false \ -&& mvn -f projects dependency:get -Dartifact=com.google.googlejavaformat:google-java-format:${GOOGLE_JAVA_FORMAT_VERSION}:jar:all-deps \ -&& mvn -f projects dependency:get -Dartifact=org.jacoco:org.jacoco.cli:${JACOCO_VERSION}:jar:nodeps \ -&& bazel version \ -&& cd .. \ -&& rm -rf batfish