Skip to content

Commit

Permalink
Move docs and gallery deployment to Cirrus, add Docker image for Linux (
Browse files Browse the repository at this point in the history
flutter#20097)

This adds a Docker image for the linux builds, replacing a lot of the setup code with a Docker build.

Added a docker image build step that has the right gcloud credentials in it.

Also, this finally moves the gallery deployment and docs publishing steps to Cirrus. They were dependent upon some environment setup that was a lot easier to do in Docker than in a setup bash script.
  • Loading branch information
gspencergoog authored Aug 4, 2018
1 parent 75960f3 commit a5c2ddd
Show file tree
Hide file tree
Showing 10 changed files with 367 additions and 42 deletions.
91 changes: 75 additions & 16 deletions .cirrus.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
container:
image: cirrusci/flutter:base
image: gcr.io/flutter-cirrus/build-flutter-image:latest

task:
env:
# Name the SDK directory to include a space so that we constantly
# test path names with spaces in them.
CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"

git_fetch_script: git fetch origin
ANDROID_HOME: "/opt/android_sdk"
git_fetch_script:
- git fetch origin
- git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
pub_cache:
folder: $HOME/.pub-cache
fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
Expand All @@ -16,22 +20,26 @@ task:
artifacts_cache:
folder: bin/cache/artifacts
fingerprint_script: echo $OS; cat bin/internal/engine.version
setup_script: |
echo "SDK directory is: $PWD"
./bin/flutter --version
# disable analytics on the bots and download Flutter dependencies
./bin/flutter config --no-analytics
# run pub get in all the repo packages
./bin/flutter update-packages
git fetch origin master
setup_script: ./dev/bots/cirrus_setup.sh
matrix:
- name: docs
env:
SHARD: docs
# For uploading docs to Firebase
FIREBASE_TOKEN: ENCRYPTED[a40fcb68452b92254b7f532a460529b5e4ca51d6d338a8fae8db9db832ad3c2a7efb962e257de79686a9345f4a18661a]
docs_script: ./dev/bots/docs.sh
- name: deploy_gallery
only_if: $CIRRUS_BRANCH == 'dev'
depends_on:
- docs
- analyze
- tests-linux
- tool_tests-linux
env:
SHARD: deploy_gallery
GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346]
ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0b3e681b4507aec433ef29c79b715f15f8c75ecd25315ea286b0b2bcb8b28d578634eead5aa2c54086a25e8da1bb219a]
test_script: ./dev/bots/deploy_gallery.sh
- name: analyze
env:
SHARD: analyze
Expand Down Expand Up @@ -61,7 +69,9 @@ task:
cpu: 4
env:
CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
git_fetch_script: git fetch origin
git_fetch_script:
- git fetch origin
- git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
pub_cache:
folder: $APPDATA\Pub\Cache
fingerprint_script:
Expand All @@ -87,13 +97,46 @@ task:
SHARD: tool_tests

task:
name: deploy_gallery-macos
only_if: $CIRRUS_BRANCH == 'dev'
pub_cache:
folder: ~/.pub-cache
depends_on:
- analyze
- tests-macos
- tool_tests-macos
env:
# Name the SDK directory to include a space so that we constantly
# test path names with spaces in them.
CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
SHARD: deploy_gallery
# Apple Certificates Match Passphrase
MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73]
# Apple Fastlane Password
FASTLANE_PASSWORD: ENCRYPTED[0bf9bb0cc2cb32a0ed18470cf2c9df0f587cce5f8b04adbd6cff15ca5bde7a74f721ee580227b132ab6b032f08e52ae0]
# Private repo for publishing certificates.
PUBLISHING_MATCH_CERTIFICATE_REPO: git@github.com:flutter/private_publishing_certificates.git
osx_instance:
image: high-sierra-xcode-9.4.1
git_fetch_script:
- git fetch origin
- git fetch origin master # To set FETCH_HEAD
setup_script:
- bin/flutter config --no-analytics
- bin/flutter update-packages
test_all_script:
- ./dev/bots/deploy_gallery.sh

task:
osx_instance:
image: high-sierra-xcode-9.4.1
depends_on:
- analyze
env:
CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
git_fetch_script: git fetch origin
git_fetch_script:
- git fetch origin
- git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
pub_cache:
folder: $HOME/.pub-cache
fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
Expand All @@ -116,3 +159,19 @@ task:
- name: tool_tests-macos
env:
SHARD: tool_tests


docker_builder:
# Only build a new docker image when we tag a release (for dev, beta, or release.)
only_if: $CIRRUS_TAG != ''
env:
GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6]
depends_on:
- docs
- analyze
- tests-linux
- tool_tests-linux
build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh"
login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh"
push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"

43 changes: 43 additions & 0 deletions dev/bots/cirrus_setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash
set -e

# This script is only meant to be run by the Cirrus CI system, not locally.
# It must be run from the root of the Flutter repo.

# Collects log output in a tmpfile, but only prints it if the command fails.
function log_on_fail() {
local COMMAND="$@"
local TMPDIR="$(mktemp -d)"
local TMPFILE="$TMPDIR/command.log"
local EXIT=0
if ("$@" > "$TMPFILE" 2>&1); then
echo "'$COMMAND' succeeded."
else
EXIT=$?
cat "$TMPFILE" 1>&2
echo "FAIL: '$COMMAND' exited with code $EXIT" 1>&2
fi
rm -rf "$TMPDIR"
return "$EXIT"
}

function accept_android_licenses() {
yes "y" | flutter doctor --android-licenses
}

echo "Flutter SDK directory is: $PWD"

# Run flutter to download dependencies and precompile things, and to disable
# analytics on the bots.
echo "Downloading build dependencies and pre-compiling Flutter snapshot"
log_on_fail ./bin/flutter config --no-analytics

# Run doctor, to print it to the log for debugging purposes.
./bin/flutter doctor -v

# Accept licenses.
log_on_fail accept_android_licenses && echo "Android licenses accepted."

# Run pub get in all the repo packages.
echo "Updating packages for Flutter."
log_on_fail ./bin/flutter update-packages
77 changes: 77 additions & 0 deletions dev/bots/deploy_gallery.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/bin/bash

set -e

function script_location() {
local script_location="${BASH_SOURCE[0]}"
# Resolve symlinks
while [[ -h "$script_location" ]]; do
DIR="$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
script_location="$(readlink "$script_location")"
[[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
done
echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
}

# So that users can run this script locally from any directory and it will work as
# expected.
SCRIPT_LOCATION="$(script_location)"
FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"

export PATH="$FLUTTER_ROOT/bin:$FLUTTER_ROOT/bin/cache/dart-sdk/bin:$PATH"

set -x

cd "$FLUTTER_ROOT"

if [[ "$SHARD" = "deploy_gallery" ]]; then
version="$(<version)"
if [[ "$OS" == "linux" ]]; then
echo "Building Flutter Gallery $version for Android..."
export ANDROID_HOME="$PWD/android-sdk"
(
cd examples/flutter_gallery
flutter build apk --release -t lib/main_publish.dart
)
echo "Android Flutter Gallery built"
if [[ -z "$CIRRUS_PULL_REQUEST" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
echo "Deploying Flutter Gellery $version to Play Store..."
(
cd examples/flutter_gallery/android
bundle install
bundle exec fastlane deploy_play_store
)
else
echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
fi
elif [[ "$OS" == "macos" ]]; then
echo "Building Flutter Gallery $version for iOS..."
(
cd examples/flutter_gallery
flutter build ios --release --no-codesign -t lib/main_publish.dart
)
echo "iOS Flutter Gallery built"
if [[ -z "$CIRRUS_PULL_REQUEST" ]]; then
if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
echo "Archiving with distribution profile and deploying to TestFlight..."
(
cd examples/flutter_gallery/ios
bundle install
bundle exec fastlane build_and_deploy_testflight upload:true
)
else
echo "Archiving with distribution profile..."
(
cd examples/flutter_gallery/ios
bundle install
bundle exec fastlane build_and_deploy_testflight
)
echo "Archive is only deployed to TestFlight on tagged dev branch commits"
fi
else
echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
fi
fi
else
echo "Doing nothing: not on the 'deploy_gallery' SHARD."
fi
70 changes: 44 additions & 26 deletions dev/bots/docs.sh
Original file line number Diff line number Diff line change
@@ -1,58 +1,76 @@
#!/bin/bash
set -e

echo "Running docs.sh"
function script_location() {
local script_location="${BASH_SOURCE[0]}"
# Resolve symlinks
while [[ -h "$script_location" ]]; do
DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)"
script_location="$(readlink "$script_location")"
[[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
done
echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
}

# If you want to run this script locally, make sure you run it from
# the root of the flutter repository.
export FLUTTER_ROOT="$PWD"
export PATH="$PWD/bin:$PATH"
# So that users can run this script from anywhere and it will work as expected.
SCRIPT_LOCATION="$(script_location)"
FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"

if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then
echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)"
exit 1
fi

FLUTTER_BIN="$FLUTTER_ROOT/bin"
DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin"
FLUTTER="$FLUTTER_BIN/flutter"
DART="$DART_BIN/dart"
PUB="$DART_BIN/pub"
export PATH="$FLUTTER_BIN:$DART_BIN:$PATH"

# This is called from travis_upload.sh on Travis.

# Make sure dart is installed
bin/flutter --version
# Make sure dart is installed by invoking flutter to download it.
"$FLUTTER" --version

# If the pub cache directory exists in the root, then use that.
FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
if [ -d "$FLUTTER_PUB_CACHE" ]; then
if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
# This has to be exported, because pub interprets setting it
# to the empty string in the same way as setting it to ".".
export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
fi

# Install dartdoc.
bin/cache/dart-sdk/bin/pub global activate dartdoc 0.20.2
"$PUB" global activate dartdoc 0.20.2

# This script generates a unified doc set, and creates
# a custom index.html, placing everything into dev/docs/doc.
(cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get)
bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart
bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart
(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get)
(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")

# Ensure google webmaster tools can verify our site.
cp dev/docs/google2ed1af765c529f57.html dev/docs/doc

# Upload new API docs when on Travis
if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
echo "This is not a pull request; considering whether to upload docs... (branch=$TRAVIS_BRANCH)"
if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_BRANCH" == "beta" ]; then
cd dev/docs
cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc"

if [ "$TRAVIS_BRANCH" == "master" ]; then
# Upload new API docs when on Cirrus
if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then
echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)"
if [[ "$CIRRUS_BRANCH" == "master" || "$CIRRUS_BRANCH" == "beta" ]]; then
if [[ "$CIRRUS_BRANCH" == "master" ]]; then
echo "Updating master docs: https://master-docs-flutter-io.firebaseapp.com/"
echo -e "User-agent: *\nDisallow: /" > doc/robots.txt
while : ; do
firebase deploy --project master-docs-flutter-io && break
echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
while true; do
(cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --project master-docs-flutter-io) && break
echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
sleep 5
done
fi

if [ "$TRAVIS_BRANCH" == "beta" ]; then
if [[ "$CIRRUS_BRANCH" == "beta" ]]; then
echo "Updating beta docs: https://docs.flutter.io/"
while : ; do
firebase deploy --project docs-flutter-io && break
while true; do
(cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --project docs-flutter-io) && break
echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
sleep 5
done
Expand Down
1 change: 1 addition & 0 deletions dev/ci/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This directory includes scripts and tools for continuous integration builds and tests.
Loading

0 comments on commit a5c2ddd

Please sign in to comment.