Slack sync upstream 2019 11 09.r0

.github/bootstrap.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+# shellcheck disable=SC2164
+
+# Copyright 2019 The Vitess Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a next-gen bootstrap which skips Python and Java tests,
+# and does not use the VTROOT/VTTOP layout.
+#
+# My original intention was to use the same bootstrap.sh and gate
+# for new features, but it has turned out to be difficult to do,
+# due to the way that Docker cache works in the CI environment.
+
+function fail() {
+  echo "ERROR: $1"
+  exit 1
+}
+
+[[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory"
+
+# Create main directories.
+
+VTROOT="$PWD"
+
+mkdir -p dist
+mkdir -p bin
+mkdir -p lib
+mkdir -p vthook
+
+source ./dev.env
+
+go version &>/dev/null  || fail "Go is not installed or is not on \$PATH"
+goversion_min 1.12 || fail "Go is not version 1.12+"
+
+# Set up required soft links.
+# TODO(mberlin): Which of these can be deleted?
+ln -snf "$VTROOT/py" "$VTROOT/py-vtdb"
+ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh"
+ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
+ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
+
+# git hooks are only required if someone intends to contribute.
+
+echo "creating git hooks"
+mkdir -p "$VTROOT/.git/hooks"
+ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit"
+ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg"
+git config core.hooksPath "$VTROOT/.git/hooks"
+
+# install_dep is a helper function to generalize the download and installation of dependencies.
+#
+# If the installation is successful, it puts the installed version string into
+# the $dist/.installed_version file. If the version has not changed, bootstrap
+# will skip future installations.
+function install_dep() {
+  if [[ $# != 4 ]]; then
+    fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*"
+  fi
+  local name="$1"
+  local version="$2"
+  local dist="$3"
+  local install_func="$4"
+
+  version_file="$dist/.installed_version"
+  if [[ -f "$version_file" && "$(cat "$version_file")" == "$version" ]]; then
+    echo "skipping $name install. remove $dist to force re-install."
+    return
+  fi
+
+  echo "installing $name $version"
+
+  # shellcheck disable=SC2064
+  trap "fail '$name build failed'; exit 1" ERR
+
+  # Cleanup any existing data and re-create the directory.
+  rm -rf "$dist"
+  mkdir -p "$dist"
+
+  # Change $CWD to $dist before calling "install_func".
+  pushd "$dist" >/dev/null
+  # -E (same as "set -o errtrace") makes sure that "install_func" inherits the
+  # trap. If here's an error, the trap will be called which will exit this
+  # script.
+  set -E
+  $install_func "$version" "$dist"
+  set +E
+  popd >/dev/null
+
+  trap - ERR
+
+  echo "$version" > "$version_file"
+}
+
+
+#
+# 1. Installation of dependencies.
+#
+
+# Wrapper around the `arch` command which plays nice with OS X
+function get_arch() {
+  case $(uname) in
+    Linux) arch;;
+    Darwin) uname -m;;
+  esac
+}
+
+# Install protoc.
+function install_protoc() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux;;
+    Darwin) local platform=osx;;
+  esac
+
+  case $(get_arch) in
+      aarch64)  local target=aarch_64;;
+      x86_64)  local target=x86_64;;
+      *)   echo "ERROR: unsupported architecture"; exit 1;;
+  esac
+
+  wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip
+  unzip "protoc-$version-$platform-${target}.zip"
+  ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
+}
+protoc_ver=3.6.1
+install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
+
+# Download and install etcd, link etcd binary into our root.
+function install_etcd() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux; local ext=tar.gz;;
+    Darwin) local platform=darwin; local ext=zip;;
+  esac
+
+  case $(get_arch) in
+      aarch64)  local target=arm64;;
+      x86_64)  local target=amd64;;
+      *)   echo "ERROR: unsupported architecture"; exit 1;;
+  esac
+
+  download_url=https://github.com/coreos/etcd/releases/download
+  file="etcd-${version}-${platform}-${target}.${ext}"
+
+  wget "$download_url/$version/$file"
+  if [ "$ext" = "tar.gz" ]; then
+    tar xzf "$file"
+  else
+    unzip "$file"
+  fi
+  rm "$file"
+  ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd"
+}
+install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
+
+echo
+echo "bootstrap finished"

.github/workflows/local-example.yml

@@ -0,0 +1,41 @@
+name: Local Example
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v1
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run bootstrap.sh
+      run: |
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        GOBIN=$PWD/bin make build
+
+    - name: Run Local Example
+      run: |
+        export PATH=$PWD/bin:$PATH
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD test/local_example.sh
+

.travis.yml

@@ -17,7 +17,7 @@
 #
 # Open TODOs:
 # - Re-add travis/check_make_proto.sh, ideally as part of test/config.json.
-# - Add a presubmit which checks that vendor/vendor.json is the same as in the Docker image. This will prevent people from making changes to it without pushing new bootstrap Docker images.
+# - Add a presubmit which checks that if bootstrap has changed, and docker image is out of date.
 
 # sudo is required because we run Docker in our builds.
 # See: https://docs.travis-ci.com/user/docker/

ADOPTERS.md

@@ -1,15 +1,21 @@
 This is an alphabetical list of known adopters of Vitess. Some have already gone into production, and others are at various stages of testing.
 
-* [YouTube](https://youtube.com)
 * [Axon](https://axon.com)
 * [BetterCloud](https://bettercloud.com)
+* [CloudSigma](https://www.cloudsigma.com/)
 * [FlipKart](https://flipkart.com)
+* [GitHub](http://github.com/)
 * [HubSpot](https://product.hubspot.com/)
 * [JD](https://jd.com/)
+* [New Relic](https://newrelic.com)
 * [Nozzle](https://nozzle.io)
+* [opensooq.com](https://www.opensooq.com/)
+* [Peak Games](https://peak.com/)
 * [Pinterest](https://pinterest.com)
 * [Pixel Federation](https://pixelfederation.com)
 * [Quiz of Kings](https://quizofkings.com)
 * [Slack](https://slack.com)
 * [Square](https://square.com)
 * [Stitch Labs](https://stitchlabs.com)
+* [Weave](https://www.getweave.com)
+* [YouTube](https://youtube.com)

GOVERNANCE.md

@@ -21,7 +21,7 @@ Users who continue to engage with the project and its community will often becom
 
 ## Contributors
 
-Contributors will be added to the [Collaborators list](https://github.com/vitessio/vitess/settings/collaboration).
+Contributors will be added to the [Contributors list](https://github.com/vitessio/vitess/graphs/contributors).
 
 Contributors are community members who contribute in concrete ways to the project. Anyone can become a contributor. There is no expectation of commitment to the project, no specific skill requirements and no selection process.
 
@@ -42,7 +42,6 @@ Contributors engage with the project through the issue tracker, mailing list, th
 As contributors gain experience and familiarity with the project, their profile within, and commitment to, the community will increase. At some stage, they may find themselves being nominated for committership.
 
 ## Committers
-Committers will be added to the [committers list](https://github.com/orgs/youtube/teams/vitess-committers) and the [pullapprove list](https://github.com/vitessio/vitess/blob/master/.pullapprove.yml). 
 
 Committers are community members who have shown that they are committed to the continued development of the project through ongoing engagement with the community. Committership allows contributors to more easily carry on with their project related activities by giving them direct access to the project’s resources. That is, they can make changes directly to project outputs, without having to submit changes via patches.
 
@@ -66,8 +65,6 @@ A committer who shows an above-average level of contribution to the project, par
 
 ## Project management committee
 
-PMC members will be added to the [list of administrators](https://github.com/orgs/youtube/teams/vitess-admin).
-
 The current list of PMC members and their github handles are:
 
 * Alain Jobart (alainjobart)

MAINTAINERS.md

@@ -2,6 +2,8 @@ This page lists all active maintainers and their areas of expertise. This can be
 
 The following is the full list, alphabetically ordered.
 
+* Andres Taylor ([systay](https://github.com/systay)) antaylor@squareup.com
+* Anthony Yeh ([enisoc](https://github.com/enisoc)) enisoc@planetscale.com
 * Dan Kozlowski ([dkhenry](https://github.com/dkhenry)) koz@planetscale.com
 * David Weitzman ([dweitzman](https://github.com/dweitzman)) dweitzman@pinterest.com
 * Deepthi Sigireddi ([deepthi](https://github.com/deepthi)) deepthi@planetscale.com
@@ -11,28 +13,29 @@ The following is the full list, alphabetically ordered.
 * Leo X. Lin ([leoxlin](https://github.com/leoxlin)) llin@hubspot.com
 * Michael Demmer ([demmer](https://github.com/demmer)) mdemmer@slack-corp.com
 * Michael Pawliszyn ([mpawliszyn](https://github.com/mpawliszyn)) mikepaw@squareup.com
+* Morgan Tocker ([morgo](https://github.com/morgo)) morgan@planetscale.com
 * Rafael Chacon ([rafael](https://github.com/rafael)) rchacon@slack-corp.com
 * Sugu Sougoumarane ([sougou](https://github.com/sougou)) sougou@planetscale.com
 
 ## Areas of expertise
 
 ### General Vitess
-sougou, demmer, rafael, dweitzman, tirsen
+sougou, demmer, rafael, dweitzman, tirsen, morgo, enisoc
 
 ### Builds
-dkhenry
+dkhenry, morgo
 
 ### Resharding
-sougou, rafael, tirsen, dweitzman
+sougou, rafael, tirsen, dweitzman, systay
 
 ### Parser
-sougou, dweitzman, deepthi
+sougou, dweitzman, deepthi, systay
 
-### Backups
-deepthi, rafael
+### Cluster Management
+deepthi, rafael, enisoc
 
 ### Java
 mpawliszyn, leoxlin, harshit-gangal
 
 ### Kubernetes
-derekperkins, dkhenry
+derekperkins, dkhenry, enisoc

Makefile

@@ -1,4 +1,4 @@
-# Copyright 2017 Google Inc.
+# Copyright 2019 The Vitess Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ MAKEFLAGS = -s
 # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time.
 .NOTPARALLEL:
 
-.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests
+.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race
 
 all: build
 
@@ -81,18 +81,17 @@ cleanall:
 	# directories created by bootstrap.sh
 	# - exclude vtdataroot and vthook as they may have data we want
 	rm -rf ../../../../bin ../../../../dist ../../../../lib ../../../../pkg
-	# keep the vendor.json file but nothing else under the vendor directory as it's not actually part of the Vitess repo
-	rm -rf vendor/cloud.google.com vendor/github.com vendor/golang.org vendor/google.golang.org vendor/gopkg.in
-	# other stuff in the go hierarchy that is not under vendor/
-	rm -rf ../../../golang.org ../../../honnef.co
-	rm -rf ../../../github.com/golang ../../../github.com/kardianos ../../../github.com/kisielk
 	# Remind people to run bootstrap.sh again
 	echo "Please run bootstrap.sh again to setup your environment"
 
 unit_test: build
 	echo $$(date): Running unit tests
 	go test $(VT_GO_PARALLEL) ./go/...
 
+e2e_test: build
+	echo $$(date): Running endtoend tests
+	go test $(VT_GO_PARALLEL) ./go/.../endtoend/...
+
 # Run the code coverage tools, compute aggregate.
 # If you want to improve in a directory, run:
 #   go test -coverprofile=coverage.out && go tool cover -html=coverage.out
@@ -102,6 +101,9 @@ unit_test_cover: build
 unit_test_race: build
 	tools/unit_test_race.sh
 
+e2e_test_race: build
+	tools/e2e_test_race.sh
+
 .ONESHELL:
 SHELL = /bin/bash
 
@@ -115,13 +117,9 @@ site_integration_test:
 
 java_test:
 	go install ./go/cmd/vtgateclienttest ./go/cmd/vtcombo
-	mvn -f java/pom.xml clean verify
+	mvn -f java/pom.xml -B clean verify
 
-# TODO(mberlin): Remove the manual copy once govendor supports a way to
-# install vendor'd programs: https://github.com/kardianos/govendor/issues/117
 install_protoc-gen-go:
-	mkdir -p $${GOPATH}/src/github.com/golang/
-	cp -a vendor/github.com/golang/protobuf $${GOPATH}/src/github.com/golang/
 	go install github.com/golang/protobuf/protoc-gen-go
 
 # Find protoc compiler.
@@ -173,7 +171,7 @@ docker_bootstrap_test:
 	flavors='$(DOCKER_IMAGES_FOR_TEST)' && ./test.go -pull=false -parallel=2 -flavor=$${flavors// /,}
 
 docker_bootstrap_push:
-	for i in $(DOCKER_IMAGES); do echo "pushing boostrap image: $$i"; docker push vitess/bootstrap:$$i || exit 1; done
+	for i in $(DOCKER_IMAGES); do echo "pushing bootstrap image: $$i"; docker push vitess/bootstrap:$$i || exit 1; done
 
 # Use this target to update the local copy of your images with the one on Dockerhub.
 docker_bootstrap_pull:
@@ -268,7 +266,6 @@ rebalance_tests:
 
 # Release a version.
 # This will generate a tar.gz file into the releases folder with the current source
-# as well as the vendored libs.
 release: docker_base
 	@if [ -z "$VERSION" ]; then \
 	  echo "Set the env var VERSION with the release version"; exit 1;\