From c976134b8f4133aeb73a6725925b8bd8ade7fa4f Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 21 Oct 2015 16:19:36 -0700 Subject: [PATCH] backup: Use GCS backup in Kubernetes example. --- .gitignore | 4 + doc/BackupAndRestore.md | 57 +++- doc/GettingStartedKubernetes.md | 314 +++++++++++------- examples/kubernetes/configure.sh | 59 ++++ examples/kubernetes/env.sh | 8 + ...r.yaml => vtctld-controller-template.yaml} | 16 +- examples/kubernetes/vtctld-up.sh | 9 +- examples/kubernetes/vttablet-down.sh | 2 + .../kubernetes/vttablet-pod-template.yaml | 27 +- examples/kubernetes/vttablet-up.sh | 5 +- examples/local/vtctld-up.sh | 2 + examples/local/vttablet-up.sh | 1 + 12 files changed, 346 insertions(+), 158 deletions(-) create mode 100755 examples/kubernetes/configure.sh rename examples/kubernetes/{vtctld-controller.yaml => vtctld-controller-template.yaml} (62%) diff --git a/.gitignore b/.gitignore index d31064e8132..44d44c7fec7 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ _test/ # php downloaded dependencies /php/composer.phar /php/vendor + +# site-local example files +/examples/kubernetes/config.sh + diff --git a/doc/BackupAndRestore.md b/doc/BackupAndRestore.md index dadf1d85231..5f39988bed6 100644 --- a/doc/BackupAndRestore.md +++ b/doc/BackupAndRestore.md @@ -7,18 +7,16 @@ Vitess. Vitess uses backups for two purposes: ## Prerequisites Vitess stores data backups on a Backup Storage service. Currently, -Vitess only supports backups to an NFS directory and can use any -network-mounted drive as the backup repository. The core Vitess software's -[interface.go](https://github.com/youtube/vitess/blob/master/go/vt/mysqlctl/backupstorage/interface.go) -file defines an interface for the Backup Storage service. The interface -defines methods for creating, listing, and removing backups. +Vitess supports backups to either [Google Cloud Storage](https://cloud.google.com/storage/) +or any network-mounted drive (such as NFS). The core Vitess software's +[BackupStorage interface](https://github.com/youtube/vitess/blob/master/go/vt/mysqlctl/backupstorage/interface.go) +defines methods for creating, listing, and removing backups. Plugins for other +storage services just need to implement the interface. Before you can back up or restore a tablet, you need to ensure that the tablet is aware of the Backup Storage system that you are using. To do so, use the following command-line flags when starting a vttablet that has -access to a local file system where you are storing backups. In practice, -you should always use these flags when starting a tablet that has access -to backups on a local file system. +access to the location where you are storing backups. @@ -28,20 +26,48 @@ to backups on a local file system. - - + + - - + + - - + + + + + + + + + +
--backup_storage_implementationSpecifies the implementation of the Backup Storage interface to use.

If you run Vitess on a machine that has access to an NFS directory where you store backups, set the flag's value to file. Otherwise, do not set this flag or either of the other remaining flags.		-backup_storage_implementationSpecifies the implementation of the Backup Storage interface to use.

+ Current plugin options available are: +
    +
  • gcs: For Google Cloud Storage.
    • +
  • file: For NFS or any other filesystem-mounted network drive.
    • +
+
--file_backup_storage_rootIdentifies the root directory for backups. Set this flag if the backup_storage_implementation flag is set to file.-file_backup_storage_rootFor the file plugin, this identifies the root directory for backups.
--restore_from_backupIndicates that, when started, the tablet should restore the most recent backup from the file_backup_storage_root directory. This flag is only relevant if the other two flags listed above are also set.-gcs_backup_storage_projectFor the gcs plugin, this identifies the project to use.
-gcs_backup_storage_bucketFor the gcs plugin, this identifies the bucket to use.
-restore_from_backupIndicates that, when started with an empty MySQL instance, the tablet should restore the most recent backup from the specified storage plugin.
+### Authentication + +Note that for the Google Cloud Storage plugin, we currently only support +[Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials), +which means that access to Cloud Storage is automatically granted by virtue of +the fact that you're already running within Google Compute Engine or Container Engine. + +For this to work, the GCE instances must have been created with the +[scope](https://cloud.google.com/compute/docs/authentication#using) that grants +read-write access to Cloud Storage. When using Container Engine, you can do this +for all the instances it creates by adding `--scopes storage-rw` to the +`gcloud container clusters create` command as shown in the [Vitess on Kubernetes guide] +(http://vitess.io/getting-started/#start-a-container-engine-cluster). + ## Creating a backup Run the following vtctl command to create a backup: @@ -74,7 +100,7 @@ In response to this command, the designated tablet performs the following sequen ## Restoring a backup When a tablet starts, Vitess checks the value of the ---restore_from_backup command-line flag to determine whether +-restore_from_backup command-line flag to determine whether to restore a backup to that tablet. * If the flag is present, Vitess tries to restore the most recent backup @@ -187,3 +213,4 @@ can control the concurrency using command-line flags: If the network link is fast enough, the concurrency matches the CPU usage of the process during the backup or restore process. + diff --git a/doc/GettingStartedKubernetes.md b/doc/GettingStartedKubernetes.md index 0920cf65cdc..a19945a8ece 100644 --- a/doc/GettingStartedKubernetes.md +++ b/doc/GettingStartedKubernetes.md @@ -132,13 +132,17 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl 1. Create a Container Engine cluster: ``` sh - $ gcloud container clusters create example --machine-type n1-standard-4 --num-nodes 5 + $ gcloud container clusters create example --machine-type n1-standard-4 --num-nodes 5 --scopes storage-rw ### example output: # Creating cluster example...done. # Created [https://container.googleapis.com/v1/projects/vitess/zones/us-central1-b/clusters/example]. # kubeconfig entry generated for example. ``` + **Note:** The `--scopes storage-rw` argument is necessary to allow + [built-in backup/restore](http://vitess.io/user-guide/backup-and-restore.html) + to access [Google Cloud Storage](https://cloud.google.com/storage/). + 1. The command's output includes the IP of the Kubernetes master server: ``` @@ -174,6 +178,18 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl # username: admin ``` +1. Create a Cloud Storage bucket: + + To use the Cloud Storage plugin for built-in backups, first create a + [bucket](https://cloud.google.com/storage/docs/concepts-techniques#concepts) + for Vitess backup data. See the + [bucket naming guidelines](https://cloud.google.com/storage/docs/bucket-naming) + if you're new to Cloud Storage. + + ``` sh + $ gsutil mb gs://my-backup-bucket + ``` + ## Start a Vitess cluster 1. **Navigate to your local Vitess source code** @@ -182,9 +198,41 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl `vtctlclient`: ``` sh - $ cd $GOPATH/src/github.com/youtube/vitess + $ cd $GOPATH/src/github.com/youtube/vitess/examples/kubernetes + ``` + +1. **Configure site-local settings** + + Run the `configure.sh` script to generate a `config.sh` file, which will be + used to customize your cluster settings. + + Currently, we have out-of-the-box support for storing + [backups](http://vitess.io/user-guide/backup-and-restore.html) in + [Google Cloud Storage](https://cloud.google.com/storage/). If you're using + GCS, fill in the fields requested by the configure script, including the + name of the bucket you created above. + + ``` sh + vitess/examples/kubernetes$ ./configure.sh + ### example output: + # Backup Storage (file, gcs) [gcs]: + # Google Developers Console Project [my-project]: + # Google Cloud Storage bucket for Vitess backups: my-backup-bucket + # Saving config.sh... ``` + For other platforms, you'll need to choose the `file` backup storage plugin, + and mount a read-write network volume into the `vttablet` and `vtctld` pods. + For example, you can mount any storage service accessible through NFS into a + [Kubernetes volume](http://kubernetes.io/v1.0/docs/user-guide/volumes.html#nfs). + Then provide the mount path to the configure script here. + + Direct support for other cloud blob stores like Amazon S3 can be added by + implementing the Vitess [BackupStorage plugin interface] + (https://github.com/youtube/vitess/blob/master/go/vt/mysqlctl/backupstorage/interface.go). + Let us know on the [discussion forum](https://groups.google.com/forum/#!forum/vitess) + if you have any specific plugin requests. + 1. **Start an etcd cluster** The Vitess [topology service](http://vitess.io/overview/concepts.html#topology-service) @@ -195,7 +243,6 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl Kubernetes itself. ``` sh - vitess$ cd examples/kubernetes vitess/examples/kubernetes$ ./etcd-up.sh ### example output: # Generating discovery token for global cell... @@ -288,7 +335,7 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl by `vtctld-up.sh` above. In this example, the web UI would be at `http://2.3.4.5:30000`. -1. **Use `vtctlclient` to call `vtctld`** +1. **Use vtctlclient to call vtctld** You can now run `vtctlclient` locally to issue commands to the `vtctld` service on your Kubernetes cluster. @@ -350,10 +397,13 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl # pods/vttablet-104 ``` - In the vtctld web UI, you should soon see a keyspace named `test_keyspace` - with a single shard named `0`. Click on the shard name to see the list of - tablets. When all 5 tablets show up on the shard status page, you're ready - to continue. + In the vtctld web UI, you should soon see a + [keyspace](http://vitess.io/overview/concepts.html#keyspace) named `test_keyspace` + with a single [shard](http://vitess.io/overview/concepts.html#shard) named `0`. + Click on the shard name to see the list of tablets. When all 5 tablets + show up on the shard status page, you're ready to continue. Note that it's + normal for the tablets to be unhealthy at this point, since you haven't + initialized the databases on them yet. It can take some time for the tablets to come up for the first time if a pod was scheduled on a node that hasn't downloaded the [Vitess Docker image] @@ -363,26 +413,17 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl ``` sh vitess/examples/kubernetes$ ./kvtctl.sh ListAllTablets test ### example output: - # test-0000000100 test_keyspace 0 replica 10.64.1.6:15002 10.64.1.6:3306 [] - # test-0000000101 test_keyspace 0 replica 10.64.2.5:15002 10.64.2.5:3306 [] - # test-0000000102 test_keyspace 0 replica 10.64.0.7:15002 10.64.0.7:3306 [] - # test-0000000103 test_keyspace 0 rdonly 10.64.1.7:15002 10.64.1.7:3306 [] - # test-0000000104 test_keyspace 0 rdonly 10.64.2.6:15002 10.64.2.6:3306 [] + # test-0000000100 test_keyspace 0 spare 10.64.1.6:15002 10.64.1.6:3306 [] + # test-0000000101 test_keyspace 0 spare 10.64.2.5:15002 10.64.2.5:3306 [] + # test-0000000102 test_keyspace 0 spare 10.64.0.7:15002 10.64.0.7:3306 [] + # test-0000000103 test_keyspace 0 spare 10.64.1.7:15002 10.64.1.7:3306 [] + # test-0000000104 test_keyspace 0 spare 10.64.2.6:15002 10.64.2.6:3306 [] ``` - Note that of the 5 tablets, the first 3 were assigned to be - **replica** type (for serving live web traffic), while the last 2 - were assigned to be **rdonly** type (for offline processing). - These allocations can be configured in the `vttablet-up.sh` script. - See the [tablet](http://vitess.io/overview/concepts.html#tablet) - reference for more about the available tablet types. +1. **Initialize MySQL databases** - By bringing up tablets in a previously empty - [keyspace](http://vitess.io/overview/concepts.html#keyspace), - you have effectively just created a new - [shard](http://vitess.io/overview/concepts.html#shard). - To complete the initialization of the keyspace for the new - shard, call the `kvtctl.sh RebuildKeyspaceGraph` command: + Once all the tablets show up, you're ready to initialize the underlying + MySQL databases. First, rebuild the keyspace to propagate the new shard: ``` sh vitess/examples/kubernetes$ ./kvtctl.sh RebuildKeyspaceGraph test_keyspace @@ -390,93 +431,11 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl **Note:** Many `vtctlclient` commands produce no output on success. - **_Status pages for vttablets_** - - Each `vttablet` serves a set of HTML status pages on its primary port. - The `vtctld` interface provides a **STATUS** link for each tablet, but the - links are actually to internal, per-pod IPs that can only be accessed from - within Kubernetes. - - As a workaround, you can access tablet status pages through the - [apiserver proxy](http://kubernetes.io/v1.0/docs/user-guide/accessing-the-cluster.html#accessing-services-running-on-the-cluster), - provided by the Kubernetes master. For example, to see the status - page for the tablet with ID 100 (recall that our Kubernetes master is - on public IP 1.2.3.4), you could navigate to: - - ``` - https://1.2.3.4/api/v1/proxy/namespaces/default/pods/vttablet-100:15002/debug/status - ``` - - In the future, we plan to have vtctld directly link through this proxy from - the **STATUS** link. - - **_Direct connection to mysqld_** - - Since the `mysqld` within the `vttablet` pod is only meant to be accessed - via vttablet, our default bootstrap settings only allow connections from - localhost. - - If you want to check or manipulate the underlying mysqld, you can issue - simple queries or commands through `vtctlclient` like this: - - ``` sh - # Send a query to tablet 100 in cell 'test'. - vitess/examples/kubernetes$ ./kvtctl.sh ExecuteFetchAsDba test-0000000100 "SELECT VERSION()" - ### example output: - # { - # "Fields": null, - # "RowsAffected": 1, - # "InsertId": 0, - # "Rows": [ - # [ - # "10.0.20-MariaDB-1~wheezy-log" - # ] - # ], - # "Err": null - # } - ``` - - If you need a truly direct connection to mysqld for bulk operations, - you can SSH to the Kubernetes node on which the pod is running. - Then use - [docker exec](https://docs.docker.com/reference/commandline/exec/) - to launch a bash shell inside the mysql container, and connect with the - `mysql` command-line client: - - ``` sh - # For example, to connect to the mysql container within the vttablet-100 pod: - $ kubectl get pods -o wide | grep vttablet-100 - ### example output: - # vttablet-100 2/2 Running 0 17m gke-example-960176fd-node-3mkd - $ gcloud compute ssh gke-example-960176fd-node-3mkd - gke-example-960176fd-node-3mkd:~$ sudo docker ps | grep vttablet-100 | grep k8s_mysql - ### example output: - # ef40b4ff08fa vitess/lite:latest [...] k8s_mysql.16e2a810_vttablet-100[...] - k8s-example-3c0115e4-node-x6jc:~$ sudo docker exec -ti ef40b4ff08fa bash - # Now you're in a shell inside the mysql container. - # We need to tell the mysql client the username and socket file to use. - vttablet-100:/# TERM=ansi mysql -u vt_dba -S /vt/vtdataroot/vt_0000000100/mysql.sock - ``` - - **Note:** `gcloud compute ssh` uses an SSH key to login to the Kubernetes - node. If you haven't done yet, `gcloud` will create an SSH key for - you and ask you for a passphrase for the SSH key. - For subsequent logins via SSH, it may prompt you for the passphrase again. - -1. **Elect a master vttablet** - - The tablets all start as slaves by default. In this step, you - designate one of the tablets to be the master. Vitess - automatically connects the other slaves' mysqld instances - so that they start replicating from the master's mysqld. - - Since this is the first time the shard has been started, - the tablets are not already doing any replication, and the - tablet types are all replica or rdonly. As a - result, the following command uses the `-force` - flag when calling the `InitShardMaster` command - to be able to promote one instance to master. - + Next, designate one of the tablets to be the initial master. Vitess will + automatically connect the other slaves' mysqld instances so that they start + replicating from the master's mysqld. This is also when the default database + is created. Since our keyspace is named `test_keyspace`, the MySQL database + will be named `vt_test_keyspace`. ``` sh vitess/examples/kubernetes$ ./kvtctl.sh InitShardMaster -force test_keyspace/0 test-0000000100 @@ -485,20 +444,13 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl # master-elect tablet test-0000000100 is not a master in the shard, proceeding anyway as -force was used ``` - **Note:** If you do not include the `-force` flag - here, the command will first check to ensure the provided - tablet is the only tablet of type master in the shard. - However, since none of the slaves are masters, and we're not - replicating at all, that check would fail and the command - would fail as well. + **Note:** Since this is the first time the shard has been started, the + tablets are not already doing any replication, and there is no existing + master. The `InitShardMaster` command above uses the `-force` flag to bypass + the usual sanity checks that would apply if this wasn't a brand new shard. - After running this command, go back to the **Shard Status** page - in the `vtctld` web interface. When you refresh the - page, you should see that one tablet is the master - and the others are replica or rdonly. - - You can also run this command on the command line to see the - same data: + After the tablets finish updating, you should see one **master**, and + several **replica** and **rdonly** tablets: ``` sh vitess/examples/kubernetes$ ./kvtctl.sh ListAllTablets test @@ -510,6 +462,11 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl # test-0000000104 test_keyspace 0 rdonly 10.64.2.6:15002 10.64.2.6:3306 [] ``` + The **replica** tablets are used for serving live web traffic, while the + **rdonly** tablets are used for offline processing, such as batch jobs and backups. + The amount of each [tablet type](http://vitess.io/overview/concepts.html#tablet) + that you launch can be configured in the `vttablet-up.sh` script. + 1. **Create a table** The `vtctlclient` tool can be used to apply the database schema @@ -555,7 +512,32 @@ $ export KUBECTL=/example/path/to/google-cloud-sdk/bin/kubectl # ... ``` -1. **Start `vtgate`** +1. **Take a backup** + + Now that the initial schema is applied, it's a good time to take the first + [backup](http://vitess.io/user-guide/backup-and-restore.html). This backup + will be used to automatically restore any additional replicas that you run, + before they connect themselves to the master and catch up on replication. + If an existing tablet goes down and comes back up without its data, it will + also automatically restore from the latest backup and then resume replication. + + Select one of the **rdonly** tablets and tell it to take a backup. We use a + **rdonly** tablet instead of a **replica** because the tablet will pause + replication and stop serving during data copy to create a consistent snapshot. + + ``` sh + vitess/examples/kubernetes$ ./kvtctl.sh Backup test-0000000104 + ``` + + After the backup completes, you can list available backups for the shard: + + ``` sh + vitess/examples/kubernetes$ ./kvtctl.sh ListBackups test_keyspace/0 + ### example output: + # 2015-10-21.042940.test-0000000104 + ``` + +1. **Start vtgate** Vitess uses [vtgate](http://vitess.io/overview/#vtgate) to route each client query to the correct `vttablet`. In Kubernetes, a `vtgate` service @@ -715,7 +697,7 @@ $ gcloud compute firewall-rules delete vtctld guestbook ## Troubleshooting -### Server Logs +### Server logs If a pod enters the `Running` state, but the server doesn't respond as expected, use the `kubectl logs` @@ -733,7 +715,7 @@ Post the logs somewhere and send a link to the [Vitess mailing list](https://groups.google.com/forum/#!forum/vitess) to get more help. -### Root Certificates +### Root certificates If you see in the logs a message like this: @@ -746,3 +728,79 @@ that puts root certificates in a different place than our configuration expects by default (for example, Fedora). See the comments in the [etcd controller template](https://github.com/youtube/vitess/blob/master/examples/kubernetes/etcd-controller-template.yaml) for examples of how to set the right location for your host OS. +You'll also need to adjust the same certificate path settings in the +`vtctld` and `vttablet` templates. + +### Status pages for vttablets + +Each `vttablet` serves a set of HTML status pages on its primary port. +The `vtctld` interface provides a **STATUS** link for each tablet, but the +links are actually to internal, per-pod IPs that can only be accessed from +within Kubernetes. + +As a workaround, you can access tablet status pages through the +[apiserver proxy](http://kubernetes.io/v1.0/docs/user-guide/accessing-the-cluster.html#accessing-services-running-on-the-cluster), +provided by the Kubernetes master. For example, to see the status +page for the tablet with ID 100 (recall that our Kubernetes master is +on public IP 1.2.3.4), you could navigate to: + +``` +https://1.2.3.4/api/v1/proxy/namespaces/default/pods/vttablet-100:15002/debug/status +``` + +In the future, we plan to have vtctld directly link through this proxy from +the **STATUS** link. + +### Direct connection to mysqld + +Since the `mysqld` within the `vttablet` pod is only meant to be accessed +via vttablet, our default bootstrap settings only allow connections from +localhost. + +If you want to check or manipulate the underlying mysqld, you can issue +simple queries or commands through `vtctlclient` like this: + +``` sh +# Send a query to tablet 100 in cell 'test'. +vitess/examples/kubernetes$ ./kvtctl.sh ExecuteFetchAsDba test-0000000100 "SELECT VERSION()" +### example output: +# { +# "Fields": null, +# "RowsAffected": 1, +# "InsertId": 0, +# "Rows": [ +# [ +# "10.0.20-MariaDB-1~wheezy-log" +# ] +# ], +# "Err": null +# } +``` + +If you need a truly direct connection to mysqld for bulk operations, +you can SSH to the Kubernetes node on which the pod is running. +Then use +[docker exec](https://docs.docker.com/reference/commandline/exec/) +to launch a bash shell inside the mysql container, and connect with the +`mysql` command-line client: + +``` sh +# For example, to connect to the mysql container within the vttablet-100 pod: +$ kubectl get pods -o wide | grep vttablet-100 +### example output: +# vttablet-100 2/2 Running 0 17m gke-example-960176fd-node-3mkd +$ gcloud compute ssh gke-example-960176fd-node-3mkd +gke-example-960176fd-node-3mkd:~$ sudo docker ps | grep vttablet-100 | grep k8s_mysql +### example output: +# ef40b4ff08fa vitess/lite:latest [...] k8s_mysql.16e2a810_vttablet-100[...] +k8s-example-3c0115e4-node-x6jc:~$ sudo docker exec -ti ef40b4ff08fa bash +# Now you're in a shell inside the mysql container. +# We need to tell the mysql client the username and socket file to use. +vttablet-100:/# TERM=ansi mysql -u vt_dba -S /vt/vtdataroot/vt_0000000100/mysql.sock +``` + +**Note:** `gcloud compute ssh` uses an SSH key to login to the Kubernetes +node. If you haven't done yet, `gcloud` will create an SSH key for +you and ask you for a passphrase for the SSH key. +For subsequent logins via SSH, it may prompt you for the passphrase again. + diff --git a/examples/kubernetes/configure.sh b/examples/kubernetes/configure.sh new file mode 100755 index 00000000000..b4fe4ac4a87 --- /dev/null +++ b/examples/kubernetes/configure.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# This script generates config.sh, which is a site-local config file that is not +# checked into source control. + +# Select and configure Backup Storage Implementation. +storage=gcs +read -p "Backup Storage (file, gcs) [gcs]: " +if [ -n "$REPLY" ]; then storage="$REPLY"; fi + +case "$storage" in +gcs) + # Google Cloud Storage + project=$(gcloud config list project | grep 'project\s*=' | sed -r 's/^.*=\s*(.*)$/\1/') + read -p "Google Developers Console Project [$project]: " + if [ -n "$REPLY" ]; then project="$REPLY"; fi + if [ -z "$project" ]; then + echo "ERROR: Project name must not be empty." + exit 1 + fi + + read -p "Google Cloud Storage bucket for Vitess backups: " bucket + if [ -z "$bucket" ]; then + echo "ERROR: Bucket name must not be empty." + exit 1 + fi + echo + echo "NOTE: If you haven't already created this bucket, you can do so by running:" + echo " gsutil mb gs://$bucket" + echo + + backup_flags=$(echo -backup_storage_implementation gcs \ + -gcs_backup_storage_project "'$project'" \ + -gcs_backup_storage_bucket "'$bucket'") + ;; +file) + # Mounted volume (e.g. NFS) + read -p "Root directory for backups (usually an NFS mount): " file_root + if [ -z "$file_root" ]; then + echo "ERROR: Root directory must not be empty." + exit 1 + fi + echo + echo "NOTE: You must add your NFS mount to the vtctld-controller-template" + echo " and vttablet-pod-template as described in the Kubernetes docs:" + echo " http://kubernetes.io/v1.0/docs/user-guide/volumes.html#nfs" + echo + + backup_flags=$(echo -backup_storage_implementation file \ + -file_backup_storage_root "'$file_root'") + ;; +*) + echo "ERROR: Unsupported backup storage implementation: $storage" + exit 1 +esac + +echo "Saving config.sh..." +echo "backup_flags=\"$backup_flags\"" > config.sh + diff --git a/examples/kubernetes/env.sh b/examples/kubernetes/env.sh index 51e487060cc..b9ebb7de9df 100644 --- a/examples/kubernetes/env.sh +++ b/examples/kubernetes/env.sh @@ -25,3 +25,11 @@ get_vtctld_addr() { echo "$VTCTLD_ADDR" } +config_file=`dirname "${BASH_SOURCE}"`/config.sh +if [ ! -f $config_file ]; then + echo "Please run ./configure.sh first to generate config.sh file." + exit 1 +fi + +source $config_file + diff --git a/examples/kubernetes/vtctld-controller.yaml b/examples/kubernetes/vtctld-controller-template.yaml similarity index 62% rename from examples/kubernetes/vtctld-controller.yaml rename to examples/kubernetes/vtctld-controller-template.yaml index bb8523d9276..7b522712b6b 100644 --- a/examples/kubernetes/vtctld-controller.yaml +++ b/examples/kubernetes/vtctld-controller-template.yaml @@ -18,6 +18,11 @@ spec: mountPath: /dev/log - name: vtdataroot mountPath: /vt/vtdataroot + - name: certs + readOnly: true + # Mount root certs from the host OS into the location + # expected for our container OS (Debian): + mountPath: /etc/ssl/certs/ca-certificates.crt resources: limits: memory: "128Mi" @@ -39,9 +44,18 @@ spec: -topo_implementation etcd -tablet_protocol grpc -tablet_manager_protocol grpc - -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT" vitess + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + {{backup_flags}}" vitess volumes: - name: syslog hostPath: {path: /dev/log} - name: vtdataroot emptyDir: {} + - name: certs + # Uncomment one of the following lines to configure the location + # of the root certificates file on your host OS. We need this so + # we can import it into the container OS. + # If your host OS is Fedora/RHEL: + #hostPath: {path: /etc/pki/tls/certs/ca-bundle.crt} + # If your host OS is Debian/Ubuntu/Gentoo: + hostPath: {path: /etc/ssl/certs/ca-certificates.crt} diff --git a/examples/kubernetes/vtctld-up.sh b/examples/kubernetes/vtctld-up.sh index 18474fc3a0e..cef1ee3952b 100755 --- a/examples/kubernetes/vtctld-up.sh +++ b/examples/kubernetes/vtctld-up.sh @@ -11,7 +11,14 @@ echo "Creating vtctld service..." $KUBECTL create -f vtctld-service.yaml echo "Creating vtctld replicationcontroller..." -$KUBECTL create -f vtctld-controller.yaml +# Expand template variables +sed_script="" +for var in backup_flags; do + sed_script+="s,{{$var}},${!var},g;" +done + +# Instantiate template and send to kubectl. +cat vtctld-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - server=$(get_vtctld_addr) echo diff --git a/examples/kubernetes/vttablet-down.sh b/examples/kubernetes/vttablet-down.sh index 174592c296e..570413d5cbe 100755 --- a/examples/kubernetes/vttablet-down.sh +++ b/examples/kubernetes/vttablet-down.sh @@ -28,9 +28,11 @@ for shard in `seq 1 $num_shards`; do printf -v alias '%s-%010d' $cell $uid if [ -n "$server" ]; then + set +e echo "Removing tablet $alias from Vitess topology..." vtctlclient -server $server ScrapTablet -force $alias vtctlclient -server $server DeleteTablet $alias + set -e fi echo "Deleting pod for tablet $alias..." diff --git a/examples/kubernetes/vttablet-pod-template.yaml b/examples/kubernetes/vttablet-pod-template.yaml index a751000e232..3c0131bc00a 100644 --- a/examples/kubernetes/vttablet-pod-template.yaml +++ b/examples/kubernetes/vttablet-pod-template.yaml @@ -17,6 +17,11 @@ spec: mountPath: /dev/log - name: vtdataroot mountPath: /vt/vtdataroot + - name: certs + readOnly: true + # Mount root certs from the host OS into the location + # expected for our container OS (Debian): + mountPath: /etc/ssl/certs/ca-certificates.crt resources: limits: memory: "1Gi" @@ -27,20 +32,10 @@ spec: - >- set -e - mysql_socket="$VTDATAROOT/{{tablet_subdir}}/mysql.sock" - mkdir -p $VTDATAROOT/tmp chown -R vitess /vt - while [ ! -e $mysql_socket ]; do - echo "[$(date)] waiting for $mysql_socket" ; - sleep 1 ; - done - - su -p -s /bin/bash -c "mysql -u vt_dba -S $mysql_socket - -e 'CREATE DATABASE IF NOT EXISTS vt_{{keyspace}}'" vitess - su -p -s /bin/bash -c "/vt/bin/vttablet -topo_implementation etcd -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT @@ -55,6 +50,7 @@ spec: -init_keyspace {{keyspace}} -init_shard {{shard}} -target_tablet_type {{tablet_type}} + -health_check_interval 5s -mysqlctl_socket $VTDATAROOT/mysqlctl.sock -db-config-app-uname vt_app -db-config-app-dbname vt_{{keyspace}} @@ -70,7 +66,8 @@ spec: -db-config-filtered-charset utf8 -enable-rowcache -rowcache-bin /usr/bin/memcached - -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock" vitess + -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock + -restore_from_backup {{backup_flags}}" vitess - name: mysql image: vitess/lite volumeMounts: @@ -115,4 +112,12 @@ spec: hostPath: {path: /dev/log} - name: vtdataroot {{vtdataroot_volume}} + - name: certs + # Uncomment one of the following lines to configure the location + # of the root certificates file on your host OS. We need this so + # we can import it into the container OS. + # If your host OS is Fedora/RHEL: + #hostPath: {path: /etc/pki/tls/certs/ca-bundle.crt} + # If your host OS is Debian/Ubuntu/Gentoo: + hostPath: {path: /etc/ssl/certs/ca-certificates.crt} diff --git a/examples/kubernetes/vttablet-up.sh b/examples/kubernetes/vttablet-up.sh index 378cc2cb387..31543c82630 100755 --- a/examples/kubernetes/vttablet-up.sh +++ b/examples/kubernetes/vttablet-up.sh @@ -25,11 +25,12 @@ if [ -n "$VTDATAROOT_VOLUME" ]; then fi uid_base=$UID_BASE +indices=${TASKS:-`seq 0 $(($TABLETS_PER_SHARD-1))`} for shard in $(echo $SHARDS | tr "," " "); do cell_index=0 for cell in `echo $CELLS | tr ',' ' '`; do echo "Creating $keyspace.shard-$shard pods in cell $CELL..." - for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do + for uid_index in $indices; do uid=$[$uid_base + $uid_index + $cell_index] printf -v alias '%s-%010d' $cell $uid printf -v tablet_subdir 'vt_%010d' $uid @@ -47,7 +48,7 @@ for shard in $(echo $SHARDS | tr "," " "); do # Expand template variables sed_script="" - for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir vtdataroot_volume tablet_type; do + for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir vtdataroot_volume tablet_type backup_flags; do sed_script+="s,{{$var}},${!var},g;" done diff --git a/examples/local/vtctld-up.sh b/examples/local/vtctld-up.sh index 6932df6cf95..5c422f1a602 100755 --- a/examples/local/vtctld-up.sh +++ b/examples/local/vtctld-up.sh @@ -23,6 +23,8 @@ $VTROOT/bin/vtctld -debug -templates $VTTOP/go/cmd/vtctld/templates \ -tablet_protocol grpc \ -tablet_manager_protocol grpc \ -service_map 'bsonrpc-vt-vtctl' \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ -log_dir $VTDATAROOT/tmp -port $port > $VTDATAROOT/tmp/vtctld.out 2>&1 & disown -a diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh index 1e6a1998644..047820044a3 100755 --- a/examples/local/vttablet-up.sh +++ b/examples/local/vttablet-up.sh @@ -92,6 +92,7 @@ for uid_index in $uids; do -init_keyspace $keyspace \ -init_shard $shard \ -target_tablet_type $tablet_type \ + -health_check_interval 5s \ -enable-rowcache \ -rowcache-bin $memcached_path \ -rowcache-socket $VTDATAROOT/$tablet_dir/memcache.sock \