feat(upgrade): Add automatic Snuba migration for upgrades (getsentry#292)

Upgrades existing events from last `$SENTRY_EVENT_RETENTION_DAYS`
to Snuba automatically. Relies on getsentry/sentry#15934.

Author: BYK

install.sh

@@ -16,7 +16,7 @@ cleanup () {
     return 0;
   fi
   echo "Cleaning up..."
-  docker-compose down &> /dev/null
+  docker-compose stop &> /dev/null
   DID_CLEAN_UP=1
 }
 trap cleanup ERR INT TERM
@@ -55,30 +55,8 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then
     exit -1
 fi
 
-# Very naively check whether there's an existing sentry-postgres volume and the PG version in it
-if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then
-    # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume
-    docker run --rm \
-    -v sentry-postgres:/var/lib/postgresql/9.5/data \
-    -v sentry-postgres-new:/var/lib/postgresql/9.6/data \
-    tianon/postgres-upgrade:9.5-to-9.6
-
-    # Get rid of the old volume as we'll rename the new one to that
-    docker volume rm sentry-postgres
-    docker volume create --name sentry-postgres
-    # There's no rename volume in Docker so copy the contents from old to new name
-    # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6`
-    # doesn't do that automatically.
-    docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \
-     "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf"
-    # Finally, remove the new old volume as we are all in sentry-postgres now
-    docker volume rm sentry-postgres-new
-fi
-
-echo ""
-ensure_file_from_example $SENTRY_CONFIG_PY
-ensure_file_from_example $SENTRY_CONFIG_YML
-ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS
+# Ensure nothing is working while we install/update
+docker-compose stop
 
 echo ""
 echo "Creating volumes for persistent storage..."
@@ -90,6 +68,11 @@ echo "Created $(docker volume create --name=sentry-kafka)."
 echo "Created $(docker volume create --name=sentry-clickhouse)."
 echo "Created $(docker volume create --name=sentry-symbolicator)."
 
+echo ""
+ensure_file_from_example $SENTRY_CONFIG_PY
+ensure_file_from_example $SENTRY_CONFIG_YML
+ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS
+
 echo ""
 echo "Generating secret key..."
 # This is to escape the secret key to be used in sed below
@@ -106,6 +89,26 @@ docker-compose build --force-rm
 echo ""
 echo "Docker images built."
 
+# Very naively check whether there's an existing sentry-postgres volume and the PG version in it
+if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then
+    # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume
+    docker run --rm \
+    -v sentry-postgres:/var/lib/postgresql/9.5/data \
+    -v sentry-postgres-new:/var/lib/postgresql/9.6/data \
+    tianon/postgres-upgrade:9.5-to-9.6
+
+    # Get rid of the old volume as we'll rename the new one to that
+    docker volume rm sentry-postgres
+    docker volume create --name sentry-postgres
+    # There's no rename volume in Docker so copy the contents from old to new name
+    # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6`
+    # doesn't do that automatically.
+    docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \
+     "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf"
+    # Finally, remove the new old volume as we are all in sentry-postgres now
+    docker volume rm sentry-postgres-new
+fi
+
 echo ""
 echo "Setting up database..."
 if [ $CI ]; then
@@ -120,6 +123,13 @@ else
   docker-compose run --rm web upgrade
 fi
 
+SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l")
+if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then
+  echo "Migrating file storage..."
+  docker run --rm -v sentry-data:/data alpine ash -c \
+    "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files"
+fi
+
 echo "Boostrapping Snuba..."
 docker-compose up -d kafka redis clickhouse
 until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do
@@ -130,12 +140,12 @@ until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --que
 done;
 echo ""
 
-SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l")
-if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then
-  echo "Migrating file storage..."
-  docker run --rm -v sentry-data:/data alpine ash -c \
-    "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files"
-fi
+set -o allexport
+source .env
+set +o allexport
+echo "Migrating old events for the last $SENTRY_EVENT_RETENTION_DAYS days..."
+docker-compose run --rm web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS
+echo ""
 
 cleanup