feat(sentry10): Make on-premise work for Sentry 10 (getsentry#220)

.env

@@ -0,0 +1 @@
+SENTRY_EVENT_RETENTION_DAYS=90

.gitignore

@@ -68,9 +68,11 @@ target/
 # https://docs.docker.com/compose/extends/
 docker-compose.override.yml
 
-# env config
-.env
-
 *.tar
 data/
 .vscode/tags
+
+# custom Sentry config
+sentry/sentry.conf.py
+sentry/config.yml
+sentry/requirements.txt

.travis.yml

@@ -2,12 +2,17 @@ language: bash
 services: docker
 
 env:
-  - SENTRY_IMAGE=sentry:9.1.2
-  - SENTRY_IMAGE=getsentry/sentry:latest
+  - DOCKER_COMPOSE_VERSION=1.24.1
+
+before_install:
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 
 script:
   - ./install.sh
-  - docker-compose run --rm web createuser --superuser --email test@sentry.io --password test123TEST
+  - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST
   - docker-compose up -d
   - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done'
   - ./test.sh

README.md

@@ -1,30 +1,38 @@
-# Sentry On-Premise [![Build Status][build-status-image]][build-status-url]
+# Sentry 10 On-Premise BETA [![Build Status][build-status-image]][build-status-url]
 
 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/).
 
+**NOTE:** If you are not installing Sentry from scratch, visit [On-Premise Stable for Sentry 9.1.2](https://github.com/getsentry/onpremise/tree/stable) as this version is not fully backward compatible.
+
 ## Requirements
 
  * Docker 17.05.0+
- * Compose 1.17.0+
+ * Compose 1.19.0+
 
 ## Minimum Hardware Requirements:
 
- * You need at least 3GB RAM
+ * You need at least 2400MB RAM
 
 ## Setup
 
 To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out.
 
-There may need to be modifications to the included `docker-compose.yml` file to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script.
+There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script.
 
 The recommended way to customize your configuration is using the files below, in that order:
 
  * `config.yml`
  * `sentry.conf.py`
  * `.env` w/ environment variables
 
+We currently support a very minimal set of environment variables to promote other means of configuration.
+
 If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service!
 
+## Event Retention
+
+Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file.
+
 ## Securing Sentry with SSL/TLS
 
 If you'd like to protect your Sentry install with SSL/TLS, there are

cron/Dockerfile

@@ -0,0 +1,6 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+RUN apt-get update && apt-get install -y --no-install-recommends cron && \
+    rm -r /var/lib/apt/lists/*
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

cron/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Prior art:
+# - https://git.io/fjNOg
+# - https://blog.knoldus.com/running-a-cron-job-in-docker-container/
+
+declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env
+
+{ for cron_job in "$@"; do echo -e "SHELL=/bin/bash
+BASH_ENV=/container.env
+${cron_job} > /proc/1/fd/1 2>/proc/1/fd/2"; done } \
+    | sed --regexp-extended 's/\\(.)/\1/g' \
+    | crontab -
+crontab -l
+exec cron -f -l -L 15

docker-compose.yml

@@ -1,67 +1,159 @@
-# NOTE: This docker-compose.yml is meant to be just an example of how
-# you could accomplish this on your own. It is not intended to work in
-# all use-cases and must be adapted to fit your needs. This is merely
-# a guideline.
-
-# See docs.getsentry.com/on-premise/server/ for full
-# instructions
-
 version: '3.4'
-
-x-defaults: &defaults
+x-restart-policy: &restart_policy
   restart: unless-stopped
+x-sentry-defaults: &sentry_defaults
+  << : *restart_policy
   build:
-    context: .
+    context: ./sentry
+    args:
+      - SENTRY_IMAGE
+  image: sentry-onpremise-local
   depends_on:
     - redis
     - postgres
     - memcached
     - smtp
-  env_file: .env
+    - snuba-api
+    - snuba-consumer
+    - snuba-replacer
+    - symbolicator
+    - kafka
   environment:
-    SENTRY_MEMCACHED_HOST: memcached
-    SENTRY_REDIS_HOST: redis
-    SENTRY_POSTGRES_HOST: postgres
-    SENTRY_EMAIL_HOST: smtp
+    SNUBA: 'http://snuba-api:1218'
   volumes:
-    - sentry-data:/var/lib/sentry/files
-
-
+    - 'sentry-data:/var/lib/sentry/files'
+x-snuba-defaults: &snuba_defaults
+  << : *restart_policy
+  depends_on:
+    - redis
+    - clickhouse
+    - kafka
+  image: 'getsentry/snuba:latest'
+  environment:
+    SNUBA_SETTINGS: docker
+    CLICKHOUSE_HOST: clickhouse
+    DEFAULT_BROKERS: 'kafka:9093'
+    REDIS_HOST: redis
+    # TODO: Remove these after getsentry/snuba#353
+    UWSGI_MAX_REQUESTS: '10000'
+    UWSGI_DISABLE_LOGGING: 'true'
+    UWSGI_ENABLE_THREADS: 'true'
+    UWSGI_DIE_ON_TERM: 'true'
+    UWSGI_NEED_APP: 'true'
+    UWSGI_IGNORE_SIGPIPE: 'true'
+    UWSGI_IGNORE_WRITE_ERRORS: 'true'
+    UWSGI_DISABLE_WRITE_EXCEPTION:  'true'
 services:
   smtp:
-    restart: unless-stopped
+    << : *restart_policy
     image: tianon/exim4
-
   memcached:
-    restart: unless-stopped
-    image: memcached:1.5-alpine
-
+    << : *restart_policy
+    image: 'memcached:1.5-alpine'
   redis:
-    restart: unless-stopped
-    image: redis:3.2-alpine
-
+    << : *restart_policy
+    image: 'redis:5.0-alpine'
+    volumes:
+      - 'sentry-redis:/data'
   postgres:
-    restart: unless-stopped
-    image: postgres:9.5
+    << : *restart_policy
+    image: 'postgres:9.6'
+    volumes:
+      - 'sentry-postgres:/var/lib/postgresql/data'
+  zookeeper:
+    << : *restart_policy
+    image: 'confluentinc/cp-zookeeper:5.1.2'
+    environment:
+      ZOOKEEPER_CLIENT_PORT: '2181'
+      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'ERROR'
+      ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'ERROR'
+    volumes:
+      - 'sentry-zookeeper:/var/lib/zookeeper'
+  kafka:
+    << : *restart_policy
+    depends_on:
+      - zookeeper
+    image: 'confluentinc/cp-kafka:5.1.2'
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENERS: 'INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092'
+      KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka:9093,EXTERNAL://kafka:9092'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT'
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
+      KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN'
+      KAFKA_LOG4J_ROOT_LOGLEVEL: 'ERROR'
+      KAFKA_TOOLS_LOG4J_LOGLEVEL: 'ERROR'
+    volumes:
+      - 'sentry-kafka:/var/lib/kafka'
+  clickhouse:
+    << : *restart_policy
+    image: 'yandex/clickhouse-server:19.4'
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - 'sentry-clickhouse:/var/lib/clickhouse'
+  snuba-api:
+    << : *snuba_defaults
+  snuba-consumer:
+    << : *snuba_defaults
+    command: consumer --auto-offset-reset=latest --max-batch-time-ms 750
+  snuba-replacer:
+    << : *snuba_defaults
+    command: replacer --auto-offset-reset=latest --max-batch-size 3
+  snuba-cleanup:
+    << : *snuba_defaults
+    image: snuba-cleanup-onpremise-local
+    build:
+      context: ./cron
+      args:
+        BASE_IMAGE: 'getsentry/snuba:latest'
+    command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"'
+  symbolicator:
+    << : *restart_policy
+    image: us.gcr.io/sentryio/symbolicator:latest
     volumes:
-      - sentry-postgres:/var/lib/postgresql/data
-
+      - 'sentry-symbolicator:/data'
+    command: run
+  symbolicator-cleanup:
+    image: symbolicator-cleanup-onpremise-local
+    build:
+      context: ./cron
+      args:
+        BASE_IMAGE: 'us.gcr.io/sentryio/symbolicator:latest'
+    command: '"55 23 * * * gosu symbolicator symbolicator cleanup"'
   web:
-    <<: *defaults
+    << : *sentry_defaults
     ports:
-      - '9000:9000'
-
+      - '9000:9000/tcp'
   cron:
-    <<: *defaults
+    << : *sentry_defaults
     command: run cron
-
   worker:
-    <<: *defaults
+    << : *sentry_defaults
     command: run worker
-
-
+  sentry-cleanup:
+    << : *sentry_defaults
+    image: sentry-cleanup-onpremise-local
+    build:
+      context: ./cron
+      args:
+        BASE_IMAGE: 'sentry-onpremise-local'
+    command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"'
 volumes:
-    sentry-data:
-      external: true
-    sentry-postgres:
-      external: true
+  sentry-data:
+    external: true
+  sentry-postgres:
+    external: true
+  sentry-redis:
+    external: true
+  sentry-zookeeper:
+    external: true
+  sentry-kafka:
+    external: true
+  sentry-clickhouse:
+    external: true
+  sentry-symbolicator:
+    external: true