Skip to content

Commit

Permalink
Run separate web server, registry watcher and build servers in docker…
Browse files Browse the repository at this point in the history
…-compose
  • Loading branch information
Nemo157 committed Nov 18, 2023
1 parent 801e478 commit 0366ef5
Show file tree
Hide file tree
Showing 7 changed files with 202 additions and 156 deletions.
48 changes: 8 additions & 40 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,8 @@ jobs:

- name: Launch postgres
run: |
cp .env.sample .env
mkdir -p ${DOCSRS_PREFIX}/public-html
docker compose up -d db
# Give the database enough time to start up
sleep 5
# Make sure the database is actually working
psql "${DOCSRS_DATABASE_URL}"
touch .docker.env
docker compose up --wait --wait-timeout 30 db
- name: run database migrations
run: cargo run -- database migrate
Expand All @@ -60,9 +55,6 @@ jobs:
--check \
-- --all-targets --all-features
- name: Clean up the database
run: docker compose down --volumes

build:
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -110,13 +102,8 @@ jobs:

- name: Launch postgres and min.io
run: |
cp .env.sample .env
mkdir -p ${DOCSRS_PREFIX}/public-html
docker compose up -d db s3
# Give the database enough time to start up
sleep 5
# Make sure the database is actually working
psql "${DOCSRS_DATABASE_URL}"
touch .docker.env
docker compose up --wait --wait-timeout 30 db s3
- name: run tests
shell: bash
Expand All @@ -127,9 +114,6 @@ jobs:
$f || exit 1
done
- name: Clean up the database
run: docker compose down --volumes

GUI_test:
runs-on: ubuntu-latest
needs: build
Expand All @@ -145,20 +129,12 @@ jobs:

- name: Launch postgres and min.io
run: |
cp .env.sample .env
mkdir -p ${DOCSRS_PREFIX}/public-html
docker compose up -d db s3
# Give the database enough time to start up
sleep 5
# Make sure the database is actually working
psql "${DOCSRS_DATABASE_URL}"
touch .docker.env
docker compose up --wait --wait-timeout 30 db s3
- name: Run GUI tests
run: ./dockerfiles/run-gui-tests.sh

- name: Clean up the database
run: docker compose down --volumes

build_tests:
runs-on: ubuntu-latest
needs: build
Expand All @@ -174,13 +150,8 @@ jobs:

- name: Launch postgres and min.io
run: |
cp .env.sample .env
mkdir -p ${DOCSRS_PREFIX}/public-html
docker compose up -d db s3
# Give the database enough time to start up
sleep 5
# Make sure the database is actually working
psql "${DOCSRS_DATABASE_URL}"
touch .docker.env
docker compose up --wait --wait-timeout 30 db s3
- name: slow tests
env:
Expand All @@ -193,9 +164,6 @@ jobs:
$f --ignored --test-threads=1 || exit 1
done
- name: Clean up the database
run: docker compose down --volumes

fmt:
name: Rustfmt
runs-on: ubuntu-latest
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,7 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Build the Docker image
run: docker build -t docs-rs -f dockerfiles/Dockerfile .
- run: docker build --target web-server -f dockerfiles/Dockerfile .
- run: docker build --target build-server -f dockerfiles/Dockerfile .
- run: docker build --target registry-watcher -f dockerfiles/Dockerfile .
- run: docker build --target cli -f dockerfiles/Dockerfile .
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/ignored
/.env
/.docker.env
/src/web/badge/Cargo.lock
target
*.css
Expand Down
32 changes: 18 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index
# Builds the docs.rs binary
cargo build
# Start the external services.
docker compose up -d db s3
docker compose up --wait db s3
# anything that doesn't run via docker-compose needs the settings defined in
# .env. Either via `. ./.env` as below, or via any dotenv shell integration.
. ./.env
Expand Down Expand Up @@ -115,21 +115,25 @@ which uses docker-compose for the web server as well.
This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes -
but makes sure that you're in a known environment so you should have fewer problems getting started.

You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
You'll need to `touch .docker.env` first, this file can have any environment
variable overrides you want to use in docker containers.

You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):

```sh
# update the toolchain
docker compose run --rm builder-a build update-toolchain
# run a build for a single crate
docker compose run web build crate regex 1.3.1
# or build essential files
docker compose run web build add-essential-files
# rebuild the web container when you changed code.
docker compose up -d web --build
docker compose run --rm builder-a build crate regex 1.3.1
# rebuild containers when you changed code.
docker compose up --wait --build
```

You can also run other commands like the setup above from within the container:
You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container:

```sh
docker compose run --rm cli database migrate
docker compose run --rm cli build update-toolchain
docker compose run --rm cli queue add regex 1.3.1
```

Note that running tests is not supported when using pure docker-compose.
Expand All @@ -151,7 +155,7 @@ Three services are defined:

#### Rebuilding Containers

To rebuild the site, run `docker compose build`.
To rebuild the site, run `docker compose --profile all build`.
Note that docker-compose caches the build even if you change the source code,
so this will be necessary anytime you make changes.

Expand All @@ -172,7 +176,7 @@ This is probably because you have `git.autocrlf` set to true,

##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds.

You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above.
You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above.

See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly.

Expand Down Expand Up @@ -200,11 +204,11 @@ cargo run -- start-web-server
```sh
# Builds <CRATE_NAME> <CRATE_VERSION> and adds it into database
# This is the main command to build and add a documentation into docs.rs.
# For example, `docker compose run web build crate regex 1.1.6`
# For example, `docker compose run --rm builder-a build crate regex 1.1.6`
cargo run -- build crate <CRATE_NAME> <CRATE_VERSION>

# alternatively, via the web container
docker compose run web build crate <CRATE_NAME> <CRATE_VERSION>
# alternatively, within docker-compose containers
docker compose run --rm builder-a build crate <CRATE_NAME> <CRATE_VERSION>

# Builds every crate on crates.io and adds them into database
# (beware: this may take months to finish)
Expand Down
147 changes: 109 additions & 38 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,35 +1,118 @@
version: "3"
version: "3.4"

x-healthcheck: &healthcheck-interval
interval: 1s
timeout: 1s
start_period: 10s
# TODO: https://github.com/docker/compose/issues/10461
# interval: 10s
# start_interval: 1s

x-environment: &environment
RUST_BACKTRACE: true

DOCSRS_PREFIX: /opt/docsrs/prefix

DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
DOCSRS_MIN_POOL_SIZE: 2
DOCSRS_MAX_POOL_SIZE: 10

DOCSRS_STORAGE_BACKEND: s3

S3_ENDPOINT: http://s3:9000
AWS_ACCESS_KEY_ID: cratesfyi
AWS_SECRET_ACCESS_KEY: secret_key

DOCSRS_RENDER_THREADS: 2

DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
DOCSRS_DOCKER: true
DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro
DOCSRS_BUILD_CPU_LIMIT: 2
DOCSRS_INCLUDE_DEFAULT_TARGETS: false

x-builder: &builder
build:
context: .
dockerfile: ./dockerfiles/Dockerfile
target: build-server
depends_on:
- db
- s3
environment: *environment
env_file:
- .docker.env
healthcheck:
<< : *healthcheck-interval
test: curl --silent --fail localhost:3000/about/metrics

services:
web:
build:
context: .
dockerfile: ./dockerfiles/Dockerfile
target: web-server
platform: "linux/amd64"
depends_on:
- db
- s3
ports:
- "3000:3000"
# for metrics
expose: ["3000"]
- "3000:80"
environment: *environment
env_file:
- .docker.env
healthcheck:
<< : *healthcheck-interval
test: curl --silent --fail localhost:80/about/metrics

# Include the registry watcher with `docker compose --profile watch up -d`
registry-watcher:
build:
context: .
dockerfile: ./dockerfiles/Dockerfile
target: registry-watcher
platform: "linux/amd64"
depends_on:
- db
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- ".rustwide-docker:/opt/docsrs/rustwide"
- "cratesio-index:/opt/docsrs/prefix/crates.io-index"
environment:
DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
DOCSRS_STORAGE_BACKEND: s3
S3_ENDPOINT: http://s3:9000
AWS_ACCESS_KEY_ID: cratesfyi
AWS_SECRET_ACCESS_KEY: secret_key
environment: *environment
env_file:
- .env
- .docker.env
profiles:
- watch
- all
healthcheck:
test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"]
interval: 10s
timeout: 5s
retries: 10
<< : *healthcheck-interval
test: curl --silent --fail localhost:3000/about/metrics

builder-a:
<< : *builder
volumes:
- ".rustwide-docker/builder-a:/opt/docsrs/rustwide"
- "/var/run/docker.sock:/var/run/docker.sock"

builder-b:
<< : *builder
volumes:
- ".rustwide-docker/builder-b:/opt/docsrs/rustwide"
- "/var/run/docker.sock:/var/run/docker.sock"

cli:
build:
context: .
dockerfile: ./dockerfiles/Dockerfile
target: cli
depends_on:
- db
- s3
volumes:
- ".rustwide-docker/cli:/opt/docsrs/rustwide"
environment: *environment
env_file:
- .docker.env
profiles:
- all

db:
build:
Expand All @@ -44,10 +127,8 @@ services:
# Use a non-standard port on the host to avoid conflicting with existing postgres servers
- "127.0.0.1:15432:5432"
healthcheck:
test: ["CMD", "pg_isready", "--username", "cratesfyi"]
interval: 10s
timeout: 5s
retries: 10
<< : *healthcheck-interval
test: pg_isready --username cratesfyi

s3:
image: minio/minio
Expand All @@ -65,17 +146,8 @@ services:
MINIO_ROOT_USER: cratesfyi
MINIO_ROOT_PASSWORD: secret_key
healthcheck:
test:
[
"CMD",
"curl",
"--silent",
"--fail",
"localhost:9000/minio/health/ready",
]
interval: 10s
timeout: 5s
retries: 10
<< : *healthcheck-interval
test: mc ready local

prometheus:
build:
Expand All @@ -84,11 +156,8 @@ services:
ports:
- "127.0.0.1:9090:9090"
healthcheck:
test:
["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"]
interval: 10s
timeout: 5s
retries: 10
<< : *healthcheck-interval
test: promtool check healthy

gui_tests:
build:
Expand All @@ -99,6 +168,8 @@ services:
- "host.docker.internal:host-gateway"
volumes:
- "${PWD}:/build/out"
profiles:
- all

volumes:
postgres-data: {}
Expand Down
Loading

0 comments on commit 0366ef5

Please sign in to comment.