Skip to content

Commit

Permalink
chore: Update to debian bookworm / Python 3.12 (#1567)
Browse files Browse the repository at this point in the history
* chore: Update to debian bookworm / Python 3.12
  • Loading branch information
jrconlin authored Jun 10, 2024
1 parent 6ea9c85 commit 8f9e1c2
Show file tree
Hide file tree
Showing 8 changed files with 108 additions and 86 deletions.
14 changes: 9 additions & 5 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ commands:
- run:
name: Setup python
command: |
sudo apt-get update && sudo apt-get install -y python3-dev python3-pip
sudo apt-get update && sudo apt-get install -y python3-dev python3-pip python3-venv
python3 -mvenv venv
. venv/bin/activate
pip3 install flake8 hawkauthlib konfig pyramid pyramid_hawkauth requests simplejson unittest2 WebTest WSGIProxy2
pip3 install -r requirements.txt
rust-check:
Expand All @@ -40,6 +42,7 @@ commands:
- run:
name: Core Python Checks
command: |
. venv/bin/activate
flake8 syncserver/src/tokenserver
flake8 tools/integration_tests
flake8 tools/tokenserver
Expand Down Expand Up @@ -116,6 +119,7 @@ commands:
- run:
name: Tokenserver scripts tests
command: >
. venv/bin/activate &&
pip3 install -r tools/tokenserver/requirements.txt &&
python3 tools/tokenserver/run_tests.py
environment:
Expand Down Expand Up @@ -161,7 +165,7 @@ commands:
jobs:
checks:
docker:
- image: cimg/rust:1.75.0
- image: cimg/rust:1.78.0 # RUST_VER
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
Expand All @@ -178,7 +182,7 @@ jobs:

build-and-test:
docker:
- image: cimg/rust:1.75.0
- image: cimg/rust:1.78.0 # RUST_VER
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
Expand Down Expand Up @@ -214,7 +218,7 @@ jobs:
#- save-sccache-cache
build-mysql-image:
docker:
- image: cimg/rust:1.75.0
- image: cimg/rust:1.78.0 # RUST_VER
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
Expand Down Expand Up @@ -245,7 +249,7 @@ jobs:

build-spanner-image:
docker:
- image: cimg/rust:1.75.0
- image: cimg/rust:1.78.0 # RUST_VER
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
Expand Down
33 changes: 21 additions & 12 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml
FROM docker.io/lukemathwalker/cargo-chef:0.1.62-rust-1.75-bullseye as chef
# NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml # RUST_VER
FROM docker.io/lukemathwalker/cargo-chef:0.1.67-rust-1.78-bookworm as chef
WORKDIR /app

FROM chef AS planner
Expand All @@ -14,7 +14,7 @@ RUN \
# Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs
# which wants the mariadb
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
apt-get -q update && \
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake

Expand All @@ -31,14 +31,16 @@ COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME
RUN \
# Fetch and load the MySQL public key
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
# mysql_pubkey.asc from:
# https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html
# related:
# https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup
apt-get -q update && \
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \
pip3 install -r requirements.txt && \
apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go pkg-config python3-dev python3-pip python3-setuptools python3-wheel python3-venv && \
python3 -mvenv /app/venv && \
. /app/venv/bin/activate && \
pip3 install -r /app/requirements.txt && \
rm -rf /var/lib/apt/lists/*

ENV PATH=$PATH:/root/.cargo/bin
Expand All @@ -49,7 +51,7 @@ RUN \
cargo install --path ./syncserver --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --features=py_verifier --locked --root /app && \
if [ "$DATABASE_BACKEND" = "spanner" ] ; then cargo install --path ./syncstorage-spanner --locked --root /app --bin purge_ttl ; fi

FROM docker.io/library/debian:bullseye-slim
FROM docker.io/library/debian:bookworm-slim
WORKDIR /app
COPY --from=builder /app/requirements.txt /app
# Due to a build error that occurs with the Python cryptography package, we
Expand All @@ -69,16 +71,18 @@ RUN \
apt-get -q update && \
# and ca-certificates needed for https://repo.mysql.com
apt-get install -y gnupg ca-certificates wget && \
echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
# Fetch and load the MySQL public key
wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
# update again now that we trust repo.mysql.com
apt-get -q update && \
# Fetch and load the MySQL public key
apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \
apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 pkg-config python3-dev python3-pip python3-setuptools python3-wheel python3-venv cargo curl jq && \
# The python3-cryptography debian package installs version 2.6.1, but we
# we want to use the version specified in requirements.txt. To do this,
# we have to remove the python3-cryptography package here.
apt-get -q remove -y python3-cryptography && \
python3 -mvenv /app/venv && \
. /app/venv/bin/activate && \
pip3 install -r /app/requirements.txt && \
rm -rf /var/lib/apt/lists/*

Expand All @@ -88,11 +92,16 @@ COPY --from=builder /app/tools/spanner /app/tools/spanner
COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
COPY --from=builder /app/tools/tokenserver /app/tools/tokenserver
COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
COPY --from=builder /app/scripts/start_mock_fxa_server.sh /app/scripts/start_mock_fxa_server.sh
COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl

RUN chmod +x /app/scripts/prepare-spanner.sh
RUN pip3 install -r /app/tools/integration_tests/requirements.txt
RUN pip3 install -r /app/tools/tokenserver/requirements.txt
RUN \
. /app/venv/bin/activate && \
pip3 install -r /app/tools/integration_tests/requirements.txt
RUN \
. /app/venv/bin/activate && \
pip3 install -r /app/tools/tokenserver/requirements.txt

USER app:app

Expand Down
3 changes: 2 additions & 1 deletion docker-compose.e2e.mysql.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ services:
entrypoint: >
/bin/sh -c "
sleep 15;
. /app/venv/bin/activate;
/app/bin/syncserver;
"
mysql-e2e-tests:
Expand Down Expand Up @@ -43,5 +44,5 @@ services:
TOKENSERVER_HOST: http://localhost:8000
entrypoint: >
/bin/sh -c "
sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
sleep 28; . /app/venv/bin/activate; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
"
3 changes: 2 additions & 1 deletion docker-compose.e2e.spanner.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ services:
entrypoint: >
/bin/sh -c "
sleep 15;
. /app/venv/bin/activate;
/app/bin/syncserver;
"
spanner-e2e-tests:
Expand Down Expand Up @@ -44,5 +45,5 @@ services:
TOKENSERVER_HOST: http://localhost:8000
entrypoint: >
/bin/sh -c "
sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
sleep 28; . /app/venv/bin/activate; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
"
4 changes: 3 additions & 1 deletion docker-compose.mysql.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ services:
restart: always
ports:
- "3306"
command: --explicit_defaults_for_timestamp
environment:
#MYSQL_RANDOM_ROOT_PASSWORD: yes
MYSQL_ROOT_PASSWORD: random
Expand All @@ -31,6 +32,7 @@ services:
restart: always
ports:
- "3306"
command: --explicit_defaults_for_timestamp
environment:
#MYSQL_RANDOM_ROOT_PASSWORD: yes
MYSQL_ROOT_PASSWORD: random
Expand All @@ -41,7 +43,7 @@ services:
mock-fxa-server:
image: app:build
restart: "no"
entrypoint: python3 /app/tools/integration_tests/tokenserver/mock_fxa_server.py
entrypoint: "sh scripts/start_mock_fxa_server.sh"
environment:
MOCK_FXA_SERVER_HOST: 0.0.0.0
MOCK_FXA_SERVER_PORT: 6000
Expand Down
130 changes: 65 additions & 65 deletions docker-compose.spanner.yaml
Original file line number Diff line number Diff line change
@@ -1,69 +1,69 @@
version: '3'
version: "3"
services:
sync-db:
# Getting sporadic errors in spanner.
# These errors are "INTERNAL: ZETASQL_RET_CHECK failure"
# in the `backend/query/query_engine.cc` file for spanner.
# These result in a 500 error, which causes the test to fail.
# I believe come from the SQL parser, but am not sure. I am
# unable to produce these errors locally, and am using the cited
# version.
image: gcr.io/cloud-spanner-emulator/emulator:1.5.13
ports:
- "9010:9010"
- "9020:9020"
environment:
PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
sync-db-setup:
image: app:build
depends_on:
- sync-db
restart: "no"
entrypoint: "/app/scripts/prepare-spanner.sh"
environment:
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9020
tokenserver-db:
image: docker.io/library/mysql:5.7
volumes:
- tokenserver_db_data:/var/lib/mysql
restart: always
ports:
- "3306"
environment:
#MYSQL_RANDOM_ROOT_PASSWORD: yes
MYSQL_ROOT_PASSWORD: random
MYSQL_DATABASE: tokenserver
MYSQL_USER: test
MYSQL_PASSWORD: test
mock-fxa-server:
image: app:build
restart: "no"
entrypoint: "python3 /app/tools/integration_tests/tokenserver/mock_fxa_server.py"
environment:
MOCK_FXA_SERVER_HOST: 0.0.0.0
MOCK_FXA_SERVER_PORT: 6000
syncserver:
# NOTE: The naming in the rest of this repository has been updated to reflect the fact
# that Syncstorage and Tokenserver are now part of one repository/server called
# "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for
# backwards compatibility with previous Docker images.
image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest}
restart: always
ports:
- "8000:8000"
depends_on:
- sync-db-setup
environment:
SYNC_HOST: 0.0.0.0
SYNC_MASTER_SECRET: secret0
SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010
SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
SYNC_TOKENSERVER__RUN_MIGRATIONS: "true"
sync-db:
# Getting sporadic errors in spanner.
# These errors are "INTERNAL: ZETASQL_RET_CHECK failure"
# in the `backend/query/query_engine.cc` file for spanner.
# These result in a 500 error, which causes the test to fail.
# I believe come from the SQL parser, but am not sure. I am
# unable to produce these errors locally, and am using the cited
# version.
image: gcr.io/cloud-spanner-emulator/emulator:1.5.13
ports:
- "9010:9010"
- "9020:9020"
environment:
PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
sync-db-setup:
image: app:build
depends_on:
- sync-db
restart: "no"
entrypoint: "/app/scripts/prepare-spanner.sh"
environment:
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9020
tokenserver-db:
image: docker.io/library/mysql:5.7
volumes:
- tokenserver_db_data:/var/lib/mysql
restart: always
ports:
- "3306"
environment:
#MYSQL_RANDOM_ROOT_PASSWORD: yes
MYSQL_ROOT_PASSWORD: random
MYSQL_DATABASE: tokenserver
MYSQL_USER: test
MYSQL_PASSWORD: test
mock-fxa-server:
image: app:build
restart: "no"
entrypoint: "sh /app/scripts/start_mock_fxa_server.sh"
environment:
MOCK_FXA_SERVER_HOST: 0.0.0.0
MOCK_FXA_SERVER_PORT: 6000
syncserver:
# NOTE: The naming in the rest of this repository has been updated to reflect the fact
# that Syncstorage and Tokenserver are now part of one repository/server called
# "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for
# backwards compatibility with previous Docker images.
image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest}
restart: always
ports:
- "8000:8000"
depends_on:
- sync-db-setup
environment:
SYNC_HOST: 0.0.0.0
SYNC_MASTER_SECRET: secret0
SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010
SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver
SYNC_TOKENSERVER__RUN_MIGRATIONS: "true"

volumes:
tokenserver_db_data:
tokenserver_db_data:

# Application runs off of port 8000.
# you can test if it's available with
# curl "http://localhost:8000/__heartbeat__"
# Application runs off of port 8000.
# you can test if it's available with
# curl "http://localhost:8000/__heartbeat__"
5 changes: 5 additions & 0 deletions scripts/start_mock_fxa_server.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/sh

. /app/venv/bin/activate
python3 /app/tools/integration_tests/tokenserver/mock_fxa_server.py

2 changes: 1 addition & 1 deletion tools/spanner/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3.11-bullseye
FROM python:3.12-bookworm

COPY purge_ttl.py count_expired_rows.py count_users.py requirements.txt /app/

Expand Down

0 comments on commit 8f9e1c2

Please sign in to comment.