Add docker dev environment

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+# This should be kept in sync with the python version used in docker/Dockerfile and
+# docker/images/fakesentry/Dockerfile
+FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/python:3.11-bullseye@sha256:105bf6a63ab025206f019a371a735fec6553db0be520030c7a2fd0e002947232
+
+ARG userid=10001
+ARG groupid=10001
+
+WORKDIR /app
+
+# add a non-privileged user for installing and running the application
+# We use --non-unique in case $groupid/$userid collide with the existing "vscode" user.
+# useradd -g app --uid $userid --non-unique --shell /usr/sbin/nologin --create-home app && \
+RUN groupadd --gid $groupid --non-unique app && \
+    useradd -g app --uid $userid --non-unique --shell /bin/bash --create-home app && \
+    chown app:app /app/
+
+# Install Debian packages
+RUN apt-get update && \
+    apt-get install -y ripgrep tig
+
+# Install Python dependencies
+COPY requirements.txt /app/
+RUN pip install -U 'pip>=20' && \
+    pip install --no-cache-dir --no-deps --only-binary :all: -r requirements.txt && \
+    pip install --no-cache-dir ipython && \
+    pip check --disable-pip-version-check

.devcontainer/devcontainer.json

@@ -0,0 +1,28 @@
+{
+  "name": "Obs Common",
+  "dockerComposeFile": [
+    "../docker-compose.yml"
+  ],
+  "service": "devcontainer",
+  "runServices": [
+    "devcontainer"
+  ],
+  "shutdownAction": "none",
+  "workspaceFolder": "/app",
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "python.testing.unittestEnabled": false,
+        "python.testing.pytestEnabled": true,
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff"
+        }
+      },
+      "extensions": [
+        "charliermarsh.ruff"
+      ]
+    }
+  },
+  "remoteUser": "app",
+  "updateRemoteUserUID": false
+}

.dockerignore

@@ -0,0 +1,11 @@
+*.py[co]
+*.sw[po]
+.docker-build
+.DS_Store
+.env
+.pytest_cache
+.python-version
+build
+dist/
+obs_common.egg-info/
+venv/

.github/workflows/build.yaml

@@ -26,6 +26,7 @@ jobs:
         run: |
           python -m venv venv
           venv/bin/pip install -r requirements.txt
+          venv/bin/pip install -e . --no-deps
       - name: Verify requirements.txt
         if: ${{ matrix.python-version == '3.11' }}
         run: |
@@ -34,22 +35,18 @@ jobs:
       - name: Run lint check
         if: ${{ matrix.python-version == '3.11' }}
         run: |
-          venv/bin/ruff format --check obs_common tests
-          venv/bin/ruff check obs_common tests
+          PATH="venv/bin:$PATH" bin/lint.sh
       - name: Run tests
         env:
           SENTRY_DSN: http://public@localhost:8090/1
           STORAGE_EMULATOR_HOST: http://localhost:8001
+          PUBSUB_EMULATOR_HOST: localhost:5010
         run: |
-          docker compose up -d fakesentry gcs-emulator
-          venv/bin/pytest tests/
+          docker compose up -d fakesentry gcs-emulator pubsub
+          # Run outside docker because we are testing the matrix python version
+          PATH="venv/bin:$PATH" bin/test.sh
           # stop services immediate and ignore errors
           docker compose down -t0 || true
-      - name: License Check
-        if: ${{ matrix.python-version == '3.11' }}
-        run: |
-          venv/bin/pip install -e . --no-deps
-          venv/bin/license-check
 
   build-and-release:
     permissions:

.gitignore

@@ -1,6 +1,8 @@
 *.py[co]
 *.sw[po]
+.docker-build
 .DS_Store
+.env
 .pytest_cache
 .python-version
 build

Makefile

@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+# Include .env and export it so variables set in there are available
+# in Makefile.
+include .env
+export
+
+.DEFAULT_GOAL := help
+.PHONY: help
+help:
+	@echo "Usage: make RULE"
+	@echo ""
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \
+	  | grep -v grep \
+	    | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \
+	    | column -t  -s '|'
+	@echo ""
+	@echo "Adjust your .env file to set configuration."
+
+.docker-build:
+	make build
+
+.env:
+	touch .env
+
+.PHONY: build
+build: .env  ## | Build docker images.
+	docker compose --progress plain build
+	touch .docker-build
+
+.PHONY: shell
+shell: .env .docker-build  ## | Open a shell in docker.
+	docker compose run --rm shell
+
+.PHONY: devcontainer
+devcontainer: .env .docker-build  ## | Run VS Code development container.
+	docker compose up --detach devcontainer
+
+.PHONY: rebuildreqs
+rebuildreqs: .env .docker-build  ## | Rebuild requirements.txt file after requirements.in changes.
+	docker compose run --rm --no-deps shell pip-compile --allow-unsafe --generate-hashes --strip-extras --quiet
+
+.PHONY: lint
+lint: .env .docker-build  ## | Lint code.
+	docker compose run --rm --no-deps shell bin/lint.sh
+
+.PHONY: lintfix
+lintfix: .env .docker-build  ## | Reformat code.
+	docker compose run --rm --no-deps shell bin/lint.sh --fix
+
+.PHONY: test
+test: .env .docker-build  ## | Run tests.
+	docker compose run --rm shell bin/test.sh

bin/lint.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+# Usage: bin/run_lint.sh [--fix]
+#
+# Runs linting and code fixing.
+#
+# This should be called from inside a container or venv.
+
+set -euo pipefail
+
+FILES="bin obs_common tests"
+PYTHON_VERSION=$(python --version)
+
+
+if [[ "${1:-}" == "--fix" ]]; then
+    echo ">>> ruff fix (${PYTHON_VERSION})"
+    ruff format $FILES
+    ruff check --fix $FILES
+
+else
+    echo ">>> ruff (${PYTHON_VERSION})"
+    ruff check $FILES
+    ruff format --check $FILES
+
+    echo ">>> license check (${PYTHON_VERSION})"
+    if [[ -d ".git" ]]; then
+        # If the .git directory exists, we can let license-check do
+        # git ls-files.
+        license-check
+    else
+        # The .git directory doesn't exist, so run it on all the Python
+        # files in the tree.
+        license-check .
+    fi
+fi

bin/test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+# Usage: bin/run_tests.sh
+#
+# Runs tests.
+#
+# This should be called from inside a container and after the dependent
+# services have been launched. It depends on:
+#
+# * elasticsearch
+# * postgresql
+
+set -euo pipefail
+
+# Wait for services to be ready (both have the same endpoint url)
+echo ">>> wait for services"
+urlwait "http://${PUBSUB_EMULATOR_HOST}" 10
+urlwait "${STORAGE_EMULATOR_HOST}/storage/v1/b" 10
+waitfor --verbose --codes=200,404 "${SENTRY_DSN}"
+
+# Run tests
+echo ">>> pytest"
+pytest $@

docker-compose.yml

@@ -1,9 +1,43 @@
 services:
+  shell:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+      args:
+        userid: ${USE_UID:-10001}
+        groupid: ${USE_GID:-10001}
+    image: local/obs-common-shell
+    env_file:
+      - docker/config/local_dev.env
+    links:
+      - fakesentry
+      - gcs-emulator
+      - pubsub
+    volumes:
+      - .:/app
+
+  devcontainer:
+    build:
+      dockerfile: .devcontainer/Dockerfile
+      args:
+        userid: ${USE_UID:-10001}
+        groupid: ${USE_GID:-10001}
+    image: local/obs-common-devcontainer
+    entrypoint: ["sleep", "inf"]
+    env_file:
+      - docker/config/local_dev.env
+    links:
+      - fakesentry
+      - gcs-emulator
+      - pubsub
+    volumes:
+      - .:/app
+
   # https://github.com/willkg/kent
   fakesentry:
     build:
       context: docker/images/fakesentry
-    image: local/tecken_fakesentry
+    image: local/obs-common-fakesentry
     ports:
       - "${EXPOSE_SENTRY_PORT:-8090}:8090"
     command: run --host 0.0.0.0 --port 8090

docker/Dockerfile

@@ -0,0 +1,35 @@
+# This should be kept in sync with the python version used in .devcontainer/Dockerfile and
+# docker/images/fakesentry/Dockerfile
+FROM --platform=linux/amd64 python:3.11.10-slim-bullseye@sha256:f6a64ef0a5cc14855b15548056a8fc77f4c3526b79883fa6709a8e23f676ac34
+
+# Set up user and group
+ARG groupid=10001
+ARG userid=10001
+
+WORKDIR /app/
+RUN groupadd --gid $groupid app && \
+    useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app && \
+    chown app:app /app/
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git \
+        ripgrep \
+        tig && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install Python dependencies
+COPY requirements.txt /app/
+
+RUN pip install -U 'pip>=20' && \
+    pip install --no-cache-dir --no-deps --only-binary :all: -r requirements.txt && \
+    pip install --no-cache-dir ipython && \
+    pip check --disable-pip-version-check
+
+COPY . /app
+
+RUN pip install -e . --no-deps
+
+CMD ["/bin/bash"]

docker/config/local_dev.env

@@ -0,0 +1,8 @@
+# Set Pub/Sub library to use emulator
+PUBSUB_EMULATOR_HOST=pubsub:5010
+
+# Set GCS library to use emulator
+STORAGE_EMULATOR_HOST=http://gcs-emulator:8001
+
+# Set up fakesentry
+SENTRY_DSN=http://public@fakesentry:8090/1

obs_common/waitfor.py

@@ -59,6 +59,8 @@ def main(args=None):
                 if resp.code in ok_codes:
                     sys.exit(0)
                 last_fail = f"HTTP status code: {resp.code}"
+        except ConnectionResetError as error:
+            last_fail = f"ConnectionResetError: {error}"
         except TimeoutError as error:
             last_fail = f"TimeoutError: {error}"
         except urllib.error.URLError as error:

pyproject.toml

@@ -23,6 +23,7 @@ service-status = "obs_common.service_status:main"
 gcs-cli = "obs_common.gcs_cli:main"
 pubsub-cli = "obs_common.pubsub_cli:main"
 sentry-wrap = "obs_common.sentry_wrap:cli_main"
+waitfor = "obs_common.waitfor:main"
 
 [build-system]
 requires = ["setuptools", "setuptools_scm[toml]>=6.2"]

requirements.in

@@ -8,3 +8,4 @@ requests==2.32.3
 ruff==0.7.1
 sentry-sdk==2.17.0
 twine==5.1.1
+urlwait==1.0

requirements.txt

@@ -640,6 +640,10 @@ urllib3==2.2.2 \
     #   requests
     #   sentry-sdk
     #   twine
+urlwait==1.0 \
+    --hash=sha256:a9bf2da792fa6983fa93f6360108e16615066ab0f9cfb7f53e5faee5f5dffaac \
+    --hash=sha256:eae2c20001efc915166cac79c04bac0088ad5787ec64b36f27afd2f359953b2b
+    # via -r requirements.in
 wheel==0.44.0 \
     --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f \
     --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49