Migrate to uv for Python dependency management

.github/workflows/ci.yml

@@ -20,18 +20,21 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python
-      uses: actions/setup-python@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
       with:
-        python-version: '3.11'
-
-    - name: Cache pip dependencies
+        version: "0.5.27"
+
+    - name: Set up Python
+      run: uv python install 3.11
+
+    - name: Cache uv dependencies
       uses: actions/cache@v3
       with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
+        path: ~/.cache/uv
+        key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock') }}
         restore-keys: |
-          ${{ runner.os }}-pip-
+          ${{ runner.os }}-uv-
 
     - name: Cache embedding models
       uses: actions/cache@v3
@@ -42,9 +45,7 @@ jobs:
           ${{ runner.os }}-embeddings-
 
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      run: uv sync --locked --extra test
 
     - name: Wait for Qdrant to be ready
       run: |
@@ -56,16 +57,13 @@ jobs:
         echo "PYTHONPATH=${{ github.workspace }}/scripts:$PYTHONPATH" >> $GITHUB_ENV
         echo "CI=true" >> $GITHUB_ENV
         echo "USE_TREE_SITTER=1" >> $GITHUB_ENV
-        # Note: COLLECTION_NAME is intentionally NOT set globally.
-        # Integration tests set their own unique collection names.
-        # Unit tests mock Qdrant and don't need a real collection.
 
     - name: Pre-download embedding model
       run: |
-        python -c "from fastembed import TextEmbedding; m = TextEmbedding(model_name='BAAI/bge-base-en-v1.5'); list(m.embed(['test']))"
+        uv run python -c "from fastembed import TextEmbedding; m = TextEmbedding(model_name='BAAI/bge-base-en-v1.5'); list(m.embed(['test']))"
 
     - name: Run tests
-      run: pytest -q --junitxml=test-results.xml
+      run: uv run python -m pytest -q --junitxml=test-results.xml
 
     - name: Upload test results
       uses: actions/upload-artifact@v4

.gitignore

@@ -38,7 +38,7 @@ CLAUDE.md
 !/dev-workspace/.gitkeep
 docs/AUGMENT_COMPATIBILITY.md
 .serena/project.yml
-/.venv311
+/.venv
 /.venv311
 nueral.txt
 refrag.txt
@@ -69,3 +69,5 @@ ctx_config.json
 .contextstream/ignore
 .cursorrules
 GEMINI.md
+ctx-mcp-bridge/bin/ctxce.js
+/ctx-mcp-bridge/bin

.python-version

@@ -0,0 +1 @@
+3.11

Dockerfile

@@ -2,9 +2,14 @@
 # Supports multiple roles: memory, indexer, watcher, llamacpp
 FROM python:3.11-slim
 
+# Install uv for 10-100x faster dependency installation
+COPY --from=ghcr.io/astral-sh/uv:0.5.27 /uv /uvx /bin/
+
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
-    WORK_ROOTS="/work,/app"
+    WORK_ROOTS="/work,/app" \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
 
 # Install OS dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -13,19 +18,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-# Python deps: reuse shared requirements file for consistency across services
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt
+# Copy dependency files first for better caching
+COPY pyproject.toml uv.lock /app/
+
+# Install dependencies using uv (cached layer)
+WORKDIR /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project --no-dev
 
 # Copy scripts and templates for all services
 COPY scripts /app/scripts
 COPY templates /app/templates
 
+# Activate the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
+
 # Create directories
 WORKDIR /work
 
 # Expose all necessary ports
 EXPOSE 8000 8001 8002 8003 18000 18001 18002 18003
 
 # Default to memory server
-CMD ["python", "/app/scripts/mcp_memory_server.py"]
+CMD ["python", "/app/scripts/mcp_memory_server.py"]

Dockerfile.indexer

@@ -1,39 +1,36 @@
 # Dockerized code indexer for Qdrant
 FROM python:3.11-slim
 
+COPY --from=ghcr.io/astral-sh/uv:0.5.27 /uv /uvx /bin/
+
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     WORK_ROOTS="/work,/app" \
-    PIP_DEFAULT_TIMEOUT=120 \
-    PIP_RETRIES=10 \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
     LOG_LEVEL=INFO
 
-# OS packages needed: git for history ingestion, curl for model downloads
 RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl && rm -rf /var/lib/apt/lists/*
 
-# Python deps: reuse shared requirements file
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir --upgrade --timeout=${PIP_DEFAULT_TIMEOUT} --retries=${PIP_RETRIES} -r /tmp/requirements.txt
+COPY pyproject.toml uv.lock /app/
+
+WORKDIR /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project --no-dev
 
-# Download reranker model and tokenizer during build
-# Cross-encoder for reranking (ms-marco-MiniLM) + BGE tokenizer for micro-chunking
 ARG RERANKER_ONNX_URL=https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2/resolve/main/onnx/model.onnx
 ARG TOKENIZER_URL=https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json
 RUN mkdir -p /app/models && \
     curl -L --fail --retry 3 -o /app/models/reranker.onnx "${RERANKER_ONNX_URL}" && \
     curl -L --fail --retry 3 -o /app/models/tokenizer.json "${TOKENIZER_URL}"
 
-# Set default paths for reranker (can be overridden via env)
 ENV RERANKER_ONNX_PATH=/app/models/reranker.onnx \
-    RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json
+    RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json \
+    PATH="/app/.venv/bin:$PATH"
 
-# Bake scripts into the image so we can mount arbitrary code at /work
 COPY scripts /app/scripts
-
-# Copy plugins for Neo4j graph backend and other extensions
 COPY plugins /app/plugins
 
 WORKDIR /work
 
-# Default command shows help; Makefile/compose will override entrypoint
 CMD ["python", "/app/scripts/ingest_code.py", "--help"]

Dockerfile.mcp

@@ -1,25 +1,29 @@
 # Minimal image to run the Qdrant MCP server with SSE transport
 FROM python:3.11-slim
 
+COPY --from=ghcr.io/astral-sh/uv:0.5.27 /uv /uvx /bin/
+
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     WORK_ROOTS="/work,/app" \
     HF_HOME=/tmp/cache \
-    TRANSFORMERS_CACHE=/tmp/cache
+    TRANSFORMERS_CACHE=/tmp/cache \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
+
+COPY pyproject.toml uv.lock /app/
 
-# Python deps: reuse shared requirements file for consistency across services
-# Create cache/rerank directories in same layer
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir --upgrade --timeout 300 --retries 3 -r /tmp/requirements.txt \
+WORKDIR /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project --no-dev \
     && mkdir -p /tmp/cache && chmod 755 /tmp/cache \
     && mkdir -p /tmp/rerank_events /tmp/rerank_weights \
     && chmod 777 /tmp/rerank_events /tmp/rerank_weights
 
-# Bake scripts into image so server can run even when /work points elsewhere
+ENV PATH="/app/.venv/bin:$PATH"
+
 COPY scripts /app/scripts
 
-# Expose SSE port
 EXPOSE 8000
 
-# Default command: run the server with SSE transport (env provides host/port)
 CMD ["python", "/app/scripts/mcp_memory_server.py"]

Dockerfile.mcp-indexer

@@ -1,23 +1,24 @@
 # Companion MCP server that exposes indexing/pruning/list tools over SSE
-# Reuse the same deps as the indexer image so we can call the scripts directly.
 FROM python:3.11-slim
 
+COPY --from=ghcr.io/astral-sh/uv:0.5.27 /uv /uvx /bin/
+
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     WORK_ROOTS="/work,/app" \
-    PIP_DEFAULT_TIMEOUT=120 \
-    PIP_RETRIES=10 \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
     LOG_LEVEL=INFO
 
-# OS deps (git for history if we extend later, curl for model downloads, build-essential for compiling native extensions)
 RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-# Python deps: reuse shared requirements (includes FastMCP + OpenAI SDK)
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir --upgrade --timeout=${PIP_DEFAULT_TIMEOUT} --retries=${PIP_RETRIES} -r /tmp/requirements.txt
+COPY pyproject.toml uv.lock /app/
+
+WORKDIR /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project --no-dev
 
-# Create rerank dirs + download reranker model/tokenizer in single layer
 ARG RERANKER_ONNX_URL=https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2/resolve/main/onnx/model.onnx
 ARG TOKENIZER_URL=https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json
 RUN mkdir -p /tmp/rerank_events /tmp/rerank_weights \
@@ -26,22 +27,16 @@ RUN mkdir -p /tmp/rerank_events /tmp/rerank_weights \
     && curl -L --fail --retry 3 -o /app/models/reranker.onnx "${RERANKER_ONNX_URL}" \
     && curl -L --fail --retry 3 -o /app/models/tokenizer.json "${TOKENIZER_URL}"
 
-# Set default paths for reranker (can be overridden via env)
 ENV RERANKER_ONNX_PATH=/app/models/reranker.onnx \
-    RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json
+    RERANKER_TOKENIZER_PATH=/app/models/tokenizer.json \
+    PATH="/app/.venv/bin:$PATH"
 
-# Bake scripts into the image so entrypoints don't rely on /work
 COPY scripts /app/scripts
-
 COPY bench /app/bench
-
-# Copy plugins for Neo4j graph backend and other extensions
 COPY plugins /app/plugins
 
-# Expose SSE port for this companion server
 EXPOSE 8001
 
 WORKDIR /work
 
-# Default command runs the companion MCP server
 CMD ["python", "/app/scripts/mcp_indexer_server.py"]

Dockerfile.upload-service

@@ -1,53 +1,46 @@
 # Dockerfile for Context-Engine Delta Upload Service
 FROM python:3.11-slim
 
-# Set environment variables
+COPY --from=ghcr.io/astral-sh/uv:0.5.27 /uv /uvx /bin/
+
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
-    PIP_NO_CACHE_DIR=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PYTHONPATH=/app
+    PYTHONPATH=/app \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
 
-# Install system dependencies
 RUN apt-get update && apt-get install -y \
     git \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-# Create app directory
 WORKDIR /app
 
-# Copy requirements first for better caching
-COPY requirements.txt .
+COPY pyproject.toml uv.lock ./
 
-# Install Python dependencies
-RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-install-project --no-dev
 
-# Copy application code
 COPY scripts/ ./scripts/
 COPY . .
 
-# Create work dir, non-root user, and set ownership in single layer
+ENV PATH="/app/.venv/bin:$PATH"
+
 RUN mkdir -p /work && chmod 755 /work \
     && useradd --create-home --shell /bin/bash app \
     && chown -R app:app /app /work
 USER app
 
-# Expose port
 EXPOSE 8002
 
-# Health check
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
     CMD curl -f http://localhost:8002/health || exit 1
 
-# Default environment variables
 ENV UPLOAD_SERVICE_HOST=0.0.0.0 \
     UPLOAD_SERVICE_PORT=8002 \
     QDRANT_URL=http://qdrant:6333 \
     WORK_DIR=/work \
     MAX_BUNDLE_SIZE_MB=100 \
     UPLOAD_TIMEOUT_SECS=300
 
-# Run the upload service
-CMD ["python", "scripts/upload_service.py"]
+CMD ["python", "scripts/upload_service.py"]

Makefile

@@ -5,15 +5,21 @@ SHELL := /bin/bash
 export DOCKER_HOST =
 
 .PHONY: help up down up-redis down-redis restart-redis logs ps restart rebuild index reindex watch watch-remote env hybrid bootstrap history rerank-local setup-reranker prune warm health test-e2e
-.PHONY: venv venv-install dev-remote-up dev-remote-down dev-remote-logs dev-remote-restart dev-remote-bootstrap dev-remote-test dev-remote-client dev-remote-clean
+.PHONY: venv venv-install uv-sync uv-run dev-remote-up dev-remote-down dev-remote-logs dev-remote-restart dev-remote-bootstrap dev-remote-test dev-remote-client dev-remote-clean
 .PHONY: rerank-eval rerank-eval-ablations rerank-benchmark
 
 .PHONY: qdrant-status qdrant-list qdrant-prune qdrant-index-root
 
-venv: ## create local virtualenv .venv
+uv-sync: ## install dependencies via uv (10-100x faster than pip)
+	uv sync
+
+uv-run: ## run a command via uv (auto-syncs if needed): make uv-run CMD="python scripts/foo.py"
+	uv run $(CMD)
+
+venv: ## create local virtualenv .venv (legacy, prefer uv-sync)
 	python3 -m venv .venv && . .venv/bin/activate && pip install -U pip
 
-venv-install: ## install project dependencies into .venv
+venv-install: ## install project dependencies into .venv (legacy, prefer uv-sync)
 	[ -d .venv ] || $(MAKE) venv
 	. .venv/bin/activate && pip install -r requirements.txt
 

docker-compose.yml

@@ -132,6 +132,11 @@ services:
       - "6334:6334"
     volumes:
       - qdrant_storage_dev_remote:/qdrant/storage
+    environment:
+      # FOR LOCAL DEV: Larger WAL buffer = fewer disk flushes during bulk inserts
+      - QDRANT__STORAGE__WAL__WAL_CAPACITY_MB=${QDRANT_WAL_CAPACITY_MB:-64}
+      # FOR LOCAL DEV: Batch optimizer runs for better insert throughput
+      - QDRANT__STORAGE__OPTIMIZERS__INDEXING_THRESHOLD_KB=${QDRANT_INDEXING_THRESHOLD_KB:-20000}
     networks:
       - dev-remote-network