security(auth): fail closed when the password-epoch check cannot reach Redis (#16411)

autobot-backend/auth_middleware.py

@@ -17,6 +17,7 @@
 
 from fastapi import HTTPException, Request, status
 
+from auth_revocation import reject_if_revoked_by_password_change
 from autobot_shared.auth.jwt_core import (
     decode_jwt_multi,
     encode_jwt,
@@ -29,9 +30,6 @@
 from autobot_shared.singleton_factory import lazy_singleton
 from autobot_shared.ssot_config import config as ssot_config
 from autobot_shared.time_utils import parse_utc_iso
-from autobot_shared.user_management.password_epoch import (
-    is_token_revoked_by_password_change,
-)
 from config.manager import get_config_manager
 from security_layer import SecurityLayer
 from utils.catalog_http_exceptions import raise_auth_error
@@ -128,6 +126,21 @@ def _jwt_key_file() -> Path:
         """
         return ssot_config.path.data_path / "service-keys" / "jwt_rsa_private.pem"
 
+    @staticmethod
+    def _write_jwt_key_file(key_file: Path, pem: str) -> None:
+        """Write PEM to the durable key file with mode 0600 (``_get_rs256_keypair``)."""
+        try:
+            key_file.parent.mkdir(parents=True, exist_ok=True)
+            key_file.write_text(pem, encoding="utf-8")
+            os.chmod(key_file, 0o600)
+            logger.info("RS256 private key written to durable file %s", key_file)
+        except Exception as exc:
+            logger.error(
+                "Failed to write RS256 key to %s: %s — key will be ephemeral this session",
+                key_file,
+                exc,
+            )
+
     def _get_rs256_keypair(self) -> Tuple[str, str, str]:
         """Load or auto-generate the RS256 keypair for signing user JWTs (#10196).
 
@@ -170,20 +183,6 @@ def _load_pem(pem: str):
                 backend=default_backend(),
             )
 
-        def _write_key_file(pem: str) -> None:
-            """Write PEM to the durable file with mode 0600."""
-            try:
-                key_file.parent.mkdir(parents=True, exist_ok=True)
-                key_file.write_text(pem, encoding="utf-8")
-                os.chmod(key_file, 0o600)
-                logger.info("RS256 private key written to durable file %s", key_file)
-            except Exception as exc:
-                logger.error(
-                    "Failed to write RS256 key to %s: %s — key will be ephemeral this session",
-                    key_file,
-                    exc,
-                )
-
         # Tier 1: env var
         pem_private = ssot_config.misc.jwt_private_key
         if pem_private:
@@ -218,7 +217,7 @@ def _write_key_file(pem: str) -> None:
             try:
                 private_key = _load_pem(stored_pem)
                 # Migrate to the durable file so the next restart reuses it
-                _write_key_file(stored_pem)
+                self._write_jwt_key_file(key_file, stored_pem)
                 logger.info("RS256 keypair migrated from security config to durable file")
                 return stored_pem, _derive_public(private_key), kid
             except Exception as exc:
@@ -241,7 +240,7 @@ def _write_key_file(pem: str) -> None:
         ).decode("utf-8")
         pem_public = _derive_public(private_key)
 
-        _write_key_file(pem_private)
+        self._write_jwt_key_file(key_file, pem_private)
 
         return pem_private, pem_public, kid
 
@@ -902,13 +901,9 @@ async def get_current_user(request: Request) -> Dict:
         # #12924: a token minted before its subject's password changed is no
         # longer valid. This is the first point on the request path that can
         # await, which is why the check lives here rather than in the
-        # synchronous extraction above.
-        if user_data.get("auth_method") == "jwt" and await is_token_revoked_by_password_change(user_data):
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
-                detail="Token is no longer valid — password was changed. Please sign in again.",
-                headers={"WWW-Authenticate": "Bearer"},
-            )
+        # synchronous extraction above. A check that cannot run denies (#16411).
+        if user_data.get("auth_method") == "jwt":
+            await reject_if_revoked_by_password_change(user_data)
 
         return user_data
 

autobot-backend/auth_revocation.py

@@ -0,0 +1,48 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""The backend's password-change revocation gate (#12924), failing closed (#16411).
+
+A token minted before its subject's password changed is refused. When the
+check reaches no verdict -- the epoch store cannot answer (#16411), or the
+marker or the token's ``iat`` cannot be read (#16422) -- the token is refused
+with the same 401: a token that may be revoked is never honoured, which is the
+rule the SLM follows (#16387). The owner accepted the cost on 2026-09-12 -- a
+Redis outage takes backend login down with it.
+"""
+
+from typing import Dict
+
+from fastapi import HTTPException, status
+
+from autobot_shared.logging_manager import get_logger
+from autobot_shared.user_management.password_epoch import (
+    RevocationCheckUnavailable,
+    is_token_revoked_by_password_change,
+)
+
+logger = get_logger(__name__)
+
+_BEARER_CHALLENGE = {"WWW-Authenticate": "Bearer"}
+
+
+async def reject_if_revoked_by_password_change(user_data: Dict) -> None:
+    """Raise a 401 unless the password-epoch check clears *user_data*'s token."""
+    try:
+        revoked = await is_token_revoked_by_password_change(user_data)
+    except RevocationCheckUnavailable as exc:
+        # The check and the helper's fixed reason only -- never the token, its
+        # claims or a stored value (#16411, #16422).
+        logger.error("password-epoch revocation check reached no verdict (%s); denying the token", exc)
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Token could not be verified. Please sign in again.",
+            headers=_BEARER_CHALLENGE,
+        ) from exc
+    if revoked:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Token is no longer valid — password was changed. Please sign in again.",
+            headers=_BEARER_CHALLENGE,
+        )

autobot-backend/tests/test_password_revocation_fail_closed_16411.py

@@ -0,0 +1,140 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""The backend refuses a token whose password-epoch check cannot run (#16411).
+
+Owner decision 2026-09-12: fail closed, the rule the SLM follows (#16387).
+These tests drive the REAL ``get_current_user`` -- under pytest
+``auth_middleware`` is the conftest stub, so it comes from
+``real_auth_middleware`` -- and fail only the Redis read inside the real shared
+helper, so each one judges the whole path: helper, gate, 401.
+"""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from fastapi import HTTPException
+from redis.exceptions import RedisError
+
+import auth_revocation
+
+_REDIS = "autobot_shared.user_management.password_epoch.get_async_redis_client"
+_JWT_USER = {"username": "alice", "sub": "alice", "iat": 1, "auth_method": "jwt", "role": "user"}
+_REQUEST = SimpleNamespace(headers={}, url=SimpleNamespace(path="/api/chat"))
+
+
+def _redis_answering(value=None, error=None) -> AsyncMock:
+    client = AsyncMock()
+    client.get = AsyncMock(return_value=value, side_effect=error)
+    return AsyncMock(return_value=client)
+
+
+def _signed_in_as(real_auth_middleware, monkeypatch, user: dict):
+    """The real dependency, seeing *user* as the signed-in JWT caller."""
+    middleware = SimpleNamespace(get_user_from_request=lambda _request: dict(user))
+    monkeypatch.setattr(real_auth_middleware, "get_auth_middleware", lambda: middleware)
+    return real_auth_middleware.get_current_user
+
+
+@pytest.fixture
+def get_current_user(real_auth_middleware, monkeypatch):
+    return _signed_in_as(real_auth_middleware, monkeypatch, _JWT_USER)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [RedisError("down"), ConnectionError("refused"), TimeoutError("slow")])
+async def test_a_redis_error_denies_the_token(get_current_user, error):
+    with patch(_REDIS, _redis_answering(error=error)):
+        with pytest.raises(HTTPException) as denied:
+            await get_current_user(_REQUEST)
+
+    assert denied.value.status_code == 401
+    assert denied.value.headers == {"WWW-Authenticate": "Bearer"}
+
+
+@pytest.mark.asyncio
+async def test_no_redis_client_denies_the_token(get_current_user):
+    with patch(_REDIS, AsyncMock(return_value=None)):
+        with pytest.raises(HTTPException) as denied:
+            await get_current_user(_REQUEST)
+
+    assert denied.value.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_a_healthy_redis_admits_a_token_with_no_password_change(get_current_user):
+    """The control: the 401s above come from the failed check, not from the harness."""
+    with patch(_REDIS, _redis_answering(value=None)):
+        user = await get_current_user(_REQUEST)
+
+    assert user["username"] == "alice"
+
+
+@pytest.mark.asyncio
+async def test_a_token_from_before_a_password_change_is_still_refused(get_current_user):
+    with patch(_REDIS, _redis_answering(value="1000")):
+        with pytest.raises(HTTPException) as denied:
+            await get_current_user(_REQUEST)
+
+    assert denied.value.status_code == 401
+    assert "password was changed" in denied.value.detail
+
+
+@pytest.mark.asyncio
+async def test_a_token_from_after_a_password_change_is_admitted(real_auth_middleware, monkeypatch):
+    """The other side of the epoch: a marker older than the token's iat does not revoke it."""
+    get_current_user = _signed_in_as(real_auth_middleware, monkeypatch, {**_JWT_USER, "iat": 2000})
+    with patch(_REDIS, _redis_answering(value="1000")):
+        user = await get_current_user(_REQUEST)
+
+    assert user["username"] == "alice"
+
+
+@pytest.mark.asyncio
+async def test_the_denial_names_the_check_and_error_type_never_the_token(get_current_user):
+    with patch(_REDIS, _redis_answering(error=RedisError("down"))):
+        with patch.object(auth_revocation.logger, "error") as log_error:
+            with pytest.raises(HTTPException):
+                await get_current_user(_REQUEST)
+
+    message, *args = log_error.call_args.args
+    rendered = message % tuple(args)
+    assert "password-epoch" in rendered
+    assert "RedisError" in rendered
+    assert "alice" not in rendered
+
+
+@pytest.mark.asyncio
+async def test_a_corrupt_epoch_marker_denies_the_token(get_current_user):
+    """#16422: a marker the check cannot read denies, like a store that cannot answer."""
+    with patch(_REDIS, _redis_answering(value="not-a-number")):
+        with pytest.raises(HTTPException) as denied:
+            await get_current_user(_REQUEST)
+
+    assert denied.value.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_a_non_integer_iat_denies_the_token(real_auth_middleware, monkeypatch):
+    get_current_user = _signed_in_as(real_auth_middleware, monkeypatch, dict(_JWT_USER, iat="not-a-number"))
+    with patch(_REDIS, _redis_answering(value=None)):
+        with pytest.raises(HTTPException) as denied:
+            await get_current_user(_REQUEST)
+
+    assert denied.value.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_a_corrupt_marker_is_logged_without_its_value(get_current_user):
+    with patch(_REDIS, _redis_answering(value="not-a-number")):
+        with patch.object(auth_revocation.logger, "error") as log_error:
+            with pytest.raises(HTTPException):
+                await get_current_user(_REQUEST)
+
+    message, *args = log_error.call_args.args
+    rendered = message % tuple(args)
+    assert "password-epoch" in rendered
+    assert "not-a-number" not in rendered
+    assert "alice" not in rendered

autobot-slm-backend/tests/api/test_auth_me_16385.py

@@ -187,26 +187,53 @@ async def exists(self, _key: str) -> int:
         return 0
 
 
-def _get_me(headers: dict | None = None):
-    """Dispatch GET /api/auth/me with the jti denylist given a working fake
-    Redis, and the password-epoch check's Redis patched to fail open.
+class _FakeEpochRedis:
+    """A working Redis stand-in for the password-epoch store (#16411).
+
+    #16411 made the epoch check fail CLOSED like the denylist: an absent client
+    now raises rather than reporting "not revoked". So the REAL helper gets a
+    Redis that answers -- with no marker by default, or with the marker a test
+    sets to place the token before or after a password change.
+    """
+
+    def __init__(self, marker: str | None = None, error: Exception | None = None) -> None:
+        self._marker = marker
+        self._error = error
+
+    async def get(self, _key: str) -> str | None:
+        if self._error is not None:
+            raise self._error
+        return self._marker
+
+
+def _get_me(
+    headers: dict | None = None,
+    epoch_marker: str | None = None,
+    epoch_error: Exception | None = None,
+    epoch_client_present: bool = True,
+):
+    """Dispatch GET /api/auth/me with both revocation checks given a working
+    fake Redis.
 
     For a valid HS256 token, get_current_user's decode path consults the jti
-    denylist (services.token_denylist.is_jti_revoked, fail-CLOSED since
-    #16387 -- needs a real answer, not an absent client) and the password-epoch
-    revocation check (autobot_shared...password_epoch.is_token_revoked_by_
-    password_change, which still fails open when Redis is unavailable --
-    #11443, #12924). Patching password-epoch's get_async_redis_client to
-    return None exercises that fail-open path deterministically, without a
-    real network call racing or timing out.
+    denylist (services.token_denylist.is_jti_revoked, fail-CLOSED since #16387)
+    and the password-epoch check (autobot_shared...password_epoch.is_token_
+    revoked_by_password_change, fail-CLOSED since #16411). Both need a Redis
+    that answers, not an absent client, and each fake answers deterministically
+    without a real network call racing or timing out. *epoch_marker* is the
+    stored password-change epoch the real epoch check reads (None: no change);
+    *epoch_error* makes that read raise, and *epoch_client_present=False*
+    leaves the epoch check with no Redis client at all (#16659).
     """
     from fastapi.testclient import TestClient
 
     import autobot_shared.user_management.password_epoch as password_epoch_mod
 
+    epoch_client = _FakeEpochRedis(epoch_marker, epoch_error) if epoch_client_present else None
+    epoch_redis = AsyncMock(return_value=epoch_client)
     with (
         patch.object(_dl_mod, "get_async_redis_client", AsyncMock(return_value=_FakeDenylistRedis())),
-        patch.object(password_epoch_mod, "get_async_redis_client", AsyncMock(return_value=None)),
+        patch.object(password_epoch_mod, "get_async_redis_client", epoch_redis),
         TestClient(app) as client,
     ):
         return client.get("/api/auth/me", headers=headers or {})
@@ -220,6 +247,36 @@ def test_valid_token_returns_2xx(self):
         assert resp.json() == {"username": "testuser", "is_admin": False, "user_type": "slm_admin"}
 
 
+class TestAuthMePasswordEpoch:
+    """The REAL password-epoch check end to end through GET /api/auth/me (#16411).
+
+    The no-marker case is TestAuthMeValidToken; these place the token's iat
+    before and after a stored password-change epoch.
+    """
+
+    def test_token_issued_before_the_password_change_gets_401(self):
+        token = _mint_token(iat=1000)
+        resp = _get_me(headers={"Authorization": f"Bearer {token}"}, epoch_marker="2000")
+        assert resp.status_code == 401, resp.text
+
+    def test_token_issued_after_the_password_change_returns_2xx(self):
+        token = _mint_token(iat=2000)
+        resp = _get_me(headers={"Authorization": f"Bearer {token}"}, epoch_marker="1000")
+        assert 200 <= resp.status_code < 300, resp.text
+
+    def test_a_redis_error_on_the_epoch_read_gets_401(self):
+        """#16659: a real store error inside get_password_epoch denies the token."""
+        token = _mint_token()
+        resp = _get_me(headers={"Authorization": f"Bearer {token}"}, epoch_error=ConnectionError("redis refused"))
+        assert resp.status_code == 401, resp.text
+
+    def test_no_epoch_redis_client_gets_401(self):
+        """#16659: with no Redis client the epoch check cannot run, so the token is denied."""
+        token = _mint_token()
+        resp = _get_me(headers={"Authorization": f"Bearer {token}"}, epoch_client_present=False)
+        assert resp.status_code == 401, resp.text
+
+
 class TestAuthMeMissingOrInvalidToken:
     def test_missing_token_returns_401(self):
         """No Authorization header: HTTPBearer's own auto_error default, not get_current_user.