Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
1b4c726
add auth
MBueschelberger Oct 10, 2022
4eb83e1
improve checking if auth enabled
MBueschelberger Oct 10, 2022
e078df6
update generic import of authentication dependencies
MBueschelberger Oct 11, 2022
5e68ead
[Auto-generated] Update dependencies (#190)
TEAM4-0 Oct 12, 2022
2776e09
Merge branch 'master' into enh/add_auth
CasperWA Oct 12, 2022
8a2b6e6
Merge branch 'master' into enh/add_auth
MBueschelberger Oct 13, 2022
0d91e3a
Merge branch 'master' into enh/add_auth
CasperWA Oct 25, 2022
44e4bea
Update app/main.py
MBueschelberger Nov 29, 2022
36b88a2
Update app/main.py
MBueschelberger Nov 29, 2022
291bff3
Update app/main.py
MBueschelberger Nov 29, 2022
c807e88
Update app/main.py
MBueschelberger Nov 29, 2022
64a9208
Update docker-compose.yml
MBueschelberger Nov 29, 2022
d09ddea
Update docker-compose_dev.yml
MBueschelberger Nov 29, 2022
bdcb80a
Update app/main.py
MBueschelberger Nov 29, 2022
60829a6
Merge branch 'master' into enh/add_auth
MBueschelberger Nov 29, 2022
651606b
Update app/main.py
MBueschelberger Nov 29, 2022
4aa8beb
resolve importing dependencies for fastapi from env-variable, add ret…
MBueschelberger Nov 29, 2022
115980b
resolve unknown attribute name in AppSettings
MBueschelberger Nov 29, 2022
cb05f7f
update env-variables in docker-compose
MBueschelberger Nov 29, 2022
005cd76
debug forwarding of secrets in function and transformation secrets, a…
MBueschelberger Dec 12, 2022
fd86f0e
rename function for pytests
MBueschelberger Dec 12, 2022
53d9388
merge master into enh/add_auth
MBueschelberger Dec 12, 2022
8166f49
update oteapi-core commit sha, update variable name in docker-compose…
MBueschelberger Dec 13, 2022
453b1ca
update setting of secret-attribute in pydantic models
MBueschelberger Dec 13, 2022
1a5a12b
add option to exclude redisadmin-router on production
MBueschelberger Dec 14, 2022
e4dcd64
Merge branch 'master' into enh/add_auth
CasperWA Dec 14, 2022
c00e8be
Merge branch 'enh/add_auth' of github.com:MBueschelberger/oteapi-serv…
MBueschelberger Dec 14, 2022
7786c18
update version for oteapi-core, update model-attributes, add settings…
MBueschelberger Jan 9, 2023
ed1468b
Merge branch 'EMMC-ASBL:master' into enh/add_auth
MBueschelberger Jan 10, 2023
57b8f3c
upgrade oteapi-core
MBueschelberger Jan 24, 2023
8c68cbd
Merge branch 'master' into enh/add_auth
MBueschelberger Jan 24, 2023
ff6f5ac
update requirements.txt
MBueschelberger Jan 24, 2023
978ffd3
update DummyCache for pytests: values in cache should be strings, sin…
MBueschelberger Jan 24, 2023
31fffaa
Merge remote-tracking branch 'origin/master' into enh/add_auth
CasperWA Feb 2, 2023
cc619bd
Clean up conftest file
CasperWA Feb 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 58 additions & 6 deletions app/main.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
"""OTE-API FastAPI application."""
import logging
from importlib import import_module
from typing import TYPE_CHECKING

from fastapi import FastAPI, status
from fastapi import Depends, FastAPI, status
from fastapi.openapi.utils import get_openapi
from fastapi_plugins import RedisSettings, redis_plugin
from oteapi.plugins import load_strategies
from oteapi.settings import OteApiCoreSettings
from pydantic import Field

from app import __version__
Expand All @@ -21,12 +24,30 @@
)

if TYPE_CHECKING: # pragma: no cover
from typing import Any, Dict
from typing import Any, Dict, List


class AppSettings(RedisSettings):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


class AppSettings(RedisSettings, OteApiCoreSettings):
"""Redis settings."""

include_redisadmin: bool = Field(
False,
description="""If set to `True`,
the router for the low-level cache interface will be included into the api.
WARNING: This might NOT be recommended for specific production cases,
since sensible data (such as secrets) in the cache might be revealed by
inspecting other user's session objects. If set to false, the cache can
only be read from an admin accessing the redis backend.""",
)

authentication_dependencies: str = Field(
"", description="List of FastAPI dependencies for authentication features."
)

api_name: str = Field(
"oteapi_services", description="Application-specific name for Redis cache."
)
Expand All @@ -44,6 +65,7 @@ class Config:
def create_app() -> FastAPI:
"""Create the FastAPI app."""
app = FastAPI(
dependencies=get_auth_deps(),
title="Open Translation Environment API",
version=__version__,
description="""OntoTrans Interfaces OpenAPI schema.
Expand All @@ -61,16 +83,18 @@ def create_app() -> FastAPI:
This service is based on [**oteapi-core**](https://github.com/EMMC-ASBL/oteapi-core).
""",
)
for router_module in (
available_routers = [
session,
dataresource,
datafilter,
function,
mapping,
transformation,
redisadmin,
triplestore,
):
]
if CONFIG.include_redisadmin:
available_routers.append(redisadmin)
for router_module in available_routers:
app.include_router(
router_module.ROUTER,
prefix=CONFIG.prefix,
Expand All @@ -85,6 +109,34 @@ def create_app() -> FastAPI:
return app


def get_auth_deps() -> "List[Depends]":
"""Get authentication dependencies

Fetch dependencies for authentication through the
`OTEAPI_AUTH_DEPS` environment variable.

Returns:
List of FastAPI dependencies with authentication functions.

"""
if CONFIG.authentication_dependencies:
modules = [
module.strip().split(":")
for module in CONFIG.authentication_dependencies.split("|")
]
imports = [
getattr(import_module(module), classname) for (module, classname) in modules
]
logger.info(
"Imported the following dependencies for authentication: %s", imports
)
dependencies = [Depends(dependency) for dependency in imports]
else:
dependencies = []
logger.info("No dependencies for authentication assigned.")
return dependencies


def custom_openapi() -> "Dict[str, Any]":
"""Improve the default look & feel when rendering using ReDocs."""
if APP.openapi_schema:
Expand Down
9 changes: 7 additions & 2 deletions app/routers/dataresource.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import TYPE_CHECKING, Optional

from aioredis import Redis
from fastapi import APIRouter, Depends, status
from fastapi import APIRouter, Depends, Request, status
from fastapi_plugins import depends_redis
from oteapi.models import ResourceConfig
from oteapi.plugins import create_strategy
Expand Down Expand Up @@ -37,6 +37,7 @@
)
async def create_dataresource(
config: ResourceConfig,
request: Request,
session_id: Optional[str] = None,
cache: Redis = Depends(depends_redis),
) -> CreateResourceResponse:
Expand All @@ -55,7 +56,11 @@ async def create_dataresource(
"""
new_resource = CreateResourceResponse()

await cache.set(new_resource.resource_id, config.json())
config.token = request.headers.get("Authorization") or config.token

resource_config = config.json()

await cache.set(new_resource.resource_id, resource_config)

if session_id:
if not await cache.exists(session_id):
Expand Down
9 changes: 7 additions & 2 deletions app/routers/function.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import TYPE_CHECKING, Optional

from aioredis import Redis
from fastapi import APIRouter, Depends, status
from fastapi import APIRouter, Depends, Request, status
from fastapi_plugins import depends_redis
from oteapi.models import FunctionConfig
from oteapi.plugins import create_strategy
Expand All @@ -30,13 +30,18 @@
)
async def create_function(
config: FunctionConfig,
request: Request,
session_id: Optional[str] = None,
cache: Redis = Depends(depends_redis),
) -> CreateFunctionResponse:
"""Create a new function configuration."""
new_function = CreateFunctionResponse()

await cache.set(new_function.function_id, config.json())
config.token = request.headers.get("Authorization") or config.token

function_config = config.json()

await cache.set(new_function.function_id, function_config)

if session_id:
if not await cache.exists(session_id):
Expand Down
10 changes: 7 additions & 3 deletions app/routers/redisadmin.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
"""Helper service for viewing redis objects."""
import json
from typing import Dict, List
from typing import Any, Dict

from aioredis import Redis
from fastapi import APIRouter, Depends
from fastapi_plugins import depends_redis

from app.models.error import httpexception_404_item_id_does_not_exist

ROUTER = APIRouter(prefix="/redis")


@ROUTER.get("/{key}", include_in_schema=False)
async def get_gey(
async def get_key(
key: str,
cache: Redis = Depends(depends_redis),
) -> Dict[str, List[str]]:
) -> Dict[str, Any]:
"""Low-level cache interface to retrieve the object-value
stored with key 'key'
"""
if not await cache.exists(key):
raise httpexception_404_item_id_does_not_exist(key, "key")
return json.loads(await cache.get(key))
9 changes: 7 additions & 2 deletions app/routers/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import TYPE_CHECKING, Optional

from aioredis import Redis
from fastapi import APIRouter, Depends, status
from fastapi import APIRouter, Depends, Request, status
from fastapi_plugins import depends_redis
from oteapi.models import TransformationConfig, TransformationStatus
from oteapi.plugins import create_strategy
Expand Down Expand Up @@ -35,13 +35,18 @@
)
async def create_transformation(
config: TransformationConfig,
request: Request,
session_id: Optional[str] = None,
cache: Redis = Depends(depends_redis),
) -> CreateTransformationResponse:
"""Create a new transformation configuration."""
new_transformation = CreateTransformationResponse()

await cache.set(new_transformation.transformation_id, config.json())
config.token = request.headers.get("Authorization") or config.token

transformation_config = config.json()

await cache.set(new_transformation.transformation_id, transformation_config)

if session_id:
if not await cache.exists(session_id):
Expand Down
3 changes: 3 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ services:
OTEAPI_REDIS_HOST: redis
OTEAPI_REDIS_PORT: 6379
OTEAPI_prefix: "${OTEAPI_prefix:-/api/v1}"
OTEAPI_INCLUDE_REDISADMIN: "${OTEAPI_INCLUDE_REDISADMIN:-False}"
OTEAPI_EXPOSE_SECRETS: "${OTEAPI_EXPOSE_SECRETS:-True}"
Comment on lines +13 to +14
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wait. So by default you don't want to include the Redis admin endpoint (for security reasons) but do want to expose secrets? :/

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, the redis admin is not exposed because the secrets are exposed.

If the secret strings for the oteapi.models are not exposed, they cannot be read in the strategies anymore, since they are censored during serialization. This pretty much makes them useless if you want to use them for connecting to an external host.

If they are exposed and the redis admin is enabled, you may simply reveal all user's in a configuration through the redis endpoint.

For this reason, the chance, that you may remotely inspect the redis-cache from other's users sessions, is lowered since you only can access the redis-cache from the strategies, and not directly through the services anymore.

In other words, the motivation is that when you put sensitive/general information into the cache, you cannot reveal it by chance if you only got the config-id, but it still can be used by the strategies internally.

Since I did not find any straight-forward user management for redis through an oauth-scheme, I think this is one of the only ways how to deal with this scenario for the moment.

Does this make sense to you?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes sense. Essentially what we need is to hash/encrypt the secrets before storing in redis. That should solve it. The strategies can then get some interface to decrypting the secrets they may need. Or it's done from the service before invoking the strategies. Not very important, but it should circumvent this issue.

However that's something for another day and another PR.

OTEAPI_PLUGIN_PACKAGES:
OTEAPI_AUTHENTICAION_DEPENDENCIES:
depends_on:
- redis
networks:
Expand Down
3 changes: 3 additions & 0 deletions docker-compose_dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@ services:
OTEAPI_REDIS_HOST: redis
OTEAPI_REDIS_PORT: 6379
OTEAPI_prefix: "${OTEAPI_prefix:-/api/v1}"
OTEAPI_INCLUDE_REDISADMIN: "${OTEAPI_INCLUDE_REDISADMIN:-True}"
OTEAPI_EXPOSE_SECRETS: "${OTEAPI_EXPOSE_SECRETS:-True}"
PATH_TO_OTEAPI_CORE:
OTEAPI_PLUGIN_PACKAGES:
OTEAPI_AUTHENTICATION_DEPENDENCIES:
depends_on:
- redis
networks:
Expand Down
78 changes: 45 additions & 33 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
"""Fixtures and configuration for PyTest."""
# pylint: disable=invalid-name,redefined-builtin,unused-argument,comparison-with-callable
import os
from pathlib import Path
from typing import TYPE_CHECKING

import pytest
from fastapi.testclient import TestClient

if TYPE_CHECKING:
from pathlib import Path
from typing import Dict, List

from fastapi.testclient import TestClient


class DummyCache:
"""Mock cache for RedisCache."""
Expand All @@ -28,7 +28,7 @@ async def get(self, id) -> dict:
"""Mock `get()` method."""
import json

return json.dumps(self.obj[id])
return json.loads(json.dumps(self.obj[id]))

async def keys(self, pattern: str) -> "List[bytes]":
"""Mock `keys()` method."""
Expand All @@ -42,45 +42,56 @@ async def exists(self, key: str) -> bool:

def pytest_configure(config):
"""Method that runs before pytest collects tests so no modules are imported"""
import os

os.environ["OTEAPI_prefix"] = ""
os.environ["OTEAPI_INCLUDE_REDISADMIN"] = "True"
os.environ["OTEAPI_EXPOSE_SECRETS"] = "True"


@pytest.fixture(scope="session")
def top_dir() -> Path:
def top_dir() -> "Path":
"""Resolved path to repository directory."""
from pathlib import Path

return Path(__file__).resolve().parent.parent.resolve()


@pytest.fixture(scope="session")
def test_data() -> "Dict[str, dict]":
def test_data() -> "Dict[str, str]":
"""Test data stored in DummyCache."""
import json

return {
# filter
"filter-961f5314-9e8e-411e-a216-ba0eb8e8bc6e": {
"filterType": "filter/demo",
"configuration": {"demo_data": [1, 2]},
},
# function
"function-a647012a-7ab9-4f2c-9c13-2564aa6d95a1": {
"functionType": "function/demo",
"configuration": {},
},
# mapping
"mapping-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e": {
"mappingType": "mapping/demo",
"prefixes": {":": "<http://namespace.example.com/ns#"},
"triples": [[":a", ":has", ":b"]],
"configuration": {},
},
# sessions
"1": {"foo": "bar"},
"2": {"foo": "bar"},
# transformation
"transformation-f752c613-fde0-4d43-a7f6-c50f68642daa": {
"transformationType": "script/demo",
"name": "script/dummy",
"configuration": {},
},
key: json.dumps(value)
for key, value in {
# filter
"filter-961f5314-9e8e-411e-a216-ba0eb8e8bc6e": {
"filterType": "filter/demo",
"configuration": {"demo_data": [1, 2]},
},
# function
"function-a647012a-7ab9-4f2c-9c13-2564aa6d95a1": {
"functionType": "function/demo",
"configuration": {},
},
# mapping
"mapping-a2d6b3d5-9b6b-48a3-8756-ae6d4fd6b81e": {
"mappingType": "mapping/demo",
"prefixes": {":": "<http://namespace.example.com/ns#"},
"triples": [[":a", ":has", ":b"]],
"configuration": {},
},
# sessions
"1": {"foo": "bar"},
"2": {"foo": "bar"},
# transformation
"transformation-f752c613-fde0-4d43-a7f6-c50f68642daa": {
"transformationType": "script/demo",
"name": "script/dummy",
"configuration": {},
},
}.items()
}


Expand Down Expand Up @@ -143,8 +154,9 @@ def load_test_strategies() -> None:


@pytest.fixture(scope="session")
def client(test_data: "Dict[str, dict]") -> TestClient:
def client(test_data: "Dict[str, dict]") -> "TestClient":
"""Return a test client."""
from fastapi.testclient import TestClient
from fastapi_plugins import depends_redis

from asgi import app
Expand Down
Loading