diff --git a/api/conftest.py b/api/conftest.py index 71ea7d54d92d..94c6668ce9e4 100644 --- a/api/conftest.py +++ b/api/conftest.py @@ -1,13 +1,19 @@ +import os import typing +import boto3 import pytest from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from flag_engine.segments.constants import EQUAL +from moto import mock_dynamodb +from mypy_boto3_dynamodb.service_resource import Table +from pytest_django.fixtures import SettingsWrapper from rest_framework.authtoken.models import Token from rest_framework.test import APIClient from api_keys.models import MasterAPIKey +from environments.dynamodb.dynamodb_wrapper import DynamoEnvironmentV2Wrapper from environments.identities.models import Identity from environments.identities.traits.models import Trait from environments.models import Environment, EnvironmentAPIKey @@ -540,3 +546,51 @@ def project_content_type(): @pytest.fixture def manage_user_group_permission(db): return OrganisationPermissionModel.objects.get(key=MANAGE_USER_GROUPS) + + +@pytest.fixture() +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + + +@pytest.fixture() +def dynamodb(aws_credentials): + # TODO: move all wrapper tests to using moto + with mock_dynamodb(): + yield boto3.resource("dynamodb") + + +@pytest.fixture() +def flagsmith_environments_v2_table(dynamodb) -> Table: + return dynamodb.create_table( + TableName="flagsmith_environments_v2", + KeySchema=[ + { + "AttributeName": "environment_id", + "KeyType": "HASH", + }, + { + "AttributeName": "document_key", + "KeyType": "RANGE", + }, + ], + AttributeDefinitions=[ + {"AttributeName": "environment_id", "AttributeType": "S"}, + {"AttributeName": "document_key", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + + +@pytest.fixture +def dynamodb_wrapper_v2( + settings: SettingsWrapper, + flagsmith_environments_v2_table: Table, +) -> DynamoEnvironmentV2Wrapper: + settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO = flagsmith_environments_v2_table.name + return DynamoEnvironmentV2Wrapper() diff --git a/api/edge_api/identities/audit.py b/api/edge_api/identities/audit.py deleted file mode 100644 index a87194d41996..000000000000 --- a/api/edge_api/identities/audit.py +++ /dev/null @@ -1,30 +0,0 @@ -import typing - -from flag_engine.features.models import FeatureStateModel - -if typing.TYPE_CHECKING: - from edge_api.identities.models import EdgeIdentity - - -def generate_change_dict( - change_type: str, - identity: "EdgeIdentity", - new: typing.Optional[FeatureStateModel] = None, - old: typing.Optional[FeatureStateModel] = None, -): - if not (new or old): - raise ValueError("Must provide one of 'current' or 'previous'") - - change_dict = {"change_type": change_type} - if new: - change_dict["new"] = { - "enabled": new.enabled, - "value": new.get_value(identity.id), - } - if old: - change_dict["old"] = { - "enabled": old.enabled, - "value": old.get_value(identity.id), - } - - return change_dict diff --git a/api/edge_api/identities/models.py b/api/edge_api/identities/models.py index 1e48a120b5a2..3cde399ad933 100644 --- a/api/edge_api/identities/models.py +++ b/api/edge_api/identities/models.py @@ -8,6 +8,13 @@ from flag_engine.identities.models import IdentityFeaturesList, IdentityModel from api_keys.models import MasterAPIKey +from edge_api.identities.tasks import ( + generate_audit_log_records, + sync_identity_document_features, + update_flagsmith_environments_v2_identity_overrides, +) +from edge_api.identities.types import IdentityChangeset +from edge_api.identities.utils import generate_change_dict from environments.dynamodb import DynamoIdentityWrapper from environments.models import Environment from features.models import FeatureState @@ -15,9 +22,6 @@ from users.models import FFAdminUser from util.mappers import map_engine_identity_to_identity_document -from .audit import generate_change_dict -from .tasks import generate_audit_log_records, sync_identity_document_features - class EdgeIdentity: dynamo_wrapper = DynamoIdentityWrapper() @@ -161,7 +165,7 @@ def remove_feature_override(self, feature_state: FeatureStateModel) -> None: def save(self, user: FFAdminUser = None, master_api_key: MasterAPIKey = None): self.dynamo_wrapper.put_item(self.to_document()) - changes = self._get_changes(self._initial_state) + changes = self._get_changes() if changes["feature_overrides"]: # TODO: would this be simpler if we put a wrapper around FeatureStateModel instead? generate_audit_log_records.delay( @@ -174,6 +178,13 @@ def save(self, user: FFAdminUser = None, master_api_key: MasterAPIKey = None): "master_api_key_id": getattr(master_api_key, "id", None), } ) + update_flagsmith_environments_v2_identity_overrides.delay( + kwargs={ + "environment_api_key": self.environment_api_key, + "changes": changes, + "identity_uuid": str(self.identity_uuid), + } + ) self._reset_initial_state() def synchronise_features(self, valid_feature_names: typing.Collection[str]) -> None: @@ -187,7 +198,8 @@ def synchronise_features(self, valid_feature_names: typing.Collection[str]) -> N def to_document(self) -> dict: return map_engine_identity_to_identity_document(self._engine_identity_model) - def _get_changes(self, previous_instance: "EdgeIdentity") -> dict: + def _get_changes(self) -> IdentityChangeset: + previous_instance = self._initial_state changes = {} feature_changes = changes.setdefault("feature_overrides", {}) previous_feature_overrides = { @@ -201,7 +213,9 @@ def _get_changes(self, previous_instance: "EdgeIdentity") -> dict: current_matching_fs = current_feature_overrides.get(uuid_) if current_matching_fs is None: feature_changes[previous_fs.feature.name] = generate_change_dict( - change_type="-", identity=self, old=previous_fs + change_type="-", + identity_id=self.id, + old=previous_fs, ) elif ( current_matching_fs.enabled != previous_fs.enabled @@ -210,7 +224,7 @@ def _get_changes(self, previous_instance: "EdgeIdentity") -> dict: ): feature_changes[previous_fs.feature.name] = generate_change_dict( change_type="~", - identity=self, + identity_id=self.id, new=current_matching_fs, old=previous_fs, ) @@ -218,7 +232,9 @@ def _get_changes(self, previous_instance: "EdgeIdentity") -> dict: for uuid_, previous_fs in current_feature_overrides.items(): if uuid_ not in previous_feature_overrides: feature_changes[previous_fs.feature.name] = generate_change_dict( - change_type="+", identity=self, new=previous_fs + change_type="+", + identity_id=self.id, + new=previous_fs, ) return changes diff --git a/api/edge_api/identities/tasks.py b/api/edge_api/identities/tasks.py index 6f1ff63ead0c..22f98f0738e0 100644 --- a/api/edge_api/identities/tasks.py +++ b/api/edge_api/identities/tasks.py @@ -5,11 +5,14 @@ from audit.models import AuditLog from audit.related_object_type import RelatedObjectType +from edge_api.identities.types import IdentityChangeset +from environments.dynamodb.dynamodb_wrapper import DynamoEnvironmentV2Wrapper from environments.models import Environment, Webhook from features.models import Feature, FeatureState from task_processor.decorators import register_task_handler from task_processor.models import TaskPriority from users.models import FFAdminUser +from util.mappers import map_identity_changeset_to_identity_override_changeset from webhooks.webhooks import WebhookEventType, call_environment_webhooks logger = logging.getLogger(__name__) @@ -89,7 +92,7 @@ def sync_identity_document_features(identity_uuid: str): ) identity.synchronise_features(valid_feature_names) - EdgeIdentity.dynamo_wrapper.put_item(identity.to_document()) + identity.save() @register_task_handler() @@ -97,13 +100,13 @@ def generate_audit_log_records( environment_api_key: str, identifier: str, identity_uuid: str, - changes: dict, - user_id: int = None, - master_api_key_id: int = None, -): + changes: IdentityChangeset, + user_id: int | None = None, + master_api_key_id: int | None = None, +) -> None: audit_records = [] - feature_override_changes = changes.get("feature_overrides") + feature_override_changes = changes["feature_overrides"] if not feature_override_changes: return @@ -113,7 +116,7 @@ def generate_audit_log_records( for feature_name, change_details in feature_override_changes.items(): action = {"+": "created", "-": "deleted", "~": "updated"}.get( - change_details.get("change_type") + change_details["change_type"] ) log = f"Feature override {action} for feature '{feature_name}' and identity '{identifier}'" audit_records.append( @@ -130,3 +133,25 @@ def generate_audit_log_records( ) AuditLog.objects.bulk_create(audit_records) + + +@register_task_handler() +def update_flagsmith_environments_v2_identity_overrides( + environment_api_key: str, + identity_uuid: str, + changes: IdentityChangeset, +) -> None: + feature_override_changes = changes["feature_overrides"] + if not feature_override_changes: + return + + environment = Environment.objects.get(api_key=environment_api_key) + dynamodb_wrapper_v2 = DynamoEnvironmentV2Wrapper() + + identity_override_changeset = map_identity_changeset_to_identity_override_changeset( + identity_changeset=changes, + identity_uuid=identity_uuid, + environment_api_key=environment_api_key, + environment_id=environment.id, + ) + dynamodb_wrapper_v2.update_identity_overrides(identity_override_changeset) diff --git a/api/edge_api/identities/types.py b/api/edge_api/identities/types.py new file mode 100644 index 000000000000..22017c62a27a --- /dev/null +++ b/api/edge_api/identities/types.py @@ -0,0 +1,15 @@ +from typing import Any, Literal, TypedDict + +from typing_extensions import NotRequired + +ChangeType = Literal["+", "-", "~"] + + +class FeatureStateChangeDetails(TypedDict): + change_type: ChangeType + old: NotRequired[dict[str, Any]] + new: NotRequired[dict[str, Any]] + + +class IdentityChangeset(TypedDict): + feature_overrides: dict[str, FeatureStateChangeDetails] diff --git a/api/edge_api/identities/utils.py b/api/edge_api/identities/utils.py new file mode 100644 index 000000000000..d9c0b2a3499d --- /dev/null +++ b/api/edge_api/identities/utils.py @@ -0,0 +1,40 @@ +import typing + +from flag_engine.features.models import FeatureStateModel + +if typing.TYPE_CHECKING: + from edge_api.identities.types import ChangeType, FeatureStateChangeDetails + + +def generate_change_dict( + change_type: "ChangeType", + identity_id: int | str | None, + new: FeatureStateModel | None = None, + old: FeatureStateModel | None = None, +) -> "FeatureStateChangeDetails": + if not (new or old): + raise ValueError("Must provide one of 'new' or 'old'") + + change_dict = {"change_type": change_type} + if new: + change_dict["new"] = _get_overridden_feature_state_dict( + identity_id=identity_id, + feature_state=new, + ) + if old: + change_dict["old"] = _get_overridden_feature_state_dict( + identity_id=identity_id, + feature_state=old, + ) + + return change_dict + + +def _get_overridden_feature_state_dict( + identity_id: int | str | None, + feature_state: FeatureStateModel, +) -> dict[str, typing.Any]: + return { + **feature_state.dict(), + "feature_state_value": feature_state.get_value(identity_id), + } diff --git a/api/environments/dynamodb/constants.py b/api/environments/dynamodb/constants.py new file mode 100644 index 000000000000..7f59ab6086e3 --- /dev/null +++ b/api/environments/dynamodb/constants.py @@ -0,0 +1,5 @@ +ENVIRONMENTS_V2_PARTITION_KEY = "environment_id" +ENVIRONMENTS_V2_SORT_KEY = "document_key" + +ENVIRONMENTS_V2_SECONDARY_INDEX = "environment_api_key-index" +ENVIRONMENTS_V2_SECONDARY_INDEX_PARTITION_KEY = "environment_api_key" diff --git a/api/environments/dynamodb/dynamodb_wrapper.py b/api/environments/dynamodb/dynamodb_wrapper.py index a589a4aa43c0..0158c775128d 100644 --- a/api/environments/dynamodb/dynamodb_wrapper.py +++ b/api/environments/dynamodb/dynamodb_wrapper.py @@ -1,7 +1,7 @@ import logging import typing from contextlib import suppress -from typing import Iterable +from typing import Any, Iterable import boto3 from boto3.dynamodb.conditions import Key @@ -14,9 +14,18 @@ from flag_engine.segments.evaluator import get_identity_segments from rest_framework.exceptions import NotFound +from environments.dynamodb.constants import ( + ENVIRONMENTS_V2_PARTITION_KEY, + ENVIRONMENTS_V2_SORT_KEY, +) +from environments.dynamodb.types import IdentityOverridesV2Changeset +from environments.dynamodb.utils import ( + get_environments_v2_identity_override_document_key, +) from util.mappers import ( map_environment_api_key_to_environment_api_key_document, map_environment_to_environment_document, + map_identity_override_to_identity_override_document, map_identity_to_identity_document, ) @@ -32,7 +41,7 @@ class BaseDynamoWrapper: def __init__(self): self._table = None - if table_name := self.table_name: + if table_name := self.get_table_name(): self._table = boto3.resource( "dynamodb", config=Config(tcp_keepalive=True) ).Table(table_name) @@ -41,6 +50,9 @@ def __init__(self): def is_enabled(self) -> bool: return self._table is not None + def get_table_name(self): + return self.table_name + class DynamoIdentityWrapper(BaseDynamoWrapper): table_name = settings.IDENTITIES_TABLE_NAME_DYNAMO @@ -167,44 +179,48 @@ def get_item(self, api_key: str) -> dict: class DynamoEnvironmentV2Wrapper(BaseDynamoEnvironmentWrapper): - table_name = settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO - - ENVIRONMENT_ID_ATTRIBUTE = "environment_id" - DOCUMENT_KEY_ATTRIBUTE = "document_key" - ENVIRONMENT_API_KEY_ATTRIBUTE = "environment_api_key" - ENVIRONMENT_API_KEY_INDEX_NAME = "environment_api_key-index" - - def get_environment_by_api_key(self, environment_api_key: str) -> dict: - get_item_kwargs = { - "IndexName": self.ENVIRONMENT_API_KEY_INDEX_NAME, - "Key": { - self.ENVIRONMENT_API_KEY_ATTRIBUTE: environment_api_key, - self.DOCUMENT_KEY_ATTRIBUTE: "META", - }, - } - try: - return self._table.get_item(**get_item_kwargs)["Item"] - except IndexError: - raise ObjectDoesNotExist() - - def get_identity_overrides( - self, environment_id: int, feature_id: int = None - ) -> typing.List[dict]: # TODO better typing? - document_key_begins_with = "identity_override" - if feature_id: - document_key_begins_with += f":{feature_id}" - key_expression_condition = Key(self.ENVIRONMENT_ID_ATTRIBUTE).eq( - environment_id - ) & Key(self.DOCUMENT_KEY_ATTRIBUTE).begins_with(document_key_begins_with) + def get_table_name(self): + return settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO + def get_identity_overrides_by_feature_id( + self, + environment_id: int, + feature_id: int, + ) -> typing.List[dict[str, Any]]: try: response = self._table.query( - KeyConditionExpression=key_expression_condition + KeyConditionExpression=Key(ENVIRONMENTS_V2_PARTITION_KEY).eq( + str(environment_id), + ) + & Key(ENVIRONMENTS_V2_SORT_KEY).begins_with( + get_environments_v2_identity_override_document_key( + feature_id=feature_id, + ), + ) ) return response["Items"] except KeyError as e: raise ObjectDoesNotExist() from e + def update_identity_overrides( + self, + changeset: IdentityOverridesV2Changeset, + ) -> None: + with self._table.batch_writer() as writer: + for identity_override_to_delete in changeset.to_delete: + writer.delete_item( + Key={ + ENVIRONMENTS_V2_PARTITION_KEY: identity_override_to_delete.environment_id, + ENVIRONMENTS_V2_SORT_KEY: identity_override_to_delete.document_key, + }, + ) + for identity_override_to_put in changeset.to_put: + writer.put_item( + Item=map_identity_override_to_identity_override_document( + identity_override_to_put + ), + ) + class DynamoEnvironmentAPIKeyWrapper(BaseDynamoWrapper): table_name = settings.ENVIRONMENTS_API_KEY_TABLE_NAME_DYNAMO diff --git a/api/environments/dynamodb/types.py b/api/environments/dynamodb/types.py index 1dd3804f3fad..d43d124e88e1 100644 --- a/api/environments/dynamodb/types.py +++ b/api/environments/dynamodb/types.py @@ -4,6 +4,8 @@ import boto3 from django.conf import settings +from flag_engine.features.models import FeatureStateModel +from pydantic import BaseModel project_metadata_table = None @@ -69,3 +71,16 @@ def finish_identity_migration(self): def _save(self): return project_metadata_table.put_item(Item=asdict(self)) + + +class IdentityOverrideV2(BaseModel): + environment_id: str + document_key: str + environment_api_key: str + feature_state: FeatureStateModel + + +@dataclass +class IdentityOverridesV2Changeset: + to_delete: list[IdentityOverrideV2] + to_put: list[IdentityOverrideV2] diff --git a/api/environments/dynamodb/utils.py b/api/environments/dynamodb/utils.py new file mode 100644 index 000000000000..40c1f14cbe4f --- /dev/null +++ b/api/environments/dynamodb/utils.py @@ -0,0 +1,23 @@ +from multimethod import overload + +# TODO This might require type: ignores in the future, but it's just so nice! + + +@overload +def get_environments_v2_identity_override_document_key() -> str: + return "identity_override:" + + +@overload +def get_environments_v2_identity_override_document_key( # noqa: F811 + feature_id: int, +) -> str: + return f"identity_override:{feature_id}:" + + +@overload +def get_environments_v2_identity_override_document_key( # noqa: F811 + feature_id: int, + identity_uuid: str, +) -> str: + return f"identity_override:{feature_id}:{identity_uuid}" diff --git a/api/poetry.lock b/api/poetry.lock index 40135a469ae3..2500e31dd8d9 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2094,6 +2094,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -2335,6 +2345,31 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] +[[package]] +name = "multimethod" +version = "1.10" +description = "Multiple argument dispatching." +optional = false +python-versions = ">=3.8" +files = [ + {file = "multimethod-1.10-py3-none-any.whl", hash = "sha256:afd84da9c3d0445c84f827e4d63ad42d17c6d29b122427c6dee9032ac2d2a0d4"}, + {file = "multimethod-1.10.tar.gz", hash = "sha256:daa45af3fe257f73abb69673fd54ddeaf31df0eb7363ad6e1251b7c9b192d8c5"}, +] + +[[package]] +name = "mypy-boto3-dynamodb" +version = "1.33.0" +description = "Type annotations for boto3.DynamoDB 1.33.0 service generated with mypy-boto3-builder 7.20.3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mypy-boto3-dynamodb-1.33.0.tar.gz", hash = "sha256:2cfe1089c89de61b1ec0e69a72ba3e6865a013ea0a37d318ab564983785d42f9"}, + {file = "mypy_boto3_dynamodb-1.33.0-py3-none-any.whl", hash = "sha256:619ea2cc311ced0ecb44b6e8d3bf3dd851fb7c53a34128b4ff6d6e6a11fdd41f"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -4393,4 +4428,4 @@ requests = ">=2.7,<3.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "a11fec03de5c44c28448991c66bfc740cf0b97fa5d0a71629cd9a7a95b49f81d" +content-hash = "c4f455c206a45222736db051afd1ab7f00a37748ac7234253d4225b8b60764f7" diff --git a/api/pyproject.toml b/api/pyproject.toml index 096081d8cf05..d08402f83319 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -103,6 +103,7 @@ pydantic = "~1.10.9" pyngo = "~1.6.0" flagsmith = "^3.4.0" python-gnupg = "^0.5.1" +multimethod = "^1.10" [tool.poetry.group.auth-controller] optional = true @@ -145,6 +146,7 @@ requests-mock = "^1.11.0" django-extensions = "^3.2.3" pdbpp = "^0.10.3" django-capture-on-commit-callbacks = "^1.11.0" +mypy-boto3-dynamodb = "^1.33.0" [build-system] requires = ["poetry-core>=1.5.0"] diff --git a/api/task_processor/decorators.py b/api/task_processor/decorators.py index 6819b85d1fd9..6cfe235fe181 100644 --- a/api/task_processor/decorators.py +++ b/api/task_processor/decorators.py @@ -6,6 +6,7 @@ from threading import Thread from django.conf import settings +from django.core.serializers.json import DjangoJSONEncoder from django.db.transaction import on_commit from django.utils import timezone @@ -18,6 +19,8 @@ logger = logging.getLogger(__name__) +_django_json_encoder_default = DjangoJSONEncoder().default + class TaskHandler(typing.Generic[P]): __slots__ = ( diff --git a/api/task_processor/models.py b/api/task_processor/models.py index a8dcf99e3660..09ec1c584f68 100644 --- a/api/task_processor/models.py +++ b/api/task_processor/models.py @@ -3,6 +3,7 @@ from datetime import datetime import simplejson as json +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils import timezone @@ -10,6 +11,8 @@ from task_processor.managers import RecurringTaskManager, TaskManager from task_processor.task_registry import registered_tasks +_django_json_encoder_default = DjangoJSONEncoder().default + class TaskPriority(models.IntegerChoices): LOWER = 100 @@ -44,8 +47,7 @@ def kwargs(self) -> typing.Dict[str, typing.Any]: @staticmethod def serialize_data(data: typing.Any): - # TODO: add datetime support if needed - return json.dumps(data) + return json.dumps(data, default=_django_json_encoder_default) @staticmethod def deserialize_data(data: typing.Any): diff --git a/api/tests/integration/edge_api/identities/test_edge_identity_featurestates_viewset.py b/api/tests/integration/edge_api/identities/test_edge_identity_featurestates_viewset.py index 3c651b91ab08..ecb7a3d18f13 100644 --- a/api/tests/integration/edge_api/identities/test_edge_identity_featurestates_viewset.py +++ b/api/tests/integration/edge_api/identities/test_edge_identity_featurestates_viewset.py @@ -224,6 +224,7 @@ def test_edge_identities_featurestate_detail_calls_sync_identity_if_deleted_feat def test_edge_identities_featurestate_delete( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -325,6 +326,7 @@ def test_edge_identities_create_featurestate_returns_400_if_feature_state_alread def test_edge_identities_create_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -388,6 +390,7 @@ def test_edge_identities_create_featurestate( def test_edge_identities_create_mv_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -467,6 +470,7 @@ def test_edge_identities_create_mv_featurestate( def test_edge_identities_update_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -555,6 +559,7 @@ def test_edge_identities_patch_returns_405( def test_edge_identities_update_mv_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -701,6 +706,7 @@ def test_edge_identities_post_returns_400_for_invalid_mvfs_allocation( "lazy_feature", [(lazy_fixture("feature")), (lazy_fixture("feature_name"))] ) def test_edge_identities_with_identifier_create_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -766,6 +772,7 @@ def test_edge_identities_with_identifier_create_featurestate( "lazy_feature", [(lazy_fixture("feature")), (lazy_fixture("feature_name"))] ) def test_edge_identities_with_identifier_delete_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, @@ -807,6 +814,7 @@ def test_edge_identities_with_identifier_delete_featurestate( def test_edge_identities_with_identifier_update_featurestate( + dynamodb_wrapper_v2, admin_client, environment, environment_api_key, diff --git a/api/tests/unit/edge_api/identities/test_edge_api_identities_serializers.py b/api/tests/unit/edge_api/identities/test_edge_api_identities_serializers.py index 4a77d199d2c8..c3cdfc79f5b4 100644 --- a/api/tests/unit/edge_api/identities/test_edge_api_identities_serializers.py +++ b/api/tests/unit/edge_api/identities/test_edge_api_identities_serializers.py @@ -30,6 +30,7 @@ def test_edge_identity_feature_state_serializer_save_allows_missing_mvfsvs( mock_dynamo_wrapper = mocker.patch( "edge_api.identities.serializers.EdgeIdentity.dynamo_wrapper" ) + mocker.patch("edge_api.identities.tasks.DynamoEnvironmentV2Wrapper") # When serializer.is_valid(raise_exception=True) @@ -73,6 +74,7 @@ def test_edge_identity_feature_state_serializer_save_calls_webhook_for_new_overr ) mocker.patch("edge_api.identities.serializers.EdgeIdentity.dynamo_wrapper") + mocker.patch("edge_api.identities.tasks.DynamoEnvironmentV2Wrapper") mock_call_environment_webhook = mocker.patch( "edge_api.identities.serializers.call_environment_webhook_for_feature_state_change" ) diff --git a/api/tests/unit/edge_api/identities/test_edge_identity_models.py b/api/tests/unit/edge_api/identities/test_edge_identity_models.py index e0f3e5140559..97ed26cffe57 100644 --- a/api/tests/unit/edge_api/identities/test_edge_identity_models.py +++ b/api/tests/unit/edge_api/identities/test_edge_identity_models.py @@ -1,10 +1,12 @@ from datetime import timedelta +from unittest.mock import MagicMock import pytest import shortuuid from django.utils import timezone from flag_engine.features.models import FeatureModel, FeatureStateModel from freezegun import freeze_time +from pytest_mock import MockerFixture from edge_api.identities.models import EdgeIdentity from features.models import FeatureSegment, FeatureState, FeatureStateValue @@ -263,13 +265,18 @@ def test_edge_identity_save_does_not_generate_audit_records_if_no_changes( mocked_generate_audit_log_records.delay.assert_not_called() -def test_edge_identity_save_called_generate_audit_records_if_feature_override_added( - mocker, edge_identity_model, edge_identity_dynamo_wrapper_mock -): +def test_edge_identity_save_called__feature_override_added__expected_tasks_called( + mocker: MockerFixture, + edge_identity_model: EdgeIdentity, + edge_identity_dynamo_wrapper_mock: MagicMock, +) -> None: # Given mocked_generate_audit_log_records = mocker.patch( "edge_api.identities.models.generate_audit_log_records" ) + mocked_update_flagsmith_environments_v2_identity_overrides = mocker.patch( + "edge_api.identities.models.update_flagsmith_environments_v2_identity_overrides" + ) feature_state_model = FeatureStateModel( feature=FeatureModel(id=1, name="test_feature", type="STANDARD"), @@ -279,6 +286,20 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_ad user = mocker.MagicMock() + expected_changes = { + "feature_overrides": { + "test_feature": { + "change_type": "+", + "new": { + **feature_state_model.dict(), + "enabled": True, + "feature_state_value": None, + }, + } + } + } + expected_identity_uuid = str(edge_identity_model.identity_uuid) + # When edge_identity_model.save(user=user) @@ -289,27 +310,32 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_ad "environment_api_key": edge_identity_model.environment_api_key, "identifier": edge_identity_model.identifier, "user_id": user.id, - "changes": { - "feature_overrides": { - "test_feature": { - "change_type": "+", - "new": {"enabled": True, "value": None}, - } - } - }, - "identity_uuid": str(edge_identity_model.identity_uuid), + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, "master_api_key_id": None, } ) + mocked_update_flagsmith_environments_v2_identity_overrides.delay.assert_called_once_with( + kwargs={ + "environment_api_key": edge_identity_model.environment_api_key, + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, + } + ) -def test_edge_identity_save_called_generate_audit_records_if_feature_override_removed( - mocker, edge_identity_model, edge_identity_dynamo_wrapper_mock -): +def test_edge_identity_save_called__feature_override_removed__expected_tasks_called( + mocker: MockerFixture, + edge_identity_model: EdgeIdentity, + edge_identity_dynamo_wrapper_mock: MagicMock, +) -> None: # Given mocked_generate_audit_log_records = mocker.patch( "edge_api.identities.models.generate_audit_log_records" ) + mocked_update_flagsmith_environments_v2_identity_overrides = mocker.patch( + "edge_api.identities.models.update_flagsmith_environments_v2_identity_overrides" + ) feature_state_model = FeatureStateModel( feature=FeatureModel(id=1, name="test_feature", type="STANDARD"), @@ -319,9 +345,24 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_re user = mocker.MagicMock() + expected_changes = { + "feature_overrides": { + "test_feature": { + "change_type": "-", + "old": { + **feature_state_model.dict(), + "enabled": True, + "feature_state_value": None, + }, + } + } + } + expected_identity_uuid = str(edge_identity_model.identity_uuid) + edge_identity_model.save(user=user) edge_identity_dynamo_wrapper_mock.reset_mock() mocked_generate_audit_log_records.reset_mock() + mocked_update_flagsmith_environments_v2_identity_overrides.reset_mock() edge_identity_model.remove_feature_override(feature_state_model) @@ -335,18 +376,18 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_re "environment_api_key": edge_identity_model.environment_api_key, "identifier": edge_identity_model.identifier, "user_id": user.id, - "changes": { - "feature_overrides": { - "test_feature": { - "change_type": "-", - "old": {"enabled": True, "value": None}, - } - } - }, - "identity_uuid": str(edge_identity_model.identity_uuid), + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, "master_api_key_id": None, } ) + mocked_update_flagsmith_environments_v2_identity_overrides.delay.assert_called_once_with( + kwargs={ + "environment_api_key": edge_identity_model.environment_api_key, + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, + } + ) @pytest.mark.parametrize( @@ -358,18 +399,21 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_re ), ) def test_edge_identity_save_called_generate_audit_records_if_feature_override_updated( - initial_enabled, - initial_value, - new_enabled, - new_value, - mocker, - edge_identity_model, - edge_identity_dynamo_wrapper_mock, -): + initial_enabled: bool, + initial_value: str, + new_enabled: bool, + new_value: str, + mocker: MockerFixture, + edge_identity_model: EdgeIdentity, + edge_identity_dynamo_wrapper_mock: MagicMock, +) -> None: # Given mocked_generate_audit_log_records = mocker.patch( "edge_api.identities.models.generate_audit_log_records" ) + mocked_update_flagsmith_environments_v2_identity_overrides = mocker.patch( + "edge_api.identities.models.update_flagsmith_environments_v2_identity_overrides" + ) feature_state_model = FeatureStateModel( feature=FeatureModel(id=1, name="test_feature", type="STANDARD"), @@ -380,9 +424,29 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_up user = mocker.MagicMock() + expected_changes = { + "feature_overrides": { + "test_feature": { + "change_type": "~", + "old": { + **feature_state_model.dict(), + "enabled": initial_enabled, + "feature_state_value": initial_value, + }, + "new": { + **feature_state_model.dict(), + "enabled": new_enabled, + "feature_state_value": new_value, + }, + } + } + } + expected_identity_uuid = str(edge_identity_model.identity_uuid) + edge_identity_model.save(user=user) edge_identity_dynamo_wrapper_mock.reset_mock() mocked_generate_audit_log_records.reset_mock() + mocked_update_flagsmith_environments_v2_identity_overrides.reset_mock() feature_override = edge_identity_model.get_feature_state_by_featurestate_uuid( str(feature_state_model.featurestate_uuid) @@ -400,16 +464,15 @@ def test_edge_identity_save_called_generate_audit_records_if_feature_override_up "environment_api_key": edge_identity_model.environment_api_key, "identifier": edge_identity_model.identifier, "user_id": user.id, - "changes": { - "feature_overrides": { - "test_feature": { - "change_type": "~", - "old": {"enabled": initial_enabled, "value": initial_value}, - "new": {"enabled": new_enabled, "value": new_value}, - } - } - }, - "identity_uuid": str(edge_identity_model.identity_uuid), + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, "master_api_key_id": None, } ) + mocked_update_flagsmith_environments_v2_identity_overrides.delay.assert_called_once_with( + kwargs={ + "environment_api_key": edge_identity_model.environment_api_key, + "changes": expected_changes, + "identity_uuid": expected_identity_uuid, + } + ) diff --git a/api/tests/unit/edge_api/identities/test_unit_edge_api_identities_tasks.py b/api/tests/unit/edge_api/identities/test_unit_edge_api_identities_tasks.py index 937cb734cf30..91958089eea9 100644 --- a/api/tests/unit/edge_api/identities/test_unit_edge_api_identities_tasks.py +++ b/api/tests/unit/edge_api/identities/test_unit_edge_api_identities_tasks.py @@ -2,6 +2,7 @@ import pytest from django.utils import timezone +from pytest_mock import MockerFixture from audit.models import AuditLog from audit.related_object_type import RelatedObjectType @@ -9,8 +10,13 @@ call_environment_webhook_for_feature_state_change, generate_audit_log_records, sync_identity_document_features, + update_flagsmith_environments_v2_identity_overrides, ) -from environments.models import Webhook +from environments.dynamodb.types import ( + IdentityOverridesV2Changeset, + IdentityOverrideV2, +) +from environments.models import Environment, Webhook from webhooks.webhooks import WebhookEventType @@ -283,8 +289,8 @@ def test_sync_identity_document_features_removes_deleted_features( "feature_overrides": { "test_feature": { "change_type": "~", - "old": {"enabled": False, "value": None}, - "new": {"enabled": True, "value": None}, + "old": {"enabled": False, "feature_state_value": None}, + "new": {"enabled": True, "feature_state_value": None}, } } }, @@ -296,7 +302,7 @@ def test_sync_identity_document_features_removes_deleted_features( "feature_overrides": { "test_feature": { "change_type": "+", - "new": {"enabled": True, "value": None}, + "new": {"enabled": True, "feature_state_value": None}, } } }, @@ -308,7 +314,7 @@ def test_sync_identity_document_features_removes_deleted_features( "feature_overrides": { "test_feature": { "change_type": "-", - "old": {"enabled": True, "value": None}, + "old": {"enabled": True, "feature_state_value": None}, } } }, @@ -339,3 +345,144 @@ def test_generate_audit_log_records( related_object_uuid=identity_uuid, environment=environment, ).exists() + + +def test_update_flagsmith_environments_v2_identity_overrides__call_expected( + mocker: MockerFixture, + environment: Environment, +) -> None: + # Given + dynamodb_wrapper_v2_cls_mock = mocker.patch( + "edge_api.identities.tasks.DynamoEnvironmentV2Wrapper" + ) + dynamodb_wrapper_v2_mock = dynamodb_wrapper_v2_cls_mock.return_value + identity_uuid = "a35a02f2-fefd-4932-8f5c-e84a0bf542c7" + changes = { + "feature_overrides": { + "test_feature": { + "change_type": "~", + "old": { + "enabled": False, + "feature_state_value": None, + "featurestate_uuid": "0729f130-8caa-4106-aa5c-95a6d15e820f", + "feature": {"id": 1, "name": "test_feature", "type": "STANDARD"}, + }, + "new": { + "enabled": True, + "feature_state_value": "updated", + "featurestate_uuid": "0729f130-8caa-4106-aa5c-95a6d15e820f", + "feature": {"id": 1, "name": "test_feature", "type": "STANDARD"}, + }, + }, + "test_feature2": { + "change_type": "+", + "new": { + "enabled": True, + "feature_state_value": "new", + "featurestate_uuid": "726c833a-5c9b-4c2c-954c-ddc46dd50bbb", + "feature": {"id": 2, "name": "test_feature2", "type": "STANDARD"}, + }, + }, + "test_feature3": { + "change_type": "-", + "old": { + "enabled": True, + "feature_state_value": "deleted", + "featurestate_uuid": "80f6dbdd-97c0-47de-9333-cd1e1c100713", + "feature": {"id": 3, "name": "test_feature3", "type": "STANDARD"}, + }, + }, + } + } + expected_identity_overrides_changeset = IdentityOverridesV2Changeset( + to_delete=[ + IdentityOverrideV2.parse_obj( + { + "document_key": f"identity_override:3:{identity_uuid}", + "environment_id": str(environment.id), + "environment_api_key": environment.api_key, + "feature_state": { + "enabled": True, + "feature_state_value": "deleted", + "featurestate_uuid": "80f6dbdd-97c0-47de-9333-cd1e1c100713", + "feature": { + "id": 3, + "name": "test_feature3", + "type": "STANDARD", + }, + }, + } + ) + ], + to_put=[ + IdentityOverrideV2.parse_obj( + { + "document_key": f"identity_override:1:{identity_uuid}", + "environment_id": str(environment.id), + "environment_api_key": environment.api_key, + "feature_state": { + "enabled": True, + "feature_state_value": "updated", + "featurestate_uuid": "0729f130-8caa-4106-aa5c-95a6d15e820f", + "feature": { + "id": 1, + "name": "test_feature", + "type": "STANDARD", + }, + }, + } + ), + IdentityOverrideV2.parse_obj( + { + "document_key": f"identity_override:2:{identity_uuid}", + "environment_id": str(environment.id), + "environment_api_key": environment.api_key, + "feature_state": { + "enabled": True, + "feature_state_value": "new", + "featurestate_uuid": "726c833a-5c9b-4c2c-954c-ddc46dd50bbb", + "feature": { + "id": 2, + "name": "test_feature2", + "type": "STANDARD", + }, + }, + } + ), + ], + ) + + # When + update_flagsmith_environments_v2_identity_overrides( + environment_api_key=environment.api_key, + identity_uuid=identity_uuid, + changes=changes, + ) + + # Then + dynamodb_wrapper_v2_mock.update_identity_overrides.assert_called_once_with( + expected_identity_overrides_changeset, + ) + + +def test_update_flagsmith_environments_v2_identity_overrides__no_overrides__call_expected( + mocker: MockerFixture, + environment: Environment, +) -> None: + # Given + dynamodb_wrapper_v2_cls_mock = mocker.patch( + "edge_api.identities.tasks.DynamoEnvironmentV2Wrapper" + ) + dynamodb_wrapper_v2_mock = dynamodb_wrapper_v2_cls_mock.return_value + identity_uuid = "a35a02f2-fefd-4932-8f5c-e84a0bf542c7" + changes = {"feature_overrides": []} + + # When + update_flagsmith_environments_v2_identity_overrides( + environment_api_key=environment.api_key, + identity_uuid=identity_uuid, + changes=changes, + ) + + # Then + dynamodb_wrapper_v2_mock.update_identity_overrides.assert_not_called() diff --git a/api/tests/unit/environments/dynamodb/test_unit_dynamodb_environment_v2_wrapper.py b/api/tests/unit/environments/dynamodb/test_unit_dynamodb_environment_v2_wrapper.py new file mode 100644 index 000000000000..01825897ebfa --- /dev/null +++ b/api/tests/unit/environments/dynamodb/test_unit_dynamodb_environment_v2_wrapper.py @@ -0,0 +1,130 @@ +import uuid + +from mypy_boto3_dynamodb.service_resource import Table +from pytest_django.fixtures import SettingsWrapper + +from environments.dynamodb.dynamodb_wrapper import DynamoEnvironmentV2Wrapper +from environments.dynamodb.types import ( + IdentityOverridesV2Changeset, + IdentityOverrideV2, +) +from environments.models import Environment +from features.models import Feature, FeatureState +from util.mappers import ( + map_environment_to_environment_v2_document, + map_feature_state_to_engine, + map_identity_override_to_identity_override_document, +) + + +def test_environment_v2_wrapper__get_identity_overrides__return_expected( + settings: SettingsWrapper, + environment: Environment, + flagsmith_environments_v2_table: Table, + feature: Feature, +) -> None: + # Given + settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO = flagsmith_environments_v2_table.name + wrapper = DynamoEnvironmentV2Wrapper() + + identity_uuid = str(uuid.uuid4()) + identifier = "identity1" + override_document = { + "environment_id": str(environment.id), + "document_key": f"identity_override:{feature.id}:{identity_uuid}", + "environment_api_key": environment.api_key, + "identifier": identifier, + "feature_state": {}, + } + + environment_document = map_environment_to_environment_v2_document(environment) + + flagsmith_environments_v2_table.put_item(Item=override_document) + flagsmith_environments_v2_table.put_item(Item=environment_document) + + # When + results = wrapper.get_identity_overrides_by_feature_id( + environment_id=environment.id, + feature_id=feature.id, + ) + + # Then + assert len(results) == 1 + assert results[0] == override_document + + +def test_environment_v2_wrapper__update_identity_overrides__put_expected( + settings: SettingsWrapper, + environment: Environment, + flagsmith_environments_v2_table: Table, + feature: Feature, + feature_state: FeatureState, +) -> None: + # Given + settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO = flagsmith_environments_v2_table.name + wrapper = DynamoEnvironmentV2Wrapper() + + identity_uuid = str(uuid.uuid4()) + override_document = IdentityOverrideV2.parse_obj( + { + "environment_id": str(environment.id), + "document_key": f"identity_override:{feature.id}:{identity_uuid}", + "environment_api_key": environment.api_key, + "feature_state": map_feature_state_to_engine(feature_state), + } + ) + + # When + wrapper.update_identity_overrides( + changeset=IdentityOverridesV2Changeset( + to_delete=[], + to_put=[override_document], + ), + ) + + # Then + results = flagsmith_environments_v2_table.scan()["Items"] + assert len(results) == 1 + assert results[0] == map_identity_override_to_identity_override_document( + override_document, + ) + + +def test_environment_v2_wrapper__update_identity_overrides__delete_expected( + settings: SettingsWrapper, + environment: Environment, + flagsmith_environments_v2_table: Table, + feature: Feature, + feature_state: FeatureState, +) -> None: + # Given + settings.ENVIRONMENTS_V2_TABLE_NAME_DYNAMO = flagsmith_environments_v2_table.name + wrapper = DynamoEnvironmentV2Wrapper() + + identity_uuid = str(uuid.uuid4()) + override_document_data = map_identity_override_to_identity_override_document( + IdentityOverrideV2.parse_obj( + { + "environment_id": str(environment.id), + "document_key": f"identity_override:{feature.id}:{identity_uuid}", + "environment_api_key": environment.api_key, + "feature_state": map_feature_state_to_engine(feature_state), + } + ) + ) + + flagsmith_environments_v2_table.put_item(Item=override_document_data) + + override_document = IdentityOverrideV2.parse_obj(override_document_data) + + # When + wrapper.update_identity_overrides( + changeset=IdentityOverridesV2Changeset( + to_delete=[override_document], + to_put=[], + ), + ) + + # Then + results = flagsmith_environments_v2_table.scan()["Items"] + assert len(results) == 0 diff --git a/api/tests/unit/util/mappers/test_unit_mappers_dynamodb.py b/api/tests/unit/util/mappers/test_unit_mappers_dynamodb.py index f11ccd3083a1..cd1100b751f7 100644 --- a/api/tests/unit/util/mappers/test_unit_mappers_dynamodb.py +++ b/api/tests/unit/util/mappers/test_unit_mappers_dynamodb.py @@ -127,3 +127,67 @@ def test_map_identity_to_identity_document__call_expected( "identity_uuid": mocker.ANY, } assert uuid.UUID(result["identity_uuid"]) + + +def test_map_environment_to_environment_v2_document__call_expected( + environment: "Environment", + feature_state: "FeatureState", +) -> None: + # Given + expected_api_key = environment.api_key + expected_updated_at = environment.updated_at.isoformat() + expected_featurestate_uuid = str(feature_state.uuid) + + # When + result = dynamodb.map_environment_to_environment_v2_document(environment) + + # Then + assert result == { + "document_key": "META", + "environment_id": str(environment.id), + "allow_client_traits": True, + "amplitude_config": None, + "api_key": expected_api_key, + "dynatrace_config": None, + "feature_states": [ + { + "django_id": Decimal(feature_state.pk), + "enabled": False, + "feature": { + "id": Decimal(feature_state.feature.pk), + "name": "Test Feature1", + "type": "STANDARD", + }, + "feature_segment": None, + "feature_state_value": None, + "featurestate_uuid": expected_featurestate_uuid, + "multivariate_feature_state_values": [], + } + ], + "heap_config": None, + "hide_disabled_flags": None, + "hide_sensitive_data": False, + "id": Decimal(environment.pk), + "mixpanel_config": None, + "name": "Test Environment", + "project": { + "enable_realtime_updates": False, + "hide_disabled_flags": False, + "id": Decimal(environment.project.pk), + "name": "Test Project", + "organisation": { + "feature_analytics": False, + "id": Decimal(environment.project.organisation.pk), + "name": "Test Org", + "persist_trait_data": True, + "stop_serving_flags": False, + }, + "segments": [], + "server_key_only_feature_ids": [], + }, + "rudderstack_config": None, + "segment_config": None, + "updated_at": expected_updated_at, + "use_identity_composite_key_for_hashing": True, + "webhook_config": None, + } diff --git a/api/tests/unit/util/mappers/test_unit_mappers_engine.py b/api/tests/unit/util/mappers/test_unit_mappers_engine.py index fcb61193374c..1f1295599ea6 100644 --- a/api/tests/unit/util/mappers/test_unit_mappers_engine.py +++ b/api/tests/unit/util/mappers/test_unit_mappers_engine.py @@ -224,7 +224,7 @@ def test_map_feature_state_to_engine__return_expected( # When result = engine.map_feature_state_to_engine( feature_state, - [], + mv_fs_values=[], ) # Then @@ -268,7 +268,7 @@ def test_map_feature_state_to_engine__feature_segment__return_expected( # When result = engine.map_feature_state_to_engine( segment_multivariate_feature_state, - [mv_fs_value], + mv_fs_values=[mv_fs_value], ) # Then diff --git a/api/util/mappers/__init__.py b/api/util/mappers/__init__.py index 5d3aac2543a1..86695e498087 100644 --- a/api/util/mappers/__init__.py +++ b/api/util/mappers/__init__.py @@ -1,16 +1,29 @@ from util.mappers.dynamodb import ( + map_engine_feature_state_to_identity_override, map_engine_identity_to_identity_document, map_environment_api_key_to_environment_api_key_document, map_environment_to_environment_document, + map_environment_to_environment_v2_document, + map_identity_changeset_to_identity_override_changeset, + map_identity_override_to_identity_override_document, map_identity_to_identity_document, ) -from util.mappers.engine import map_feature_to_engine, map_mv_option_to_engine +from util.mappers.engine import ( + map_feature_state_to_engine, + map_feature_to_engine, + map_mv_option_to_engine, +) __all__ = ( + "map_engine_feature_state_to_identity_override", "map_engine_identity_to_identity_document", "map_environment_api_key_to_environment_api_key_document", "map_environment_to_environment_document", + "map_environment_to_environment_v2_document", + "map_feature_state_to_engine", "map_feature_to_engine", + "map_identity_changeset_to_identity_override_changeset", + "map_identity_override_to_identity_override_document", "map_identity_to_identity_document", "map_mv_option_to_engine", ) diff --git a/api/util/mappers/dynamodb.py b/api/util/mappers/dynamodb.py index f2246a0a3aa5..2b39e0525e20 100644 --- a/api/util/mappers/dynamodb.py +++ b/api/util/mappers/dynamodb.py @@ -2,8 +2,17 @@ from decimal import Decimal from typing import TYPE_CHECKING, Any, Dict, List, TypeAlias, TypeVar, Union +from flag_engine.features.models import FeatureStateModel from pydantic import BaseModel +from edge_api.identities.types import IdentityChangeset +from environments.dynamodb.types import ( + IdentityOverridesV2Changeset, + IdentityOverrideV2, +) +from environments.dynamodb.utils import ( + get_environments_v2_identity_override_document_key, +) from util.mappers.engine import ( map_environment_api_key_to_engine, map_environment_to_engine, @@ -21,6 +30,7 @@ "map_engine_identity_to_identity_document", "map_environment_api_key_to_environment_api_key_document", "map_environment_to_environment_document", + "map_environment_to_environment_v2_document", "map_identity_to_identity_document", ) @@ -39,6 +49,16 @@ def map_environment_to_environment_document( } +def map_environment_to_environment_v2_document( + environment: "Environment", +) -> Document: + return { + **map_environment_to_environment_document(environment), + "document_key": "META", + "environment_id": str(environment.id), + } + + def map_environment_api_key_to_environment_api_key_document( environment_api_key: "EnvironmentAPIKey", ) -> Document: @@ -63,6 +83,69 @@ def map_identity_to_identity_document( return map_engine_identity_to_identity_document(map_identity_to_engine(identity)) +def map_engine_feature_state_to_identity_override( + *, + feature_state: "FeatureStateModel", + identity_uuid: str, + environment_api_key: str, + environment_id: int, +) -> list[IdentityOverrideV2]: + return IdentityOverrideV2( + document_key=get_environments_v2_identity_override_document_key( + feature_id=feature_state.feature.id, + identity_uuid=identity_uuid, + ), + environment_id=str(environment_id), + environment_api_key=environment_api_key, + feature_state=feature_state, + ) + + +def map_identity_changeset_to_identity_override_changeset( + *, + identity_changeset: "IdentityChangeset", + identity_uuid: str, + environment_api_key: str, + environment_id: int, +) -> "IdentityOverridesV2Changeset": + to_delete: list[IdentityOverrideV2] = [] + to_put: list[IdentityOverrideV2] = [] + + for _, change_details in identity_changeset["feature_overrides"].items(): + match change_details["change_type"]: + case "-": + feature_state = FeatureStateModel.parse_obj(change_details["old"]) + to_delete.append( + map_engine_feature_state_to_identity_override( + feature_state=feature_state, + identity_uuid=identity_uuid, + environment_api_key=environment_api_key, + environment_id=environment_id, + ) + ) + case _: + feature_state = FeatureStateModel.parse_obj(change_details["new"]) + to_put.append( + map_engine_feature_state_to_identity_override( + feature_state=feature_state, + identity_uuid=identity_uuid, + environment_api_key=environment_api_key, + environment_id=environment_id, + ) + ) + + return IdentityOverridesV2Changeset(to_delete=to_delete, to_put=to_put) + + +def map_identity_override_to_identity_override_document( + identity_override: IdentityOverrideV2, +) -> Document: + return { + field_name: _map_value_to_document_value(value) + for field_name, value in identity_override + } + + T = TypeVar("T") diff --git a/api/util/mappers/engine.py b/api/util/mappers/engine.py index 87b0beb81b49..d77a7efd1160 100644 --- a/api/util/mappers/engine.py +++ b/api/util/mappers/engine.py @@ -121,7 +121,8 @@ def map_webhook_config_to_engine( def map_feature_state_to_engine( feature_state: "FeatureState", - mv_fs_values: Iterable["MultivariateFeatureStateValue"], + *, + mv_fs_values: Optional[Iterable["MultivariateFeatureStateValue"]] = None, ) -> FeatureStateModel: feature = feature_state.feature feature_segment: Optional["FeatureSegment"] = feature_state.feature_segment @@ -141,7 +142,7 @@ def map_feature_state_to_engine( feature_segment=feature_segment_model, feature=map_feature_to_engine(feature), multivariate_feature_state_values=[ - map_mv_fs_value_to_engine(mv_fs_value) for mv_fs_value in mv_fs_values + map_mv_fs_value_to_engine(mv_fs_value) for mv_fs_value in mv_fs_values or [] ], ) @@ -252,7 +253,7 @@ def map_environment_to_engine( feature_states=[ map_feature_state_to_engine( feature_state, - multivariate_feature_state_values_by_feature_state_id.pop( + mv_fs_values=multivariate_feature_state_values_by_feature_state_id.pop( feature_state.pk, ), ) @@ -279,7 +280,9 @@ def map_environment_to_engine( feature_state_models = [ map_feature_state_to_engine( feature_state, - multivariate_feature_state_values_by_feature_state_id.pop(feature_state.pk), + mv_fs_values=multivariate_feature_state_values_by_feature_state_id.pop( + feature_state.pk, + ), ) for feature_state in environment_feature_states ] @@ -384,7 +387,9 @@ def map_identity_to_engine( identity_feature_state_models = [ map_feature_state_to_engine( feature_state, - multivariate_feature_state_values_by_feature_state_id.pop(feature_state.pk), + mv_fs_values=multivariate_feature_state_values_by_feature_state_id.pop( + feature_state.pk, + ), ) for feature_state in identity_feature_states ]