SK-1874: Support for the combination of tokens and redaction type in the Detokenize API. (#156)

* SK-1874: Detokenize API support

Author: saileshwar-skyflow

samples/vault_api/detokenize_records.py

@@ -52,13 +52,20 @@ def perform_detokenization():
         )
 
         # Step 4: Prepare Detokenization Data
-        detokenize_data = ['token1', 'token2', 'token3'] # Tokens to be detokenized
-        redaction_type = RedactionType.REDACTED
+        detokenize_data = [
+            {
+                'token': '<TOKEN1>',                # Token to be detokenized
+                'redaction': RedactionType.REDACTED
+            },
+            {
+                'token': '<TOKEN2>',                # Token to be detokenized
+                'redaction': RedactionType.MASKED
+            }
+        ]
 
         # Create Detokenize Request
         detokenize_request = DetokenizeRequest(
-            tokens=detokenize_data,
-            redaction_type=redaction_type,
+            data=detokenize_data,
             continue_on_error=True # Continue processing on errors
         )
 

skyflow/utils/_skyflow_messages.py

@@ -101,7 +101,7 @@ class Error(Enum):
         INVOKE_CONNECTION_FAILED = f"{error_prefix} Invoke Connection operation failed."
 
         INVALID_IDS_TYPE = f"{error_prefix} Validation error. 'ids' has a value of type {{}}. Specify 'ids' as list."
-        INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction' has a value of type {{}}. Specify 'redaction' as type Skyflow.Redaction."
+        INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction' has a value of type {{}}. Specify 'redaction' as type Skyflow.RedactionType."
         INVALID_COLUMN_NAME = f"{error_prefix} Validation error. 'column' has a value of type {{}}. Specify 'column' as a string."
         INVALID_COLUMN_VALUE = f"{error_prefix} Validation error. columnValues key has a value of type {{}}. Specify columnValues key as list."
         INVALID_FIELDS_VALUE = f"{error_prefix} Validation error. fields key has a value of type{{}}. Specify fields key as list."
@@ -117,8 +117,10 @@ class Error(Enum):
         UPDATE_FIELD_KEY_ERROR = f"{error_prefix} Validation error. Fields are empty in an update payload. Specify at least one field."
         INVALID_FIELDS_TYPE = f"{error_prefix} Validation error. The 'data' key has a value of type {{}}. Specify 'data' as a dictionary."
         IDS_KEY_ERROR = f"{error_prefix} Validation error. 'ids' key is missing from the payload. Specify an 'ids' key."
-        INVALID_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. The 'tokens' key has a value of type {{}}. Specify 'tokens' as a list."
+        INVALID_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. The 'data' field is invalid. Specify 'data' as a list of dictionaries containing 'token' and 'redaction'."
+        INVALID_DATA_FOR_DETOKENIZE = f"{error_prefix}"
         EMPTY_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. Tokens are empty in detokenize payload. Specify at lease one token"
+        INVALID_TOKEN_TYPE = f"{ERROR}: [{error_prefix}] Invalid {{}} request. Tokens should be of type string."
 
         INVALID_TOKENIZE_PARAMETERS = f"{error_prefix} Validation error. The 'values' key has a value of type {{}}. Specify 'tokenize_parameters' as a list."
         EMPTY_TOKENIZE_PARAMETERS = f"{error_prefix} Validation error. Tokenize values are empty in tokenize payload. Specify at least one parameter."

skyflow/utils/validations/_validations.py

@@ -502,19 +502,28 @@ def validate_update_request(logger, request):
                 invalid_input_error_code)
 
 def validate_detokenize_request(logger, request):
-    if not isinstance(request.redaction_type, RedactionType):
-        raise SkyflowError(SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(type(request.redaction_type)), invalid_input_error_code)
-
     if not isinstance(request.continue_on_error, bool):
         raise SkyflowError(SkyflowMessages.Error.INVALID_CONTINUE_ON_ERROR_TYPE.value, invalid_input_error_code)
 
-    if not len(request.tokens):
+    if not isinstance(request.data, list):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_LIST_VALUE.value(type(request.data)), invalid_input_error_code)
+
+    if not len(request.data):
         log_error_log(SkyflowMessages.ErrorLogs.TOKENS_REQUIRED.value.format("DETOKENIZE"), logger = logger)
         log_error_log(SkyflowMessages.ErrorLogs.EMPTY_TOKENS.value.format("DETOKENIZE"), logger = logger)
         raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_LIST_VALUE.value, invalid_input_error_code)
 
-    if not isinstance(request.tokens, list):
-        raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_LIST_VALUE.value(type(request.tokens)), invalid_input_error_code)
+    for item in request.data:
+        if 'token' not in item or 'redaction' not in item:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_LIST_VALUE.value(type(request.data)), invalid_input_error_code)
+        token = item.get('token')
+        redaction = item.get('redaction')
+
+        if not isinstance(token, str) or not token:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_TYPE.value.format("DETOKENIZE"), invalid_input_error_code)
+
+        if not isinstance(redaction, RedactionType) or not redaction:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(type(redaction)), invalid_input_error_code)
 
 def validate_tokenize_request(logger, request):
     parameters = request.values

skyflow/vault/controller/_vault.py

@@ -230,8 +230,8 @@ def detokenize(self, request: DetokenizeRequest):
         log_info(SkyflowMessages.Info.DETOKENIZE_REQUEST_RESOLVED.value, self.__vault_client.get_logger())
         self.__initialize()
         tokens_list = [
-            V1DetokenizeRecordRequest(token=token, redaction=request.redaction_type.value)
-            for token in request.tokens
+            V1DetokenizeRecordRequest(token=item.get('token'), redaction=item.get('redaction').value)
+            for item in request.data
         ]
         payload = V1DetokenizePayload(detokenization_parameters=tokens_list, continue_on_error=request.continue_on_error)
         tokens_api = self.__vault_client.get_tokens_api()

skyflow/vault/tokens/_detokenize_request.py

@@ -1,7 +1,6 @@
 from skyflow.utils.enums.redaction_type import RedactionType
 
 class DetokenizeRequest:
-    def __init__(self, tokens, redaction_type = RedactionType.PLAIN_TEXT, continue_on_error = False):
-        self.tokens = tokens
-        self.redaction_type = redaction_type
+    def __init__(self, data, continue_on_error = False):
+        self.data = data
         self.continue_on_error = continue_on_error

tests/vault/controller/test__vault.py

@@ -455,8 +455,16 @@ def test_query_successful(self, mock_parse_response, mock_validate):
     @patch("skyflow.vault.controller._vault.parse_detokenize_response")
     def test_detokenize_successful(self, mock_parse_response, mock_validate):
         request = DetokenizeRequest(
-            tokens=["token1", "token2"],
-            redaction_type=RedactionType.PLAIN_TEXT,
+            data=[
+                {
+                    'token': 'token1',
+                    'redaction': RedactionType.PLAIN_TEXT
+                },
+                {
+                    'token': 'token2',
+                    'redaction': RedactionType.PLAIN_TEXT
+                }
+            ],
             continue_on_error=False
         )