SK-1758 Add detect support in Python SDK v2

- SK-2042 Add public interfaces for DeIdentify and ReIdentify text
- SK-1758 Add detect controller

Author: skyflow-vivek

skyflow/client/skyflow.py

@@ -8,6 +8,7 @@
 from skyflow.vault.client.client import VaultClient
 from skyflow.vault.controller import Vault
 from skyflow.vault.controller import Connection
+from skyflow.vault.controller import Detect
 
 class Skyflow:
     def __init__(self, builder):
@@ -65,11 +66,15 @@ def update_log_level(self, log_level):
 
     def vault(self, vault_id = None) -> Vault:
         vault_config = self.__builder.get_vault_config(vault_id)
-        return vault_config.get("controller")
+        return vault_config.get("vault_controller")
 
     def connection(self, connection_id = None) -> Connection:
         connection_config = self.__builder.get_connection_config(connection_id)
         return connection_config.get("controller")
+    
+    def detect(self, vault_id = None) -> Detect:
+        vault_config = self.__builder.get_vault_config(vault_id)
+        return vault_config.get("detect_controller")
 
     class Builder:
         def __init__(self):
@@ -182,7 +187,8 @@ def __add_vault_config(self, config):
             vault_client = VaultClient(config)
             self.__vault_configs[vault_id] = {
                 "vault_client": vault_client,
-                "controller": Vault(vault_client)
+                "vault_controller": Vault(vault_client),
+                "detect_controller": Detect(vault_client)
             }
             log_info(SkyflowMessages.Info.VAULT_CONTROLLER_INITIALIZED.value.format(config.get("vault_id")), self.__logger)
 

skyflow/utils/_skyflow_messages.py

@@ -147,13 +147,59 @@ class Error(Enum):
         FILE_INVALID_JSON = f"{error_prefix} Initialization failed. File at {{}} is not in valid JSON format. Verify the file contents."
         INVALID_JSON_FORMAT_IN_CREDENTIALS_ENV = f"{error_prefix} Validation error. Invalid JSON format in SKYFLOW_CREDENTIALS environment variable."
 
+        INVALID_TEXT_IN_DEIDENTIFY: f"{error_prefix} Validation error. The text field is required and must be a non-empty string. Specify a valid text."
+        INVALID_ENTITIES_IN_DEIDENTIFY: f"{error_prefix} Validation error. The entities field must be an array of DetectEntities enums. Specify a valid entities."
+        INVALID_ALLOW_REGEX_LIST: f"{error_prefix} Validation error. The allowRegexList field must be an array of strings. Specify a valid allowRegexList."
+        INVALID_RESTRICT_REGEX_LIST: f"{error_prefix} Validation error. The restrictRegexList field must be an array of strings. Specify a valid restrictRegexList."
+        INVALID_TOKEN_FORMAT: f"{error_prefix} Validation error. The tokenFormat key must be an instance of TokenFormat. Specify a valid token format."
+        INVALID_TRANSFORMATIONS: f"{error_prefix} Validation error. The transformations key must be an instance of Transformations. Specify a valid transformations."
+
+        INVALID_TEXT_IN_REIDENTIFY: f"{error_prefix} Validation error. The text field is required and must be a non-empty string. Specify a valid text."
+        INVALID_REDACTED_ENTITIES_IN_REIDENTIFY: f"{error_prefix} Validation error. The redactedEntities field must be an array of DetectEntities enums. Specify a valid redactedEntities."
+        INVALID_MASKED_ENTITIES_IN_REIDENTIFY: f"{error_prefix} Validation error. The maskedEntities field must be an array of DetectEntities enums. Specify a valid maskedEntities."
+        INVALID_PLAIN_TEXT_ENTITIES_IN_REIDENTIFY: f"{error_prefix} Validation error. The plainTextEntities field must be an array of DetectEntities enums. Specify a valid plainTextEntities."
+
+        INVALID_DEIDENTIFY_FILE_REQUEST: f"{error_prefix} Validation error. Invalid deidentify file request. Specify a valid deidentify file request."
+        EMPTY_FILE_OBJECT: f"{error_prefix} Validation error. File object cannot be empty. Specify a valid file object."
+        INVALID_FILE_FORMAT: f"{error_prefix} Validation error. Invalid file format. Specify a valid file format."
+        MISSING_FILE_SOURCE: f"{error_prefix} Validation error. Provide exactly one of filePath, base64, or fileObject."
+        INVALID_FILE_OBJECT: f"{error_prefix} Validation error. Invalid file object. Specify a valid file object."
+        INVALID_BASE64_STRING: f"{error_prefix} Validation error. Invalid base64 string. Specify a valid base64 string."
+        INVALID_DEIDENTIFY_FILE_OPTIONS: f"{error_prefix} Validation error. Invalid deidentify file options. Specify a valid deidentify file options."
+        INVALID_ENTITIES: f"{error_prefix} Validation error. Invalid entities. Specify valid entities as string array."
+        EMPTY_ENTITIES: f"{error_prefix} Validation error. Entities cannot be empty. Specify valid entities."
+        EMPTY_ALLOW_REGEX_LIST: f"{error_prefix} Validation error. Allow regex list cannot be empty. Specify valid allow regex list."
+        INVALID_ALLOW_REGEX: f"{error_prefix} Validation error. Invalid allow regex. Specify valid allow regex at index {{}}."
+        EMPTY_RESTRICT_REGEX_LIST: f"{error_prefix} Validation error. Restrict regex list cannot be empty. Specify valid restrict regex list."
+        INVALID_RESTRICT_REGEX: f"{error_prefix} Validation error. Invalid restrict regex. Specify valid restrict regex at index {{}}."
+        INVALID_OUTPUT_PROCESSED_IMAGE: f"{error_prefix} Validation error. Invalid output processed image. Specify valid output processed image as string."
+        INVALID_OUTPUT_OCR_TEXT: f"{error_prefix} Validation error. Invalid output ocr text. Specify valid output ocr text as string."
+        INVALID_MASKING_METHOD: f"{error_prefix} Validation error. Invalid masking method. Specify valid masking method as string."
+        INVALID_PIXEL_DENSITY: f"{error_prefix} Validation error. Invalid pixel density. Specify valid pixel density as string."
+        INVALID_OUTPUT_TRANSCRIPTION: f"{error_prefix} Validation error. Invalid output transcription. Specify valid output transcription as string."
+        INVALID_OUTPUT_PROCESSED_AUDIO: f"{error_prefix} Validation error. Invalid output processed audio. Specify valid output processed audio as string."
+        INVALID_MAX_RESOLUTION: f"{error_prefix} Validation error. Invalid max resolution. Specify valid max resolution as string."
+        INVALID_BLEEP: f"{error_prefix} Validation error. Invalid bleep. Specify valid bleep as object."
+        INVALID_FILE_OR_ENCODED_FILE: f"{error_prefix} . Error while decoding base64 and saving file"
+        INVALID_FILE_TYPE : f"{error_prefix} Validation error. Invalid file type. Specify a valid file type."
+        INVALID_FILE_NAME: f"{error_prefix} Validation error. Invalid file name. Specify a valid file name."
+        FILE_READ_ERROR: f"{error_prefix} Validation error. Unable to read file. Verify the file path." 
+        INVALID_BASE64_HEADER: f"{error_prefix} Validation error. Invalid base64 header. Specify a valid base64 header."
+        INVALID_WAIT_TIME: f"{error_prefix} Validation error. Invalid wait time. Specify a valid wait time as number and should not be greater than 20 secs."
+        INVALID_OUTPUT_DIRECTORY: f"{error_prefix} Validation error. Invalid output directory. Specify a valid output directory as string."
+        INVALID_OUTPUT_DIRECTORY_PATH: f"{error_prefix} Validation error. Invalid output directory path. Specify a valid output directory path as string."
+        EMPTY_RUN_ID: f"{error_prefix} Validation error. Run id cannot be empty. Specify a valid run id."
+        INVALID_RUN_ID: f"{error_prefix} Validation error. Invalid run id. Specify a valid run id as string."
+        INTERNAL_SERVER_ERROR: f"{error_prefix}. Internal server error. {{}}."
+
     class Info(Enum):
         CLIENT_INITIALIZED = f"{INFO}: [{error_prefix}] Initialized skyflow client."
         VALIDATING_VAULT_CONFIG = f"{INFO}: [{error_prefix}] Validating vault config."
         VALIDATING_CONNECTION_CONFIG = f"{INFO}: [{error_prefix}] Validating connection config."
         UNABLE_TO_GENERATE_SDK_METRIC = f"{INFO}: [{error_prefix}] Unable to generate {{}} metric."
         VAULT_CONTROLLER_INITIALIZED = f"{INFO}: [{error_prefix}] Initialized vault controller with vault ID {{}}."
         CONNECTION_CONTROLLER_INITIALIZED = f"{INFO}: [{error_prefix}] Initialized connection controller with connection ID {{}}."
+        DETECT_CONTROLLER_INITIALIZED = f"{INFO}: [{error_prefix}] Initialized detect controller with vault ID {{}}."
         VAULT_CONFIG_EXISTS = f"{INFO}: [{error_prefix}] Vault config with vault ID {{}} already exists."
         VAULT_CONFIG_DOES_NOT_EXIST = f"{INFO}: [{error_prefix}] Vault config with vault ID {{}} doesn't exist."
         CONNECTION_CONFIG_EXISTS = f"{INFO}: [{error_prefix}] Connection config with connection ID {{}} already exists."
@@ -166,10 +212,9 @@ class Info(Enum):
         GET_BEARER_TOKEN_SUCCESS = f"{INFO}: [{error_prefix}] Bearer token generated."
         GET_SIGNED_DATA_TOKENS_TRIGGERED = f"{INFO}: [{error_prefix}] generate_signed_data_tokens method triggered."
         GET_SIGNED_DATA_TOKEN_SUCCESS = f"{INFO}: [{error_prefix}] Signed data tokens generated."
-        GENERATE_BEARER_TOKEN_FROM_CREDENTIALS_STRING_TRIGGERED = f"{INFO}: [{error_prefix}] generate bearer_token_from_credential_string method triggered ."
+        GENERATE_BEARER_TOKEN_FROM_CREDENTIALS_STRING_TRIGGERED = f"{INFO}: [{error_prefix}] generate bearer_token_from_credential_string method triggered."
         REUSE_BEARER_TOKEN = f"{INFO}: [{error_prefix}] Reusing bearer token."
 
-
         VALIDATE_INSERT_REQUEST = f"{INFO}: [{error_prefix}] Validating insert request."
         INSERT_TRIGGERED = f"{INFO}: [{error_prefix}] Insert method triggered."
         INSERT_SUCCESS = f"{INFO}: [{error_prefix}] Data inserted."
@@ -210,6 +255,28 @@ class Info(Enum):
         INVOKE_CONNECTION_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Invoke connection request resolved."
         INVOKE_CONNECTION_SUCCESS = f"{INFO}: [{error_prefix}] Invoke Connection Success."
 
+        DEIDENTIFY_TEXT_TRIGGERED = f"{INFO}: [{error_prefix}] Deidentify text method triggered."
+        VALIDATING_DEIDENTIFY_TEXT_INPUT = f"{INFO}: [{error_prefix}] Validating deidentify text input."
+        DEIDENTIFY_TEXT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Deidentify text request is resolved."
+        DEIDENTIFY_TEXT_SUCCESS = f"{INFO}: [{error_prefix}] data deidentified." 
+        
+        REIDENTIFY_TEXT_TRIGGERED = f"{INFO}: [{error_prefix}] Reidentify text method triggered."
+        VALIDATING_REIDENTIFY_TEXT_INPUT = f"{INFO}: [{error_prefix}] Validating reidentify text input."
+        REIDENTIFY_TEXT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Reidentify text request is resolved."
+        REIDENTIFY_TEXT_SUCCESS = f"{INFO}: [{error_prefix}] data reidentified." 
+        
+        DEIDENTIFY_FILE_TRIGGERED = f"{INFO}: [{error_prefix}] Deidentify file triggered."
+        VALIDATING_DETECT_FILE_INPUT = f"{INFO}: [{error_prefix}] Validating deidentify file input."
+        DEIDENTIFY_FILE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Deidentify file request is resolved."
+        DEIDENTIFY_FILE_SUCCESS = f"{INFO}: [{error_prefix}] file deidentified." 
+        
+        GET_DETECT_RUN_TRIGGERED = f"{INFO}: [{error_prefix}] Get detect run triggered."
+        VALIDATING_GET_DETECT_RUN_INPUT = f"{INFO}: [{error_prefix}] Validating get detect run input."
+        GET_DETECT_RUN_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Get detect run request is resolved."
+        GET_DETECT_RUN_SUCCESS = f"{INFO}: [{error_prefix}] get detect run success." # will see for a better message
+        
+        DETECT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Detect request is resolved." # Whats this for?
+
     class ErrorLogs(Enum):
         VAULTID_IS_REQUIRED = f"{ERROR}: [{error_prefix}] Invalid vault config. Vault ID is required."
         EMPTY_VAULTID = f"{ERROR}: [{error_prefix}] Invalid vault config. Vault ID can not be empty."
@@ -280,6 +347,13 @@ class ErrorLogs(Enum):
         QUERY_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Query request resulted in failure."
         GET_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Get request resulted in failure."
         INVOKE_CONNECTION_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Invoke connection request resulted in failure."
+        
+        EMPTY_RUN_ID = f"{ERROR}: [{error_prefix}] Validation error. Run id cannot be empty. Specify a valid run id."
+        INVALID_RUN_ID = f"{ERROR}: [{error_prefix}] Validation error. Invalid run id. Specify a valid run id as string."
+        DEIDENTIFY_FILE_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Deidentify file resulted in failure."
+        DETECT_RUN_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Detect get run resulted in failure."
+        DEIDENTIFY_TEXT_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Deidentify text resulted in failure."
+        REIDENTIFY_TEXT_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Reidentify text resulted in failure."
 
     class Interface(Enum):
         INSERT = "INSERT"

skyflow/utils/_utils.py

@@ -13,9 +13,11 @@
 from urllib.parse import quote
 from skyflow.error import SkyflowError
 from skyflow.generated.rest import V1UpdateRecordResponse, V1BulkDeleteRecordResponse, \
-    V1DetokenizeResponse, V1TokenizeResponse, V1GetQueryResponse, V1BulkGetRecordResponse
+    V1DetokenizeResponse, V1TokenizeResponse, V1GetQueryResponse, V1BulkGetRecordResponse, \
+    DeidentifyStringResponse, ReidentifyStringResponse
 from skyflow.generated.rest.core.http_response import HttpResponse
 from skyflow.utils.logger import log_error_log
+from skyflow.vault.detect import DeidentifyTextResponse, ReidentifyTextResponse
 from . import SkyflowMessages, SDK_VERSION
 from .constants import PROTOCOL
 from .enums import Env, ContentType, EnvUrls
@@ -364,6 +366,16 @@ def parse_invoke_connection_response(api_response: requests.Response):
             message = SkyflowMessages.Error.RESPONSE_NOT_JSON.value.format(content)
         raise SkyflowError(message, status_code)
 
+def parse_deidentify_text_response(api_response: DeidentifyStringResponse):
+    processed_text = api_response.processed_text
+    entities = api_response.entities
+    word_count = api_response.word_count
+    character_count = api_response.character_count
+    return DeidentifyTextResponse(processed_text, entities, word_count, character_count)
+
+def parse_reidentify_text_response(api_response: ReidentifyStringResponse):
+    return ReidentifyTextResponse(api_response.processed_text)
+
 def log_and_reject_error(description, status_code, request_id, http_status=None, grpc_code=None, details=None, logger = None):
     raise SkyflowError(description, status_code, request_id, grpc_code, http_status, details)
 

skyflow/utils/enums/__init__.py

@@ -1,6 +1,8 @@
 from .env import Env, EnvUrls
 from .log_level import LogLevel
 from .content_types import ContentType
+from .detect_entities import DetectEntities
 from .token_mode import TokenMode
+from .token_type import TokenType
 from .request_method import RequestMethod
 from .redaction_type import RedactionType

skyflow/utils/enums/detect_entities.py

@@ -0,0 +1,66 @@
+from enum import Enum
+
+class DetectEntities(Enum):
+    ACCOUNT_NUMBER = "account_number"
+    AGE = "age"
+    ALL = "all"
+    BANK_ACCOUNT = "bank_account"
+    BLOOD_TYPE = "blood_type"
+    CONDITION = "condition"
+    CORPORATE_ACTION = "corporate_action"
+    CREDIT_CARD = "credit_card"
+    CREDIT_CARD_EXPIRATION = "credit_card_expiration"
+    CVV = "cvv"
+    DATE = "date"
+    DATE_INTERVAL = "date_interval"
+    DOB = "dob"
+    DOSE = "dose"
+    DRIVER_LICENSE = "driver_license"
+    DRUG = "drug"
+    DURATION = "duration"
+    EMAIL_ADDRESS = "email_address"
+    EVENT = "event"
+    FILENAME = "filename"
+    FINANCIAL_METRIC = "financial_metric"
+    GENDER_SEXUALITY = "gender_sexuality"
+    HEALTHCARE_NUMBER = "healthcare_number"
+    INJURY = "injury"
+    IP_ADDRESS = "ip_address"
+    LANGUAGE = "language"
+    LOCATION = "location"
+    LOCATION_ADDRESS = "location_address"
+    LOCATION_ADDRESS_STREET = "location_address_street"
+    LOCATION_CITY = "location_city"
+    LOCATION_COORDINATE = "location_coordinate"
+    LOCATION_COUNTRY = "location_country"
+    LOCATION_STATE = "location_state"
+    LOCATION_ZIP = "location_zip"
+    MARITAL_STATUS = "marital_status"
+    MEDICAL_CODE = "medical_code"
+    MEDICAL_PROCESS = "medical_process"
+    MONEY = "money"
+    NAME = "name"
+    NAME_FAMILY = "name_family"
+    NAME_GIVEN = "name_given"
+    NAME_MEDICAL_PROFESSIONAL = "name_medical_professional"
+    NUMERICAL_PII = "numerical_pii"
+    OCCUPATION = "occupation"
+    ORGANIZATION = "organization"
+    ORGANIZATION_MEDICAL_FACILITY = "organization_medical_facility"
+    ORIGIN = "origin"
+    PASSPORT_NUMBER = "passport_number"
+    PASSWORD = "password"
+    PHONE_NUMBER = "phone_number"
+    PHYSICAL_ATTRIBUTE = "physical_attribute"
+    POLITICAL_AFFILIATION = "political_affiliation"
+    PRODUCT = "product"
+    RELIGION = "religion"
+    ROUTING_NUMBER = "routing_number"
+    SSN = "ssn"
+    STATISTICS = "statistics"
+    TIME = "time"
+    TREND = "trend"
+    URL = "url"
+    USERNAME = "username"
+    VEHICLE_ID = "vehicle_id"
+    ZODIAC_SIGN = "zodiac_sign"

skyflow/utils/enums/token_type.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+class TokenType(Enum):
+    VAULT_TOKEN = "vault_token"
+    ENTITY_UNIQUE_COUNTER = "entity_unq_counter"
+    ENTITY_ONLY = "entity_only"

skyflow/utils/validations/__init__.py

@@ -13,4 +13,6 @@
     validate_detokenize_request,
     validate_tokenize_request,
     validate_invoke_connection_params,
+    validate_deidentify_text_request,
+    validate_reidentify_text_request
 )