SK-2142 return None in case of empty error for Data APIs and include file path in deidentify file request (#188)

Author: raushan-skyflow

setup.py

@@ -7,7 +7,7 @@
 
 if sys.version_info < (3, 8):
     raise RuntimeError("skyflow requires Python 3.8+")
-current_version = '2.0.0b6'
+current_version = '2.0.0b7'
 
 setup(
     name='skyflow',

skyflow/utils/_skyflow_messages.py

@@ -169,6 +169,7 @@ class Error(Enum):
         INVALID_PLAIN_TEXT_ENTITIES_IN_REIDENTIFY= f"{error_prefix} Validation error. The plainTextEntities field must be an array of DetectEntities enums. Specify a valid plainTextEntities."
 
         INVALID_DEIDENTIFY_FILE_REQUEST= f"{error_prefix} Validation error. Invalid deidentify file request. Specify a valid deidentify file request."
+        INVALID_DEIDENTIFY_FILE_INPUT= f"{error_prefix} Validation error. Invalid deidentify file input. Please provide either a file or a file path."
         EMPTY_FILE_OBJECT= f"{error_prefix} Validation error. File object cannot be empty. Specify a valid file object."
         INVALID_FILE_FORMAT= f"{error_prefix} Validation error. Invalid file format. Specify a valid file format."
         MISSING_FILE_SOURCE= f"{error_prefix} Validation error. Provide exactly one of filePath, base64, or fileObject."
@@ -197,7 +198,7 @@ class Error(Enum):
         INVALID_FILE_OR_ENCODED_FILE= f"{error_prefix} . Error while decoding base64 and saving file"
         INVALID_FILE_TYPE = f"{error_prefix} Validation error. Invalid file type. Specify a valid file type."
         INVALID_FILE_NAME= f"{error_prefix} Validation error. Invalid file name. Specify a valid file name."
-        FILE_READ_ERROR= f"{error_prefix} Validation error. Unable to read file. Verify the file path."
+        INVALID_DEIDENTIFY_FILE_PATH= f"{error_prefix} Validation error. Invalid file path. Specify a valid file path."
         INVALID_BASE64_HEADER= f"{error_prefix} Validation error. Invalid base64 header. Specify a valid base64 header."
         INVALID_WAIT_TIME= f"{error_prefix} Validation error. Invalid wait time. Specify a valid wait time as number and should not be greater than 64 secs."
         INVALID_OUTPUT_DIRECTORY= f"{error_prefix} Validation error. Invalid output directory. Specify a valid output directory as string."

skyflow/utils/_utils.py

@@ -211,7 +211,6 @@ def get_metrics():
     }
     return details_dic
 
-
 def parse_insert_response(api_response, continue_on_error):
     # Retrieve the headers and data from the API response
     api_response_headers = api_response.headers
@@ -239,13 +238,13 @@ def parse_insert_response(api_response, continue_on_error):
                 error = {
                     'request_index': idx,
                     'request_id': request_id,
-                    'error': response['Body']['error']
+                    'error': response['Body']['error'],
+                    'http_code': response['Status'],
                 }
                 errors.append(error)
 
             insert_response.inserted_fields = inserted_fields
-            insert_response.errors = errors
-
+            insert_response.errors = errors if len(errors) > 0 else None
     else:
         for record in api_response_data.records:
             field_data = {
@@ -257,6 +256,7 @@ def parse_insert_response(api_response, continue_on_error):
 
             inserted_fields.append(field_data)
         insert_response.inserted_fields = inserted_fields
+        insert_response.errors = None
 
     return insert_response
 
@@ -275,21 +275,17 @@ def parse_delete_response(api_response: V1BulkDeleteRecordResponse):
     delete_response = DeleteResponse()
     deleted_ids = api_response.record_id_response
     delete_response.deleted_ids = deleted_ids
-    delete_response.errors = []
+    delete_response.errors = None
     return delete_response
 
-
 def parse_get_response(api_response: V1BulkGetRecordResponse):
     get_response = GetResponse()
     data = []
-    errors = []
     for record in api_response.records:
         field_data = {field: value for field, value in record.fields.items()}
         data.append(field_data)
 
     get_response.data = data
-    get_response.errors = errors
-
     return get_response
 
 def parse_detokenize_response(api_response: HttpResponse[V1DetokenizeResponse]):
@@ -320,7 +316,7 @@ def parse_detokenize_response(api_response: HttpResponse[V1DetokenizeResponse]):
     errors = errors
     detokenize_response = DetokenizeResponse()
     detokenize_response.detokenized_fields = detokenized_fields
-    detokenize_response.errors = errors
+    detokenize_response.errors = errors if len(errors) > 0 else None
 
     return detokenize_response
 
@@ -357,7 +353,7 @@ def parse_invoke_connection_response(api_response: requests.Response):
             if 'x-request-id' in api_response.headers:
                 metadata['request_id'] = api_response.headers['x-request-id']
 
-            return InvokeConnectionResponse(data=data, metadata=metadata)
+            return InvokeConnectionResponse(data=data, metadata=metadata, errors=None)
         except Exception as e:
             raise SkyflowError(SkyflowMessages.Error.RESPONSE_NOT_JSON.value.format(content), status_code)
     except HTTPError:

skyflow/utils/_version.py

@@ -1 +1 @@
-SDK_VERSION = '2.0.0b6'
+SDK_VERSION = '2.0.0b7'

skyflow/utils/validations/_validations.py

@@ -9,6 +9,7 @@
 from skyflow.utils.logger import log_info, log_error_log
 from skyflow.vault.detect import DeidentifyTextRequest, ReidentifyTextRequest, TokenFormat, Transformations, \
     GetDetectRunRequest, Bleep, DeidentifyFileRequest
+from skyflow.vault.detect._file_input import FileInput
 
 valid_vault_config_keys = ["vault_id", "cluster_id", "credentials", "env"]
 valid_connection_config_keys = ["connection_id", "connection_url", "credentials"]
@@ -257,9 +258,42 @@ def validate_update_connection_config(logger, config):
 
     return True
 
+def validate_file_from_request(file_input: FileInput):
+    if file_input is None:
+        raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_INPUT.value, invalid_input_error_code)
+    
+    has_file = hasattr(file_input, 'file') and file_input.file is not None
+    has_file_path = hasattr(file_input, 'file_path') and file_input.file_path is not None
+    
+    # Must provide exactly one of file or file_path
+    if (has_file and has_file_path) or (not has_file and not has_file_path):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_DEIDENTIFY_FILE_INPUT.value, invalid_input_error_code)
+    
+    if has_file:
+        file = file_input.file
+        # Validate file object has required attributes
+        if not hasattr(file, 'name') or not isinstance(file.name, str) or not file.name.strip():
+            raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_TYPE.value, invalid_input_error_code)
+        
+        # Validate file name
+        file_name = os.path.splitext(file.name)[0]
+        if not file_name or not file_name.strip():
+            raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_NAME.value, invalid_input_error_code)
+            
+    elif has_file_path:
+        file_path = file_input.file_path
+        if not isinstance(file_path, str) or not file_path.strip():
+            raise SkyflowError(SkyflowMessages.Error.INVALID_DEIDENTIFY_FILE_PATH.value, invalid_input_error_code)
+        
+        if not os.path.exists(file_path) or not os.path.isfile(file_path):
+            raise SkyflowError(SkyflowMessages.Error.INVALID_DEIDENTIFY_FILE_PATH.value, invalid_input_error_code)
+
 def validate_deidentify_file_request(logger, request: DeidentifyFileRequest):
     if not hasattr(request, 'file') or request.file is None:
         raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_INPUT.value, invalid_input_error_code)
+    
+    # Validate file input first
+    validate_file_from_request(request.file)
 
     # Optional: entities
     if hasattr(request, 'entities') and request.entities is not None:

skyflow/vault/connection/_invoke_connection_response.py

@@ -1,10 +1,11 @@
 class InvokeConnectionResponse:
-    def __init__(self, data=None, metadata=None):
+    def __init__(self, data=None, metadata=None, errors=None):
         self.data = data
         self.metadata = metadata if metadata else {}
+        self.errors = errors if errors else None
 
     def __repr__(self):
-        return f"ConnectionResponse('data'={self.data},'metadata'={self.metadata})"
+        return f"ConnectionResponse('data'={self.data},'metadata'={self.metadata}), 'errors'={self.errors})"
 
     def __str__(self):
         return self.__repr__()

skyflow/vault/controller/_detect.py

@@ -1,3 +1,4 @@
+import io
 import json
 import os
 from skyflow.error import SkyflowError
@@ -20,6 +21,7 @@
 from skyflow.vault.detect import DeidentifyTextRequest, DeidentifyTextResponse, ReidentifyTextRequest, \
     ReidentifyTextResponse, DeidentifyFileRequest, DeidentifyFileResponse, GetDetectRunRequest
 
+
 class Detect:
     def __init__(self, vault_client):
         self.__vault_client = vault_client
@@ -124,10 +126,22 @@ def output_to_dict_list(output):
         word_count = getattr(word_character_count, "word_count", None)
         char_count = getattr(word_character_count, "character_count", None)
 
+        base64_string = first_output.get("file", None)
+        extension = first_output.get("extension", None)
+
+        file_obj = None
+        if base64_string is not None:
+                file_bytes = base64.b64decode(base64_string)
+                file_obj = io.BytesIO(file_bytes)
+                file_obj.name = f"deidentified.{extension}" if extension else "processed_file"
+        else:
+            file_obj = None
+    
         return DeidentifyFileResponse(
-            file=first_output.get("file", None),
+            file_base64=base64_string,
+            file=file_obj,  # File class will be instantiated in DeidentifyFileResponse
             type=first_output.get("type", None),
-            extension=first_output.get("extension", None),
+            extension=extension,
             word_count=word_count,
             char_count=char_count,
             size_in_kb=size,
@@ -137,7 +151,7 @@ def output_to_dict_list(output):
             entities=entities,
             run_id=run_id_val,
             status=status_val,
-            errors=[]
+            errors=None
         )
 
     def __get_token_format(self, request):
@@ -216,16 +230,26 @@ def reidentify_text(self, request: ReidentifyTextRequest) -> ReidentifyTextRespo
             log_error_log(SkyflowMessages.ErrorLogs.REIDENTIFY_TEXT_REQUEST_REJECTED.value, self.__vault_client.get_logger())
             handle_exception(e, self.__vault_client.get_logger())
 
+    def __get_file_from_request(self, request: DeidentifyFileRequest):
+        file_input = request.file
+        
+        # Check for file
+        if hasattr(file_input, 'file') and file_input.file is not None:
+            return file_input.file
+            
+        # Check for file_path if file is not provided
+        if hasattr(file_input, 'file_path') and file_input.file_path is not None:
+                return open(file_input.file_path, 'rb')
+
     def deidentify_file(self, request: DeidentifyFileRequest):
         log_info(SkyflowMessages.Info.DETECT_FILE_TRIGGERED.value, self.__vault_client.get_logger())
         validate_deidentify_file_request(self.__vault_client.get_logger(), request)
         self.__initialize()
         files_api = self.__vault_client.get_detect_file_api().with_raw_response
-        file_obj = request.file
+        file_obj = self.__get_file_from_request(request)
         file_name = getattr(file_obj, 'name', None)
         file_extension = self._get_file_extension(file_name) if file_name else None
         file_content = file_obj.read()
-
         base64_string = base64.b64encode(file_content).decode('utf-8')
 
         try:
@@ -375,7 +399,7 @@ def deidentify_file(self, request: DeidentifyFileRequest):
                 file_name_only = 'processed-'+os.path.basename(file_name)
                 output_file_path = f"{request.output_directory}/{file_name_only}"
                 with open(output_file_path, 'wb') as output_file:
-                    output_file.write(base64.b64decode(parsed_response.file))
+                    output_file.write(base64.b64decode(parsed_response.file_base64))
             log_info(SkyflowMessages.Info.DETECT_FILE_SUCCESS.value, self.__vault_client.get_logger())
             return parsed_response
 

skyflow/vault/data/_insert_response.py

@@ -1,7 +1,5 @@
 class InsertResponse:
     def __init__(self, inserted_fields = None, errors=None):
-        if errors is None:
-            errors = list()
         self.inserted_fields = inserted_fields
         self.errors = errors
 

skyflow/vault/data/_query_response.py

@@ -1,7 +1,7 @@
 class QueryResponse:
     def __init__(self):
         self.fields = []
-        self.errors = []
+        self.errors = None
 
     def __repr__(self):
         return f"QueryResponse(fields={self.fields}, errors={self.errors})"

skyflow/vault/data/_update_response.py

@@ -1,7 +1,7 @@
 class UpdateResponse:
     def __init__(self, updated_field = None, errors=None):
         self.updated_field = updated_field
-        self.errors = errors if errors is not None else []
+        self.errors = errors
 
     def __repr__(self):
         return f"UpdateResponse(updated_field={self.updated_field}, errors={self.errors})"