Merge branch 'himanshu-2496' into aadarsh-st/SK-2521-Fix-python-issue

Author: aadarsh-st

ruff.toml

@@ -8,12 +8,13 @@ exclude = [
     "venv",
     "build",
     "dist",
-    "tests"
+    "tests",
+    "samples"
 ]
 
 line-length = 120
 
 [lint]
-select = ["N"]
+select = ["N", "PLR2004"]
 
 [lint.pep8-naming]

skyflow/utils/_skyflow_messages.py

@@ -389,6 +389,7 @@ class ErrorLogs(Enum):
         SAVING_DEIDENTIFY_FILE_FAILED = f"{ERROR}: [{error_prefix}] Error while saving deidentified file to output directory."
         REIDENTIFY_TEXT_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Reidentify text resulted in failure."
         DETECT_FILE_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] Deidentify file resulted in failure."
+        EMPTY_FILE_COLUMN_NAME = f"{ERROR}: [{error_prefix}] Empty column name in FILE_UPLOAD"
 
     class Interface(Enum):
         INSERT = "INSERT"

skyflow/utils/_utils.py

@@ -21,7 +21,8 @@
 from skyflow.vault.detect import EntityInfo, TextIndex
 from . import SkyflowMessages, SDK_VERSION
 from .constants import (PROTOCOL, HttpHeader, ApiKey, ContentType as ContentTypeConstants, 
-                        EncodingType, BooleanString, ResponseField, CredentialField)
+                        EncodingType, BooleanString, ResponseField, CredentialField, SdkPrefix, 
+                        SdkMetricsKey, ErrorDefaults, HttpStatusCode)
 from .enums import Env, ContentType, EnvUrls
 from skyflow.vault.data import InsertResponse, UpdateResponse, DeleteResponse, QueryResponse, GetResponse
 from .validations import validate_invoke_connection_params
@@ -129,7 +130,7 @@ def construct_invoke_connection_request(request, connection_url, logger) -> Prep
 
     validate_invoke_connection_params(logger, request.query_params, request.path_params)
 
-    if not hasattr(request.method, 'value'):
+    if not hasattr(request.method, ResponseField.VALUE):
         raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_METHOD.value, invalid_input_error_code)
 
     try:
@@ -187,7 +188,7 @@ def get_data_from_content_type(data, content_type):
 
 
 def get_metrics():
-    sdk_name_version = "skyflow-python@" + SDK_VERSION
+    sdk_name_version = SdkPrefix.SKYFLOW_PYTHON + SDK_VERSION
 
     try:
         sdk_client_device_model = platform.node()
@@ -205,10 +206,10 @@ def get_metrics():
         sdk_runtime_details = ""
 
     details_dic = {
-        'sdk_name_version': sdk_name_version,
-        'sdk_client_device_model': sdk_client_device_model,
-        'sdk_client_os_details': sdk_client_os_details,
-        'sdk_runtime_details': "Python " + sdk_runtime_details,
+        SdkMetricsKey.SDK_NAME_VERSION: sdk_name_version,
+        SdkMetricsKey.SDK_CLIENT_DEVICE_MODEL: sdk_client_device_model,
+        SdkMetricsKey.SDK_CLIENT_OS_DETAILS: sdk_client_os_details,
+        SdkMetricsKey.SDK_RUNTIME_DETAILS: SdkPrefix.PYTHON_RUNTIME + sdk_runtime_details,
     }
     return details_dic
 
@@ -223,7 +224,7 @@ def parse_insert_response(api_response, continue_on_error):
     insert_response = InsertResponse()
     if continue_on_error:
         for idx, response in enumerate(api_response_data.responses):
-            if response[ResponseField.STATUS] == 200:
+            if response[ResponseField.STATUS] == HttpStatusCode.OK:
                 body = response[ResponseField.BODY]
                 if ResponseField.RECORDS in body:
                     for record in body[ResponseField.RECORDS]:
@@ -235,7 +236,7 @@ def parse_insert_response(api_response, continue_on_error):
                         if ResponseField.TOKENS in record:
                             inserted_field.update(record[ResponseField.TOKENS])
                         inserted_fields.append(inserted_field)
-            elif response[ResponseField.STATUS] == 400:
+            elif response[ResponseField.STATUS] == HttpStatusCode.BAD_REQUEST:
                 error = {
                     ResponseField.REQUEST_INDEX: idx,
                     ResponseField.REQUEST_ID: request_id,
@@ -352,7 +353,7 @@ def parse_invoke_connection_response(api_response: requests.Response):
             data = json.loads(content)
             metadata = {}
             if HttpHeader.X_REQUEST_ID in api_response.headers:
-                metadata['request_id'] = api_response.headers[HttpHeader.X_REQUEST_ID]
+                metadata[ResponseField.REQUEST_ID] = api_response.headers[HttpHeader.X_REQUEST_ID]
 
             return InvokeConnectionResponse(data=data, metadata=metadata, errors=None)
         except Exception as e:
@@ -400,7 +401,7 @@ def handle_exception(error, logger):
     if (isinstance(error, httpx.ConnectError)):
         handle_generic_error(error, None, SkyflowMessages.ErrorCodes.INVALID_INPUT.value, logger)
 
-    request_id = error.headers.get(HttpHeader.X_REQUEST_ID, 'unknown-request-id')
+    request_id = error.headers.get(HttpHeader.X_REQUEST_ID, ErrorDefaults.UNKNOWN_REQUEST_ID)
     content_type = error.headers.get(HttpHeader.CONTENT_TYPE_LOWERCASE)
     data = error.body
 

skyflow/utils/constants.py

@@ -13,11 +13,13 @@ class HttpHeader:
     X_REQUEST_ID = 'x-request-id'
     ERROR_FROM_CLIENT = 'error-from-client'
     AUTHORIZATION = 'Authorization'
+    X_SKYFLOW_AUTHORIZATION_HEADER = 'X-Skyflow-Authorization'
 
 
 class HttpStatusCode:
     OK = 200
     BAD_REQUEST = 400
+    UNAUTHORIZED = 401
     INTERNAL_SERVER_ERROR = 500
 
 
@@ -123,6 +125,8 @@ class CredentialField:
     API_KEY = 'api_key'
     TOKEN = 'token'
     PATH = 'path'
+    CONTEXT = 'context'
+    ROLES = 'roles'
 
 
 class JwtField:
@@ -165,3 +169,114 @@ class RequestParameter:
     COLUMN_GROUP = 'column_group'
     REDACTION = 'redaction'
 
+
+class FileUploadField:
+    TABLE = 'table'
+    SKYFLOW_ID = 'skyflow_id'
+    COLUMN_NAME = 'column_name'
+    FILE_PATH = 'file_path'
+    BASE64 = 'base64'
+    FILE_OBJECT = 'file_object'
+    FILE_NAME = 'file_name'
+    FILE = 'file'
+    NAME = 'name'
+
+
+class DeidentifyFileRequestField:
+    ENTITIES = 'entities'
+    ALLOW_REGEX_LIST = 'allow_regex_list'
+    RESTRICT_REGEX_LIST = 'restrict_regex_list'
+    OUTPUT_PROCESSED_IMAGE = 'output_processed_image'
+    OUTPUT_OCR_TEXT = 'output_ocr_text'
+    MASKING_METHOD = 'masking_method'
+    PIXEL_DENSITY = 'pixel_density'
+    MAX_RESOLUTION = 'max_resolution'
+    OUTPUT_PROCESSED_AUDIO = 'output_processed_audio'
+    OUTPUT_TRANSCRIPTION = 'output_transcription'
+    BLEEP = 'bleep'
+    OUTPUT_DIRECTORY = 'output_directory'
+    WAIT_TIME = 'wait_time'
+
+
+class DeidentifyField:
+    TEXT = 'text'
+    ENTITY_TYPES = 'entity_types'
+    TOKEN_TYPE = 'token_type'
+    ALLOW_REGEX = 'allow_regex'
+    RESTRICT_REGEX = 'restrict_regex'
+    TRANSFORMATIONS = 'transformations'
+    FORMAT = 'format'
+    OUTPUT = 'output'
+    STATUS = 'status'
+    RUN_ID = 'run_id'
+    WORD_CHARACTER_COUNT = 'word_character_count'
+    WORD_COUNT = 'word_count'
+    CHARACTER_COUNT = 'character_count'
+    SIZE = 'size'
+    DURATION = 'duration'
+    PAGES = 'pages'
+    SLIDES = 'slides'
+    PROCESSED_FILE = 'processed_file'
+    PROCESSED_FILE_TYPE = 'processed_file_type'
+    PROCESSED_FILE_EXTENSION = 'processed_file_extension'
+    REDACTED_FILE = 'redacted_file'
+    SHIFT_DATES = 'shift_dates'
+    DEFAULT = 'default'
+    ENTITY_UNQ_COUNTER = 'entity_unq_counter'
+    ENTITY_UNIQUE_COUNTER = 'entity_unique_counter'
+    ENTITY_ONLY = 'entity_only'
+    ENTITIES = 'entities'
+    MAX_DAYS = 'max_days'
+    MIN_DAYS = 'min_days'
+    MAX = 'max'
+    MIN = 'min'
+    FILE = 'file'
+    TYPE = 'type'
+    EXTENSION = 'extension'
+    IN_PROGRESS = 'IN_PROGRESS'
+    REQUEST_OPTIONS = 'request_options'
+    BLEEP_GAIN = 'bleep_gain'
+    BLEEP_FREQUENCY = 'bleep_frequency'
+    BLEEP_START_PADDING = 'bleep_start_padding'
+    BLEEP_STOP_PADDING = 'bleep_stop_padding'
+    DENSITY = 'density'
+    TOKEN_FORMAT = 'token_format'
+    PROCESSED_FILE_RESPONSE_KEY = 'processedFile'
+    PROCESSED_FILE_TYPE_RESPONSE_KEY = 'processedFileType'
+    PROCESSED_FILE_EXTENSION_RESPONSE_KEY = 'processedFileExtension'
+
+
+class RequestOperation:
+    INSERT = 'INSERT'
+    DELETE = 'DELETE'
+    GET = 'GET'
+    UPDATE = 'UPDATE'
+    QUERY = 'QUERY'
+    TOKENIZE = 'TOKENIZE'
+    DETOKENIZE = 'DETOKENIZE'
+    FILE_UPLOAD = 'FILE_UPLOAD'
+
+
+class ConfigType:
+    VAULT = 'vault'
+    CONNECTION = 'connection'
+
+
+class SqlCommand:
+    SELECT = 'SELECT'
+
+
+class SdkPrefix:
+    SKYFLOW_PYTHON = 'skyflow-python@'
+    PYTHON_RUNTIME = 'Python '
+
+
+class SdkMetricsKey:
+    SDK_NAME_VERSION = 'sdk_name_version'
+    SDK_CLIENT_DEVICE_MODEL = 'sdk_client_device_model'
+    SDK_CLIENT_OS_DETAILS = 'sdk_client_os_details'
+    SDK_RUNTIME_DETAILS = 'sdk_runtime_details'
+
+
+class ErrorDefaults:
+    UNKNOWN_REQUEST_ID = 'unknown-request-id'