Skip to content

Commit 078f5f7

Browse files
skyflow-vivekskyflow-vivek
authored
Merge pull request #176 from skyflowapi/SK-2042
SK-2042 Add deidentify and reidentify text interfaces for Detect service
2 parents cdd67ed + 66e6377 commit 078f5f7

File tree

138 files changed

+10454
-268
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

138 files changed

+10454
-268
lines changed

‎.github/workflows/shared-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- name: 'Run Tests'
2828
run: |
2929
pip install -r requirements.txt
30-
python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover
30+
python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover
3131
3232
- name: coverage
3333
run: coverage xml -o test-coverage.xml

‎skyflow/client/skyflow.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from skyflow.vault.client.client import VaultClient
99
from skyflow.vault.controller import Vault
1010
from skyflow.vault.controller import Connection
11+
from skyflow.vault.controller import Detect
1112

1213
class Skyflow:
1314
def __init__(self, builder):
@@ -65,11 +66,15 @@ def update_log_level(self, log_level):
6566

6667
def vault(self, vault_id = None) -> Vault:
6768
vault_config = self.__builder.get_vault_config(vault_id)
68-
return vault_config.get("controller")
69+
return vault_config.get("vault_controller")
6970

7071
def connection(self, connection_id = None) -> Connection:
7172
connection_config = self.__builder.get_connection_config(connection_id)
7273
return connection_config.get("controller")
74+
75+
def detect(self, vault_id = None) -> Detect:
76+
vault_config = self.__builder.get_vault_config(vault_id)
77+
return vault_config.get("detect_controller")
7378

7479
class Builder:
7580
def __init__(self):
@@ -182,9 +187,11 @@ def __add_vault_config(self, config):
182187
vault_client = VaultClient(config)
183188
self.__vault_configs[vault_id] = {
184189
"vault_client": vault_client,
185-
"controller": Vault(vault_client)
190+
"vault_controller": Vault(vault_client),
191+
"detect_controller": Detect(vault_client)
186192
}
187193
log_info(SkyflowMessages.Info.VAULT_CONTROLLER_INITIALIZED.value.format(config.get("vault_id")), self.__logger)
194+
log_info(SkyflowMessages.Info.DETECT_CONTROLLER_INITIALIZED.value.format(config.get("vault_id")), self.__logger)
188195

189196
def __add_connection_config(self, config):
190197
validate_connection_config(self.__logger, config)

‎skyflow/generated/rest/__init__.py‎

Lines changed: 150 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,57 @@
33
# isort: skip_file
44

55
from .types import (
6+
AdvancedOptionsColumnMapping,
7+
AdvancedOptionsEntityColumnMap,
8+
AdvancedOptionsVaultSchema,
9+
AllowRegex,
10+
AudioConfigTranscriptionType,
611
AuditEventAuditResourceType,
712
AuditEventContext,
813
AuditEventData,
914
AuditEventHttpInfo,
1015
BatchRecordMethod,
1116
ContextAccessType,
1217
ContextAuthMode,
18+
DeidentifyFileOutput,
19+
DeidentifyFileOutputProcessedFileType,
20+
DeidentifyFileResponse,
21+
DeidentifyStatusResponse,
22+
DeidentifyStatusResponseOutputType,
23+
DeidentifyStatusResponseStatus,
24+
DeidentifyStatusResponseWordCharacterCount,
25+
DeidentifyStringResponse,
26+
DetectDataAccuracy,
27+
DetectDataEntities,
28+
DetectFileRequestDataType,
29+
DetectRequestDeidentifyOption,
30+
DetectedEntity,
1331
DetokenizeRecordResponseValueType,
32+
EntityLocation,
33+
EntityType,
34+
EntityTypes,
35+
ErrorResponse,
36+
ErrorResponseError,
37+
ErrorString,
1438
GooglerpcStatus,
39+
ProcessedFileOutputProcessedFileType,
1540
ProtobufAny,
1641
RedactionEnumRedaction,
42+
ReidentifyStringResponse,
1743
RequestActionType,
44+
ResourceId,
45+
RestrictRegex,
46+
TokenType,
47+
TokenTypeDefault,
48+
TokenTypeWithoutVault,
49+
TokenTypeWithoutVaultDefault,
50+
Transformations,
51+
TransformationsShiftDates,
52+
TransformationsShiftDatesEntityTypesItem,
53+
Uuid,
54+
V1AdvancedOptions,
55+
V1AudioConfig,
56+
V1AudioOptions,
1857
V1AuditAfterOptions,
1958
V1AuditEventResponse,
2059
V1AuditResponse,
@@ -29,26 +68,39 @@
2968
V1Card,
3069
V1DeleteFileResponse,
3170
V1DeleteRecordResponse,
71+
V1DetectFileResponse,
72+
V1DetectStatusResponse,
73+
V1DetectStatusResponseStatus,
74+
V1DetectTextRequest,
75+
V1DetectTextResponse,
3276
V1DetokenizeRecordRequest,
3377
V1DetokenizeRecordResponse,
3478
V1DetokenizeResponse,
3579
V1FieldRecords,
3680
V1FileAvScanStatus,
81+
V1FileDataFormat,
3782
V1GetAuthTokenResponse,
3883
V1GetFileScanStatusResponse,
3984
V1GetQueryResponse,
85+
V1ImageOptions,
4086
V1InsertRecordResponse,
87+
V1Locations,
4188
V1MemberType,
89+
V1PdfConfig,
90+
V1PdfOptions,
91+
V1ProcessedFileOutput,
4292
V1RecordMetaProperties,
93+
V1ResponseEntities,
4394
V1TokenizeRecordRequest,
4495
V1TokenizeRecordResponse,
4596
V1TokenizeResponse,
4697
V1UpdateRecordResponse,
4798
V1VaultFieldMapping,
4899
V1VaultSchemaConfig,
100+
VaultId,
49101
)
50-
from .errors import BadRequestError, NotFoundError, UnauthorizedError
51-
from . import audit, authentication, bin_lookup, query, records, tokens
102+
from .errors import BadRequestError, InternalServerError, NotFoundError, UnauthorizedError
103+
from . import audit, authentication, bin_lookup, deprecated, files, query, records, strings, tokens
52104
from .audit import (
53105
AuditServiceListAuditEventsRequestFilterOpsActionType,
54106
AuditServiceListAuditEventsRequestFilterOpsContextAccessType,
@@ -59,15 +111,41 @@
59111
)
60112
from .client import AsyncSkyflow, Skyflow
61113
from .environment import SkyflowEnvironment
114+
from .files import (
115+
DeidentifyAudioRequestFile,
116+
DeidentifyAudioRequestFileDataFormat,
117+
DeidentifyAudioRequestOutputTranscription,
118+
DeidentifyDocumentRequestFile,
119+
DeidentifyDocumentRequestFileDataFormat,
120+
DeidentifyFileRequestFile,
121+
DeidentifyFileRequestFileDataFormat,
122+
DeidentifyImageRequestFile,
123+
DeidentifyImageRequestFileDataFormat,
124+
DeidentifyImageRequestMaskingMethod,
125+
DeidentifyPdfRequestFile,
126+
DeidentifyPresentationRequestFile,
127+
DeidentifyPresentationRequestFileDataFormat,
128+
DeidentifySpreadsheetRequestFile,
129+
DeidentifySpreadsheetRequestFileDataFormat,
130+
DeidentifyStructuredTextRequestFile,
131+
DeidentifyStructuredTextRequestFileDataFormat,
132+
DeidentifyTextRequestFile,
133+
)
62134
from .records import (
63135
RecordServiceBulkGetRecordRequestOrderBy,
64136
RecordServiceBulkGetRecordRequestRedaction,
65137
RecordServiceGetRecordRequestRedaction,
66138
)
139+
from .strings import ReidentifyStringRequestFormat
67140
from .version import __version__
68141

69142
__all__ = [
143+
"AdvancedOptionsColumnMapping",
144+
"AdvancedOptionsEntityColumnMap",
145+
"AdvancedOptionsVaultSchema",
146+
"AllowRegex",
70147
"AsyncSkyflow",
148+
"AudioConfigTranscriptionType",
71149
"AuditEventAuditResourceType",
72150
"AuditEventContext",
73151
"AuditEventData",
@@ -82,18 +160,72 @@
82160
"BatchRecordMethod",
83161
"ContextAccessType",
84162
"ContextAuthMode",
163+
"DeidentifyAudioRequestFile",
164+
"DeidentifyAudioRequestFileDataFormat",
165+
"DeidentifyAudioRequestOutputTranscription",
166+
"DeidentifyDocumentRequestFile",
167+
"DeidentifyDocumentRequestFileDataFormat",
168+
"DeidentifyFileOutput",
169+
"DeidentifyFileOutputProcessedFileType",
170+
"DeidentifyFileRequestFile",
171+
"DeidentifyFileRequestFileDataFormat",
172+
"DeidentifyFileResponse",
173+
"DeidentifyImageRequestFile",
174+
"DeidentifyImageRequestFileDataFormat",
175+
"DeidentifyImageRequestMaskingMethod",
176+
"DeidentifyPdfRequestFile",
177+
"DeidentifyPresentationRequestFile",
178+
"DeidentifyPresentationRequestFileDataFormat",
179+
"DeidentifySpreadsheetRequestFile",
180+
"DeidentifySpreadsheetRequestFileDataFormat",
181+
"DeidentifyStatusResponse",
182+
"DeidentifyStatusResponseOutputType",
183+
"DeidentifyStatusResponseStatus",
184+
"DeidentifyStatusResponseWordCharacterCount",
185+
"DeidentifyStringResponse",
186+
"DeidentifyStructuredTextRequestFile",
187+
"DeidentifyStructuredTextRequestFileDataFormat",
188+
"DeidentifyTextRequestFile",
189+
"DetectDataAccuracy",
190+
"DetectDataEntities",
191+
"DetectFileRequestDataType",
192+
"DetectRequestDeidentifyOption",
193+
"DetectedEntity",
85194
"DetokenizeRecordResponseValueType",
195+
"EntityLocation",
196+
"EntityType",
197+
"EntityTypes",
198+
"ErrorResponse",
199+
"ErrorResponseError",
200+
"ErrorString",
86201
"GooglerpcStatus",
202+
"InternalServerError",
87203
"NotFoundError",
204+
"ProcessedFileOutputProcessedFileType",
88205
"ProtobufAny",
89206
"RecordServiceBulkGetRecordRequestOrderBy",
90207
"RecordServiceBulkGetRecordRequestRedaction",
91208
"RecordServiceGetRecordRequestRedaction",
92209
"RedactionEnumRedaction",
210+
"ReidentifyStringRequestFormat",
211+
"ReidentifyStringResponse",
93212
"RequestActionType",
213+
"ResourceId",
214+
"RestrictRegex",
94215
"Skyflow",
95216
"SkyflowEnvironment",
217+
"TokenType",
218+
"TokenTypeDefault",
219+
"TokenTypeWithoutVault",
220+
"TokenTypeWithoutVaultDefault",
221+
"Transformations",
222+
"TransformationsShiftDates",
223+
"TransformationsShiftDatesEntityTypesItem",
96224
"UnauthorizedError",
225+
"Uuid",
226+
"V1AdvancedOptions",
227+
"V1AudioConfig",
228+
"V1AudioOptions",
97229
"V1AuditAfterOptions",
98230
"V1AuditEventResponse",
99231
"V1AuditResponse",
@@ -108,28 +240,44 @@
108240
"V1Card",
109241
"V1DeleteFileResponse",
110242
"V1DeleteRecordResponse",
243+
"V1DetectFileResponse",
244+
"V1DetectStatusResponse",
245+
"V1DetectStatusResponseStatus",
246+
"V1DetectTextRequest",
247+
"V1DetectTextResponse",
111248
"V1DetokenizeRecordRequest",
112249
"V1DetokenizeRecordResponse",
113250
"V1DetokenizeResponse",
114251
"V1FieldRecords",
115252
"V1FileAvScanStatus",
253+
"V1FileDataFormat",
116254
"V1GetAuthTokenResponse",
117255
"V1GetFileScanStatusResponse",
118256
"V1GetQueryResponse",
257+
"V1ImageOptions",
119258
"V1InsertRecordResponse",
259+
"V1Locations",
120260
"V1MemberType",
261+
"V1PdfConfig",
262+
"V1PdfOptions",
263+
"V1ProcessedFileOutput",
121264
"V1RecordMetaProperties",
265+
"V1ResponseEntities",
122266
"V1TokenizeRecordRequest",
123267
"V1TokenizeRecordResponse",
124268
"V1TokenizeResponse",
125269
"V1UpdateRecordResponse",
126270
"V1VaultFieldMapping",
127271
"V1VaultSchemaConfig",
272+
"VaultId",
128273
"__version__",
129274
"audit",
130275
"authentication",
131276
"bin_lookup",
277+
"deprecated",
278+
"files",
132279
"query",
133280
"records",
281+
"strings",
134282
"tokens",
135283
]

‎skyflow/generated/rest/audit/client.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -200,8 +200,13 @@ def audit_service_list_audit_events(
200200
Examples
201201
--------
202202
from skyflow import Skyflow
203-
client = Skyflow(token="YOUR_TOKEN", )
204-
client.audit.audit_service_list_audit_events(filter_ops_account_id='filterOps.accountID', )
203+
204+
client = Skyflow(
205+
token="YOUR_TOKEN",
206+
)
207+
client.audit.audit_service_list_audit_events(
208+
filter_ops_account_id="filterOps.accountID",
209+
)
205210
"""
206211
_response = self._raw_client.audit_service_list_audit_events(
207212
filter_ops_account_id=filter_ops_account_id,
@@ -415,11 +420,21 @@ async def audit_service_list_audit_events(
415420
416421
Examples
417422
--------
418-
from skyflow import AsyncSkyflow
419423
import asyncio
420-
client = AsyncSkyflow(token="YOUR_TOKEN", )
424+
425+
from skyflow import AsyncSkyflow
426+
427+
client = AsyncSkyflow(
428+
token="YOUR_TOKEN",
429+
)
430+
431+
421432
async def main() -> None:
422-
await client.audit.audit_service_list_audit_events(filter_ops_account_id='filterOps.accountID', )
433+
await client.audit.audit_service_list_audit_events(
434+
filter_ops_account_id="filterOps.accountID",
435+
)
436+
437+
423438
asyncio.run(main())
424439
"""
425440
_response = await self._raw_client.audit_service_list_audit_events(

‎skyflow/generated/rest/audit/raw_client.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,9 @@ def audit_service_list_audit_events(
244244
raise NotFoundError(
245245
headers=dict(_response.headers),
246246
body=typing.cast(
247-
typing.Dict[str, typing.Optional[typing.Any]],
247+
typing.Optional[typing.Any],
248248
parse_obj_as(
249-
type_=typing.Dict[str, typing.Optional[typing.Any]], # type: ignore
249+
type_=typing.Optional[typing.Any], # type: ignore
250250
object_=_response.json(),
251251
),
252252
),
@@ -471,9 +471,9 @@ async def audit_service_list_audit_events(
471471
raise NotFoundError(
472472
headers=dict(_response.headers),
473473
body=typing.cast(
474-
typing.Dict[str, typing.Optional[typing.Any]],
474+
typing.Optional[typing.Any],
475475
parse_obj_as(
476-
type_=typing.Dict[str, typing.Optional[typing.Any]], # type: ignore
476+
type_=typing.Optional[typing.Any], # type: ignore
477477
object_=_response.json(),
478478
),
479479
),

0 commit comments

Comments
 (0)