Skip to content

Commit 7dc2604

Browse files
skyflow-bhartiskyflow-bharti
authored
Merge pull request #208 from skyflowapi/SK-2274-detokenize-public-interface-for-v-3-flow-db-in-java-sdk
SK-2274 detokenize bulk async and sync method
2 parents 294824d + c8fe733 commit 7dc2604

File tree

14 files changed

+915
-26
lines changed

14 files changed

+915
-26
lines changed

‎common/src/main/java/com/skyflow/errors/ErrorMessage.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,16 @@ public enum ErrorMessage {
150150
FailedToSaveProcessedFile("%s0 Validation error. Failed to save the processed file. Ensure the output directory is valid and writable."),
151151
InvalidAudioFileType("%s0 Validation error. The file type is not supported. Specify a valid file type mp3 or wav."),
152152
// Generic
153-
ErrorOccurred("%s0 API error. Error occurred.")
153+
ErrorOccurred("%s0 API error. Error occurred."),
154+
155+
DetokenizeRequestNull("%s0 Validation error. DetokenizeRequest object is null. Specify a valid DetokenizeRequest object."),
156+
157+
NullTokenGroupRedactions("%s0 Validation error. TokenGroupRedaction in the list is null. Specify a valid TokenGroupRedactions object."),
158+
159+
NullRedactionInTokenGroup("%s0 Validation error. Redaction in TokenGroupRedactions is null or empty. Specify a valid redaction."),
160+
161+
NullTokenGroupNameInTokenGroup("%s0 Validation error. TokenGroupName in TokenGroupRedactions is null or empty. Specify a valid tokenGroupName."),
162+
;
154163
;
155164
private final String message;
156165

‎common/src/main/java/com/skyflow/logs/ErrorLogs.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ public enum ErrorLogs {
6666
EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA("Invalid %s1 request. Token can not be null or empty in detokenize data at index %s2."),
6767
REDACTION_IS_REQUIRED("Invalid %s1 request. Redaction is required."),
6868
DETOKENIZE_REQUEST_REJECTED("Detokenize request resulted in failure."),
69+
DETOKENIZE_REQUEST_NULL("Invalid %s1 request. Detokenize request can not be null."),
70+
71+
NULL_TOKEN_REDACTION_GROUP_OBJECT("Invalid %s1 request. Token Redaction group object can not be null or empty."),
72+
73+
NULL_REDACTION_IN_TOKEN_GROUP("Invalid %s1 request. Redaction can not be null in token redaction group"),
74+
75+
NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP("Invalid %s1 request. Token group name can not be null in token redaction group"),
76+
77+
EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP("Invalid %s1 request. Redaction can not be null or empty in token redaction group"),
6978
IDS_IS_REQUIRED("Invalid %s1 request. Ids are required."),
7079
EMPTY_IDS("Invalid %s1 request. Ids can not be empty."),
7180
EMPTY_OR_NULL_ID_IN_IDS("Invalid %s1 request. Id can not be null or empty in ids at index %s2."),

‎v3/src/main/java/com/skyflow/VaultClient.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.skyflow.utils.Utils;
2020
import com.skyflow.utils.logger.LogUtil;
2121
import com.skyflow.utils.validations.Validations;
22+
import com.skyflow.vault.data.DetokenizeRequest;
2223
import io.github.cdimascio.dotenv.Dotenv;
2324
import io.github.cdimascio.dotenv.DotenvException;
2425
import okhttp3.OkHttpClient;
@@ -155,4 +156,25 @@ protected InsertRequest getBulkInsertRequestBody(com.skyflow.vault.data.InsertRe
155156

156157
}
157158

159+
protected com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest getDetokenizeRequestBody(DetokenizeRequest request) {
160+
List<String> tokens = request.getTokens();
161+
com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.Builder builder =
162+
com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.builder()
163+
.vaultId(this.vaultConfig.getVaultId())
164+
.tokens(tokens);
165+
if (request.getTokenGroupRedactions() != null){
166+
List<com.skyflow.generated.rest.types.TokenGroupRedactions> tokenGroupRedactionsList = new ArrayList<>();
167+
for (com.skyflow.vault.data.TokenGroupRedactions tokenGroupRedactions : request.getTokenGroupRedactions()) {
168+
com.skyflow.generated.rest.types.TokenGroupRedactions redactions =
169+
com.skyflow.generated.rest.types.TokenGroupRedactions.builder()
170+
.tokenGroupName(tokenGroupRedactions.getTokenGroupName())
171+
.redaction(tokenGroupRedactions.getRedaction())
172+
.build();
173+
tokenGroupRedactionsList.add(redactions);
174+
}
175+
176+
builder.tokenGroupRedactions(tokenGroupRedactionsList);
177+
}
178+
return builder.build();
179+
}
158180
}

‎v3/src/main/java/com/skyflow/utils/Constants.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ public final class Constants extends BaseConstants {
99
public static final Integer MAX_INSERT_BATCH_SIZE = 1000;
1010
public static final Integer INSERT_CONCURRENCY_LIMIT = 10;
1111
public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10;
12-
public static final Integer DETOKENIZE_BATCH_SIZE = 100;
12+
public static final Integer DETOKENIZE_BATCH_SIZE = 50;
1313
public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 10;
14+
15+
public static final Integer MAX_DETOKENIZE_BATCH_SIZE = 1000;
16+
public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 10;
17+
1418
}

‎v3/src/main/java/com/skyflow/utils/Utils.java‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@
33
import com.google.gson.JsonObject;
44
import com.skyflow.enums.Env;
55
import com.skyflow.generated.rest.core.ApiClientApiException;
6+
import com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest;
67
import com.skyflow.generated.rest.types.InsertRecordData;
78
import com.skyflow.generated.rest.types.InsertResponse;
89
import com.skyflow.generated.rest.types.RecordResponseObject;
10+
import com.skyflow.generated.rest.types.TokenGroupRedactions;
11+
import com.skyflow.vault.data.DetokenizeResponse;
912
import com.skyflow.vault.data.ErrorRecord;
1013
import com.skyflow.vault.data.Success;
1114
import com.skyflow.vault.data.Token;
@@ -36,6 +39,29 @@ public static List<List<InsertRecordData>> createBatches(List<InsertRecordData>
3639
return batches;
3740
}
3841

42+
public static List<DetokenizeRequest> createDetokenizeBatches(DetokenizeRequest request, int batchSize) {
43+
List<DetokenizeRequest> detokenizeRequests = new ArrayList<>();
44+
List<String> tokens = request.getTokens().get();
45+
46+
for (int i = 0; i < tokens.size(); i += batchSize) {
47+
// Create a sublist for the current batch
48+
List<String> batchTokens = tokens.subList(i, Math.min(i + batchSize, tokens.size()));
49+
List<TokenGroupRedactions> tokenGroupRedactions = null;
50+
if (request.getTokenGroupRedactions().isPresent() && !request.getTokenGroupRedactions().get().isEmpty() && i < request.getTokenGroupRedactions().get().size()) {
51+
tokenGroupRedactions = request.getTokenGroupRedactions().get().subList(i, Math.min(i + batchSize, request.getTokenGroupRedactions().get().size())); }
52+
// Build a new DetokenizeRequest for the current batch
53+
DetokenizeRequest batchRequest = DetokenizeRequest.builder()
54+
.vaultId(request.getVaultId())
55+
.tokens(new ArrayList<>(batchTokens))
56+
.tokenGroupRedactions(tokenGroupRedactions)
57+
.build();
58+
59+
detokenizeRequests.add(batchRequest);
60+
}
61+
62+
return detokenizeRequests;
63+
}
64+
3965
public static ErrorRecord createErrorRecord(Map<String, Object> recordMap, int indexNumber) {
4066
ErrorRecord err = null;
4167
if (recordMap != null) {
@@ -48,7 +74,7 @@ public static ErrorRecord createErrorRecord(Map<String, Object> recordMap, int i
4874
}
4975

5076
public static List<ErrorRecord> handleBatchException(
51-
Throwable ex, List<InsertRecordData> batch, int batchNumber, List<List<InsertRecordData>> batches
77+
Throwable ex, List<InsertRecordData> batch, int batchNumber
5278
) {
5379
List<ErrorRecord> errorRecords = new ArrayList<>();
5480
Throwable cause = ex.getCause();
@@ -90,6 +116,72 @@ public static List<ErrorRecord> handleBatchException(
90116
return errorRecords;
91117
}
92118

119+
public static List<ErrorRecord> handleDetokenizeBatchException(
120+
Throwable ex, DetokenizeRequest batch, int batchNumber, int batchSize
121+
) {
122+
List<ErrorRecord> errorRecords = new ArrayList<>();
123+
Throwable cause = ex.getCause();
124+
if (cause instanceof ApiClientApiException) {
125+
ApiClientApiException apiException = (ApiClientApiException) cause;
126+
Map<String, Object> responseBody = (Map<String, Object>) apiException.body();
127+
int indexNumber = batchNumber * batchSize;
128+
if (responseBody != null) {
129+
if (responseBody.containsKey("records")) {
130+
Object recordss = responseBody.get("records");
131+
if (recordss instanceof List) {
132+
List<?> recordsList = (List<?>) recordss;
133+
for (Object record : recordsList) {
134+
if (record instanceof Map) {
135+
Map<String, Object> recordMap = (Map<String, Object>) record;
136+
ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber);
137+
errorRecords.add(err);
138+
indexNumber++;
139+
}
140+
}
141+
}
142+
} else if (responseBody.containsKey("error")) {
143+
Map<String, Object> recordMap = (Map<String, Object>) responseBody.get("error");
144+
for (int j = 0; j < batch.getTokens().get().size(); j++) {
145+
ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber);
146+
errorRecords.add(err);
147+
indexNumber++;
148+
}
149+
}
150+
}
151+
} else {
152+
int indexNumber = batchNumber * batchSize;
153+
for (int j = 0; j < batch.getTokens().get().size(); j++) {
154+
ErrorRecord err = new ErrorRecord(indexNumber, ex.getMessage(), 500);
155+
errorRecords.add(err);
156+
indexNumber++;
157+
}
158+
}
159+
return errorRecords;
160+
}
161+
162+
public static DetokenizeResponse formatDetokenizeResponse(com.skyflow.generated.rest.types.DetokenizeResponse response, int batch, int batchSize) {
163+
if (response != null) {
164+
List<com.skyflow.generated.rest.types.DetokenizeResponseObject> record = response.getResponse().get();
165+
List<ErrorRecord> errorRecords = new ArrayList<>();
166+
List<com.skyflow.vault.data.DetokenizeResponseObject> successRecords = new ArrayList<>();
167+
int indexNumber = batch * batchSize;
168+
int recordsSize = response.getResponse().get().size();
169+
for (int index = 0; index < recordsSize; index++) {
170+
if (record.get(index).getError().isPresent()) {
171+
ErrorRecord errorRecord = new ErrorRecord(indexNumber, record.get(index).getError().get(), record.get(index).getHttpCode().get());
172+
errorRecords.add(errorRecord);
173+
} else {
174+
com.skyflow.vault.data.DetokenizeResponseObject success = new com.skyflow.vault.data.DetokenizeResponseObject(indexNumber, record.get(index).getToken().orElse(null), record.get(index).getValue().orElse(null), record.get(index).getTokenGroupName().orElse(null), record.get(index).getError().orElse(null), record.get(index).getMetadata().orElse(null));
175+
successRecords.add(success);
176+
}
177+
indexNumber++;
178+
}
179+
DetokenizeResponse formattedResponse = new DetokenizeResponse(successRecords, errorRecords);
180+
return formattedResponse;
181+
}
182+
return null;
183+
}
184+
93185
public static com.skyflow.vault.data.InsertResponse formatResponse(InsertResponse response, int batch, int batchSize) {
94186
com.skyflow.vault.data.InsertResponse formattedResponse = null;
95187
List<Success> successRecords = new ArrayList<>();

‎v3/src/main/java/com/skyflow/utils/validations/Validations.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
import com.skyflow.logs.ErrorLogs;
88
import com.skyflow.utils.Utils;
99
import com.skyflow.utils.logger.LogUtil;
10+
import com.skyflow.vault.data.DetokenizeRequest;
1011
import com.skyflow.vault.data.InsertRequest;
12+
import com.skyflow.vault.data.TokenGroupRedactions;
1113

1214
import java.util.ArrayList;
1315
import java.util.HashMap;
@@ -72,4 +74,48 @@ public static void validateInsertRequest(InsertRequest insertRequest) throws Sky
7274
}
7375
}
7476

77+
public static void validateDetokenizeRequest(DetokenizeRequest request) throws SkyflowException {
78+
if (request == null) {
79+
LogUtil.printErrorLog(Utils.parameterizedString(
80+
ErrorLogs.DETOKENIZE_REQUEST_NULL.getLog(), InterfaceName.DETOKENIZE.getName()
81+
));
82+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.DetokenizeRequestNull.getMessage());
83+
}
84+
ArrayList<String> tokens = request.getTokens();
85+
if (tokens == null || tokens.isEmpty()) {
86+
LogUtil.printErrorLog(Utils.parameterizedString(
87+
ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
88+
));
89+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage());
90+
}
91+
for (String token : tokens) {
92+
if (token == null || token.trim().isEmpty()) {
93+
LogUtil.printErrorLog(Utils.parameterizedString(
94+
ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
95+
));
96+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyTokenInDetokenizeData.getMessage());
97+
}
98+
}
99+
ArrayList<TokenGroupRedactions> groupRedactions = request.getTokenGroupRedactions();
100+
if (groupRedactions != null && !groupRedactions.isEmpty()) {
101+
for (TokenGroupRedactions group : groupRedactions) {
102+
if (group == null) {
103+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName()));
104+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage());
105+
}
106+
String groupName = group.getTokenGroupName();
107+
String redaction = group.getRedaction();
108+
if (groupName == null || groupName.trim().isEmpty()) {
109+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
110+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage());
111+
}
112+
if (redaction == null || redaction.trim().isEmpty()) {
113+
LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
114+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage());
115+
}
116+
}
117+
}
118+
119+
}
120+
75121
}

0 commit comments

Comments
 (0)