Skip to content

Commit 04d886d

Browse files
skyflow-shravanskyflow-vivek
skyflow-shravan
and
skyflow-vivek
authored
SK-2286 Update branch with latest code (#229)
* SK-2286 V3 release/25.9.3 (#224) * SK-2286 take vault url from env and override (#223) * SK-2286 take vault url from env and override * SK-2286 add validation for vault url format * SK-2286 add validation for vault url format * [AUTOMATED] Private Release 2.0.0-beta.4-dev-b64524d * SK-2286 throw skyflow exception for empty vault url * SK-2286 add error logs for vault url validations * SK-2286 add error logs for vault url validations * [AUTOMATED] Private Release 2.0.0-beta.4-dev-f012079 --------- Co-authored-by: skyflow-shravan <skyflow-shravan@users.noreply.github.com> * SK-2286 override vault url and accept skyflow creds from env (#225) * SK-2286 accept skyflow creds from system env variables * [AUTOMATED] Private Release 2.0.0-beta.4-dev-4b56b3d --------- Co-authored-by: skyflow-shravan <skyflow-shravan@users.noreply.github.com> * SK-2289 Fix upsert operation not working issue * [AUTOMATED] Private Release 2.0.0-beta.4-dev-d3257dc * SK-2289 Update SDK version in constants * [AUTOMATED] Public Release - 3.0.0-beta.4 --------- Co-authored-by: skyflow-shravan <skyflow-shravan@users.noreply.github.com> Co-authored-by: skyflow-vivek <vivek.jain@skyflow.com> Co-authored-by: skyflow-vivek <121149418+skyflow-vivek@users.noreply.github.com> Co-authored-by: skyflow-vivek <skyflow-vivek@users.noreply.github.com>
1 parent bb87aa9 commit 04d886d

File tree

13 files changed

+165
-47
lines changed

13 files changed

+165
-47
lines changed

‎common/src/main/java/com/skyflow/errors/ErrorMessage.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ public enum ErrorMessage {
1616
EmptyVaultId("%s0 Initialization failed. Invalid vault ID. Vault ID must not be empty."),
1717
InvalidClusterId("%s0 Initialization failed. Invalid cluster ID. Specify cluster ID."),
1818
EmptyClusterId("%s0 Initialization failed. Invalid cluster ID. Specify a valid cluster ID."),
19+
EmptyVaultUrl("%s0 Initialization failed. Vault URL is empty. Specify a valid vault URL."),
20+
InvalidVaultUrlFormat("%s0 Initialization failed. Vault URL must start with 'https://'."),
1921

2022
// Connection config
2123
InvalidConnectionId("%s0 Initialization failed. Invalid connection ID. Specify a valid connection ID."),
@@ -66,18 +68,21 @@ public enum ErrorMessage {
6668
EmptyKeyInTokens("%s0 Validation error. Invalid key tokens. Specify a valid key."),
6769
EmptyValueInTokens("%s0 Validation error. Invalid value in tokens. Specify a valid value."),
6870
EmptyUpsert("%s0 Validation error. 'upsert' key can't be empty. Specify an upsert column."),
71+
EmptyUpsertValues("%s0 Validation error. Upsert column values can't be empty. Specify at least one upsert column."),
6972
HomogenousNotSupportedWithUpsert("%s0 Validation error. 'homogenous' is not supported with 'upsert'. Specify either 'homogenous' or 'upsert'."),
7073
TokensPassedForTokenModeDisable("%s0 Validation error. 'tokenMode' wasn't specified. Set 'tokenMode' to 'ENABLE' to insert tokens."),
7174
NoTokensWithTokenMode("%s0 Validation error. Tokens weren't specified for records while 'tokenMode' was %s1. Specify tokens."),
7275
MismatchOfFieldsAndTokens("%s0 Validation error. 'fields' and 'tokens' have different columns names. Verify that 'fields' and 'tokens' columns match."),
7376
InsufficientTokensPassedForTokenModeEnableStrict("%s0 Validation error. 'tokenMode' is set to 'ENABLE_STRICT', but some fields are missing tokens. Specify tokens for all fields."),
7477
BatchInsertPartialSuccess("%s0 Insert operation completed with partial success."),
7578
BatchInsertFailure("%s0 Insert operation failed."),
79+
RecordSizeExceedError("%s0 Maximum number of records exceeded. The limit is 10000."),
7680

7781
// Detokenize
7882
InvalidDetokenizeData("%s0 Validation error. Invalid detokenize data. Specify valid detokenize data."),
7983
EmptyDetokenizeData("%s0 Validation error. Invalid data tokens. Specify at least one data token."),
8084
EmptyTokenInDetokenizeData("%s0 Validation error. Invalid data tokens. Specify a valid data token."),
85+
TokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 10000."),
8186

8287
// Get
8388
IdsKeyError("%s0 Validation error. 'ids' key is missing from the payload. Specify an 'ids' key."),

‎common/src/main/java/com/skyflow/logs/ErrorLogs.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ public enum ErrorLogs {
2525
EMPTY_ROLES("Invalid credentials. Roles can not be empty."),
2626
EMPTY_OR_NULL_ROLE_IN_ROLES("Invalid credentials. Role can not be null or empty in roles at index %s1."),
2727
EMPTY_OR_NULL_CONTEXT("Invalid credentials. Context can not be empty."),
28+
EMPTY_VAULT_URL("Invalid vault config. Vault URL can not be empty."),
29+
INVALID_VAULT_URL_FORMAT("Invalid vault config. Vault URL format is incorrect"),
2830

2931
// Bearer token generation
3032
INVALID_BEARER_TOKEN("Bearer token is invalid or expired."),
@@ -49,9 +51,12 @@ public enum ErrorLogs {
4951
EMPTY_TABLE_NAME("Invalid %s1 request. Table name can not be empty."),
5052
VALUES_IS_REQUIRED("Invalid %s1 request. Values are required."),
5153
EMPTY_VALUES("Invalid %s1 request. Values can not be empty."),
54+
RECORD_SIZE_EXCEED("Maximum number of records exceeded. The limit is 10000."),
55+
TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 10000."),
5256
EMPTY_OR_NULL_VALUE_IN_VALUES("Invalid %s1 request. Value can not be null or empty in values for key \"%s2\"."),
5357
EMPTY_OR_NULL_KEY_IN_VALUES("Invalid %s1 request. Key can not be null or empty in values"),
5458
EMPTY_UPSERT("Invalid %s1 request. Upsert can not be empty."),
59+
EMPTY_UPSERT_VALUES("Invalid %s1 request. Upsert values can not be empty."),
5560
HOMOGENOUS_NOT_SUPPORTED_WITH_UPSERT("Invalid %s1 request. Homogenous is not supported when upsert is passed."),
5661
TOKENS_NOT_ALLOWED_WITH_TOKEN_MODE_DISABLE("Invalid %s1 request. Tokens are not allowed when tokenMode is DISABLE."),
5762
TOKENS_REQUIRED_WITH_TOKEN_MODE("Invalid %s1 request. Tokens are required when tokenMode is %s2."),

‎common/src/main/java/com/skyflow/utils/BaseUtils.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import com.skyflow.serviceaccount.util.BearerToken;
1212
import com.skyflow.serviceaccount.util.Token;
1313
import com.skyflow.utils.logger.LogUtil;
14+
import io.github.cdimascio.dotenv.Dotenv;
15+
import io.github.cdimascio.dotenv.DotenvException;
1416
import org.apache.commons.codec.binary.Base64;
1517

1618
import java.io.File;
@@ -43,6 +45,7 @@ public static String getVaultURL(String clusterId, Env env, String vaultDomain)
4345
return sb.toString();
4446
}
4547

48+
4649
public static String generateBearerToken(Credentials credentials) throws SkyflowException {
4750
String bearerToken;
4851
if (credentials.getPath() != null) {

‎common/src/main/java/com/skyflow/utils/validations/BaseValidations.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ public static void validateVaultConfig(VaultConfig vaultConfig) throws SkyflowEx
3434
} else if (clusterId.trim().isEmpty()) {
3535
LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog());
3636
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage());
37-
} else if (credentials != null) {
37+
}
38+
else if (credentials != null) {
3839
validateCredentials(credentials);
3940
}
4041
}

‎v3/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</parent>
1212

1313
<artifactId>skyflow-java</artifactId>
14-
<version>3.0.0-beta.3</version>
14+
<version>3.0.0-beta.4</version>
1515
<packaging>jar</packaging>
1616
<name>${project.groupId}:${project.artifactId}</name>
1717
<description>Skyflow V3 SDK for the Java programming language</description>

‎v3/src/main/java/com/skyflow/Skyflow.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public SkyflowClientBuilder() {
5656

5757
public SkyflowClientBuilder addVaultConfig(VaultConfig vaultConfig) throws SkyflowException {
5858
LogUtil.printInfoLog(InfoLogs.VALIDATING_VAULT_CONFIG.getLog());
59-
Validations.validateVaultConfig(vaultConfig);
59+
Validations.validateVaultConfiguration(vaultConfig);
6060
VaultConfig vaultConfigCopy;
6161
try {
6262
vaultConfigCopy = (VaultConfig) vaultConfig.clone();

‎v3/src/main/java/com/skyflow/VaultClient.java‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public class VaultClient {
3939
private String token;
4040
private String apiKey;
4141

42-
protected VaultClient(VaultConfig vaultConfig, Credentials credentials) {
42+
protected VaultClient(VaultConfig vaultConfig, Credentials credentials) throws SkyflowException {
4343
super();
4444
this.vaultConfig = vaultConfig;
4545
this.commonCredentials = credentials;
@@ -79,8 +79,11 @@ protected void setBearerToken() throws SkyflowException {
7979
this.apiClient = this.apiClientBuilder.build();
8080
}
8181

82-
private void updateVaultURL() {
83-
String vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv());
82+
private void updateVaultURL() throws SkyflowException {
83+
String vaultURL = Utils.getEnvVaultURL();
84+
if (vaultURL == null || vaultURL.isEmpty()) {
85+
vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv());
86+
}
8487
this.apiClientBuilder.url(vaultURL);
8588
}
8689

@@ -92,11 +95,13 @@ private void prioritiseCredentials() throws SkyflowException {
9295
} else if (this.commonCredentials != null) {
9396
this.finalCredentials = this.commonCredentials;
9497
} else {
95-
Dotenv dotenv = Dotenv.load();
96-
String sysCredentials = dotenv.get(Constants.ENV_CREDENTIALS_KEY_NAME);
98+
String sysCredentials = System.getenv(Constants.ENV_CREDENTIALS_KEY_NAME);
99+
if (sysCredentials == null) {
100+
Dotenv dotenv = Dotenv.load();
101+
sysCredentials = dotenv.get(Constants.ENV_CREDENTIALS_KEY_NAME);
102+
}
97103
if (sysCredentials == null) {
98-
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(),
99-
ErrorMessage.EmptyCredentials.getMessage());
104+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyCredentials.getMessage());
100105
} else {
101106
this.finalCredentials = new Credentials();
102107
this.finalCredentials.setCredentialsString(sysCredentials);
@@ -126,6 +131,7 @@ protected void updateExecutorInHTTP() {
126131
.build();
127132
apiClientBuilder.httpClient(httpClient);
128133
}
134+
129135
protected InsertRequest getBulkInsertRequestBody(com.skyflow.vault.data.InsertRequest request, VaultConfig config) throws SkyflowException {
130136
List<HashMap<String, Object>> values = request.getValues();
131137
List<InsertRecordData> insertRecordDataList = new ArrayList<>();
@@ -137,12 +143,12 @@ protected InsertRequest getBulkInsertRequestBody(com.skyflow.vault.data.InsertRe
137143
.vaultId(config.getVaultId())
138144
.records(insertRecordDataList)
139145
.tableName(request.getTable());
140-
if(request.getUpsert() != null && !request.getUpsert().isEmpty()){
146+
if (request.getUpsert() != null && !request.getUpsert().isEmpty()) {
141147
if (request.getUpsertType() != null) {
142148
EnumUpdateType updateType = null;
143-
if(request.getUpsertType() == UpdateType.REPLACE){
149+
if (request.getUpsertType() == UpdateType.REPLACE) {
144150
updateType = EnumUpdateType.REPLACE;
145-
} else if (request.getUpsertType() == UpdateType.REPLACE) {
151+
} else if (request.getUpsertType() == UpdateType.UPDATE) {
146152
updateType = EnumUpdateType.UPDATE;
147153
}
148154
Upsert upsert = Upsert.builder().uniqueColumns(request.getUpsert()).updateType(updateType).build();
@@ -162,7 +168,7 @@ protected com.skyflow.generated.rest.resources.recordservice.requests.Detokenize
162168
com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.builder()
163169
.vaultId(this.vaultConfig.getVaultId())
164170
.tokens(tokens);
165-
if (request.getTokenGroupRedactions() != null){
171+
if (request.getTokenGroupRedactions() != null) {
166172
List<com.skyflow.generated.rest.types.TokenGroupRedactions> tokenGroupRedactionsList = new ArrayList<>();
167173
for (com.skyflow.vault.data.TokenGroupRedactions tokenGroupRedactions : request.getTokenGroupRedactions()) {
168174
com.skyflow.generated.rest.types.TokenGroupRedactions redactions =

‎v3/src/main/java/com/skyflow/utils/Constants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
public final class Constants extends BaseConstants {
44
public static final String SDK_NAME = "Skyflow Java SDK ";
5-
public static final String SDK_VERSION = "3.0.0-beta.3";
5+
public static final String SDK_VERSION = "3.0.0-beta.4";
66
public static final String VAULT_DOMAIN = ".skyvault.";
77
public static final String SDK_PREFIX = SDK_NAME + SDK_VERSION;
88
public static final Integer INSERT_BATCH_SIZE = 50;

‎v3/src/main/java/com/skyflow/utils/Utils.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,23 @@
22

33
import com.google.gson.JsonObject;
44
import com.skyflow.enums.Env;
5+
import com.skyflow.errors.ErrorCode;
6+
import com.skyflow.errors.ErrorMessage;
7+
import com.skyflow.errors.SkyflowException;
58
import com.skyflow.generated.rest.core.ApiClientApiException;
69
import com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest;
710
import com.skyflow.generated.rest.types.InsertRecordData;
811
import com.skyflow.generated.rest.types.InsertResponse;
912
import com.skyflow.generated.rest.types.RecordResponseObject;
1013
import com.skyflow.generated.rest.types.TokenGroupRedactions;
14+
import com.skyflow.logs.ErrorLogs;
15+
import com.skyflow.utils.logger.LogUtil;
1116
import com.skyflow.vault.data.DetokenizeResponse;
1217
import com.skyflow.vault.data.ErrorRecord;
1318
import com.skyflow.vault.data.Success;
1419
import com.skyflow.vault.data.Token;
20+
import io.github.cdimascio.dotenv.Dotenv;
21+
import io.github.cdimascio.dotenv.DotenvException;
1522

1623
import java.util.ArrayList;
1724
import java.util.HashMap;
@@ -239,4 +246,24 @@ public static com.skyflow.vault.data.InsertResponse formatResponse(InsertRespons
239246
return formattedResponse;
240247
}
241248

249+
public static String getEnvVaultURL() throws SkyflowException {
250+
try {
251+
String vaultURL = System.getenv("VAULT_URL");
252+
if (vaultURL == null) {
253+
Dotenv dotenv = Dotenv.load();
254+
vaultURL = dotenv.get("VAULT_URL");
255+
}
256+
if (vaultURL != null && vaultURL.trim().isEmpty()) {
257+
LogUtil.printErrorLog(ErrorLogs.EMPTY_VAULT_URL.getLog());
258+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultUrl.getMessage());
259+
} else if (vaultURL != null && !vaultURL.startsWith(BaseConstants.SECURE_PROTOCOL)) {
260+
LogUtil.printErrorLog(ErrorLogs.INVALID_VAULT_URL_FORMAT.getLog());
261+
throw new SkyflowException( ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidVaultUrlFormat.getMessage());
262+
}
263+
return vaultURL;
264+
} catch (DotenvException e) {
265+
return null;
266+
}
267+
}
268+
242269
}

‎v3/src/main/java/com/skyflow/utils/validations/Validations.java‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.skyflow.utils.validations;
22

3+
import com.skyflow.config.Credentials;
4+
import com.skyflow.config.VaultConfig;
35
import com.skyflow.enums.InterfaceName;
46
import com.skyflow.errors.ErrorCode;
57
import com.skyflow.errors.ErrorMessage;
@@ -20,7 +22,6 @@ private Validations() {
2022
super();
2123
}
2224

23-
// add validations specific to v3 SDK
2425
public static void validateInsertRequest(InsertRequest insertRequest) throws SkyflowException {
2526
String table = insertRequest.getTable();
2627
ArrayList<HashMap<String, Object>> values = insertRequest.getValues();
@@ -46,12 +47,15 @@ public static void validateInsertRequest(InsertRequest insertRequest) throws Sky
4647
ErrorLogs.EMPTY_VALUES.getLog(), InterfaceName.INSERT.getName()
4748
));
4849
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyValues.getMessage());
49-
} else if (upsert != null && upsert.isEmpty()){
50+
} else if (values.size() > 10000) {
51+
LogUtil.printErrorLog(ErrorLogs.RECORD_SIZE_EXCEED.getLog());
52+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.RecordSizeExceedError.getMessage());
53+
} else if (upsert != null && upsert.isEmpty()) {
5054
LogUtil.printErrorLog(Utils.parameterizedString(
51-
ErrorLogs.EMPTY_UPSERT.getLog(), InterfaceName.INSERT.getName()
55+
ErrorLogs.EMPTY_UPSERT_VALUES.getLog(), InterfaceName.INSERT.getName()
5256
));
57+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyUpsertValues.getMessage());
5358
}
54-
// upsert
5559

5660
for (HashMap<String, Object> valuesMap : values) {
5761
for (String key : valuesMap.keySet()) {
@@ -82,6 +86,10 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S
8286
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.DetokenizeRequestNull.getMessage());
8387
}
8488
List<String> tokens = request.getTokens();
89+
if (tokens.size() > 10000) {
90+
LogUtil.printErrorLog(ErrorLogs.TOKENS_SIZE_EXCEED.getLog());
91+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.TokensSizeExceedError.getMessage());
92+
}
8593
if (tokens == null || tokens.isEmpty()) {
8694
LogUtil.printErrorLog(Utils.parameterizedString(
8795
ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
@@ -118,4 +126,26 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S
118126

119127
}
120128

129+
public static void validateVaultConfiguration(VaultConfig vaultConfig) throws SkyflowException {
130+
String vaultId = vaultConfig.getVaultId();
131+
String clusterId = vaultConfig.getClusterId();
132+
Credentials credentials = vaultConfig.getCredentials();
133+
if (vaultId == null) {
134+
LogUtil.printErrorLog(ErrorLogs.VAULT_ID_IS_REQUIRED.getLog());
135+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidVaultId.getMessage());
136+
} else if (vaultId.trim().isEmpty()) {
137+
LogUtil.printErrorLog(ErrorLogs.EMPTY_VAULT_ID.getLog());
138+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultId.getMessage());
139+
} else if (Utils.getEnvVaultURL() == null) {
140+
if (clusterId == null) {
141+
LogUtil.printErrorLog(ErrorLogs.CLUSTER_ID_IS_REQUIRED.getLog());
142+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidClusterId.getMessage());
143+
} else if (clusterId.trim().isEmpty()) {
144+
LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog());
145+
throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage());
146+
}
147+
} else if (credentials != null) {
148+
validateCredentials(credentials);
149+
}
150+
}
121151
}

0 commit comments

Comments
 (0)