Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Smart Substitution for JSON fields in Mongo Form #5136

Merged
merged 5 commits into from
Jun 16, 2021
Merged

[Feature] Smart Substitution for JSON fields in Mongo Form #5136

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e18f15b
Reduced verbosity of the command names for Insert, Find and Delete
trishaanand Jun 11, 2021
6038927
WIP : Smart substitution untested changes for mongo form
trishaanand Jun 15, 2021
0f2be8d
Tested code for smart substitution working for mongo form fields
trishaanand Jun 15, 2021
12d2c15
Added test case for assert for smart substitution
trishaanand Jun 15, 2021
7591128
Merge branch 'release' into feature/ss-mongo-form
trishaanand Jun 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 106 additions & 37 deletions app/server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/MongoPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.appsmith.external.models.DatasourceStructure;
import com.appsmith.external.models.DatasourceTestResult;
import com.appsmith.external.models.Endpoint;
import com.appsmith.external.models.Param;
import com.appsmith.external.models.ParsedDataType;
import com.appsmith.external.models.Property;
import com.appsmith.external.models.RequestParamDTO;
Expand Down Expand Up @@ -61,6 +62,7 @@
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
Expand All @@ -76,8 +78,24 @@
import java.util.stream.Collectors;

import static com.appsmith.external.constants.ActionConstants.ACTION_CONFIGURATION_BODY;
import static com.external.plugins.MongoPluginUtils.validConfigurationPresent;
import static com.external.plugins.constants.ConfigurationIndex.AGGREGATE_PIPELINE;
import static com.external.plugins.constants.ConfigurationIndex.COMMAND;
import static com.external.plugins.constants.ConfigurationIndex.COUNT_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.DELETE_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.DISTINCT_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.FIND_PROJECTION;
import static com.external.plugins.constants.ConfigurationIndex.FIND_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.FIND_SORT;
import static com.external.plugins.constants.ConfigurationIndex.INPUT_TYPE;
import static com.external.plugins.constants.ConfigurationIndex.INSERT_DOCUMENT;
import static com.external.plugins.constants.ConfigurationIndex.SMART_BSON_SUBSTITUTION;
import static com.external.plugins.constants.ConfigurationIndex.UPDATE_MANY_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.UPDATE_MANY_UPDATE;
import static com.external.plugins.constants.ConfigurationIndex.UPDATE_ONE_QUERY;
import static com.external.plugins.constants.ConfigurationIndex.UPDATE_ONE_SORT;
import static com.external.plugins.constants.ConfigurationIndex.UPDATE_ONE_UPDATE;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;

public class MongoPlugin extends BasePlugin {
Expand All @@ -102,8 +120,6 @@ public class MongoPlugin extends BasePlugin {

private static final int TEST_DATASOURCE_TIMEOUT_SECONDS = 15;

private static final int SMART_BSON_SUBSTITUTION_INDEX = 0;

/*
* - The regex matches the following two pattern types:
* - mongodb+srv://user:pass@some-url/some-db....
Expand Down Expand Up @@ -140,6 +156,21 @@ public class MongoPlugin extends BasePlugin {

private static final Integer MONGO_COMMAND_EXCEPTION_UNAUTHORIZED_ERROR_CODE = 13;

private static final Set<Integer> bsonFields = new HashSet<>(Arrays.asList(AGGREGATE_PIPELINE,
COUNT_QUERY,
DELETE_QUERY,
DISTINCT_QUERY,
FIND_QUERY,
FIND_SORT,
FIND_PROJECTION,
INSERT_DOCUMENT,
UPDATE_MANY_QUERY,
UPDATE_MANY_UPDATE,
UPDATE_ONE_QUERY,
UPDATE_ONE_SORT,
UPDATE_ONE_UPDATE
));

public MongoPlugin(PluginWrapper wrapper) {
super(wrapper);
}
Expand Down Expand Up @@ -181,8 +212,8 @@ public Mono<ActionExecutionResult> executeParameterized(MongoClient mongoClient,
smartBsonSubstitution = false;

// Since properties is not empty, we are guaranteed to find the first property.
} else if (properties.get(SMART_BSON_SUBSTITUTION_INDEX) != null) {
Object ssubValue = properties.get(SMART_BSON_SUBSTITUTION_INDEX).getValue();
} else if (properties.get(SMART_BSON_SUBSTITUTION) != null) {
Object ssubValue = properties.get(SMART_BSON_SUBSTITUTION).getValue();
if (ssubValue instanceof Boolean) {
smartBsonSubstitution = (Boolean) ssubValue;
} else if (ssubValue instanceof String) {
Expand All @@ -196,32 +227,31 @@ public Mono<ActionExecutionResult> executeParameterized(MongoClient mongoClient,

// Smartly substitute in actionConfiguration.body and replace all the bindings with values.
if (TRUE.equals(smartBsonSubstitution)) {
// Do smart replacements in BSON body
if (actionConfiguration.getBody() != null) {

// First extract all the bindings in order
List<String> mustacheKeysInOrder = MustacheHelper.extractMustacheKeysInOrder(actionConfiguration.getBody());
// Replace all the bindings with a ? as expected in a prepared statement.
String updatedBody = MustacheHelper.replaceMustacheWithQuestionMark(actionConfiguration.getBody(), mustacheKeysInOrder);

try {
updatedBody = (String) smartSubstitutionOfBindings(updatedBody,
mustacheKeysInOrder,
executeActionDTO.getParams(),
parameters);
} catch (AppsmithPluginException e) {
ActionExecutionResult errorResult = new ActionExecutionResult();
errorResult.setStatusCode(AppsmithPluginError.PLUGIN_ERROR.getAppErrorCode().toString());
errorResult.setIsExecutionSuccess(false);
errorResult.setBody(e.getMessage());
return Mono.just(errorResult);
}

actionConfiguration.setBody(updatedBody);
if (isFormInput(actionConfiguration.getPluginSpecifiedTemplates())) {
List<Property> updatedTemplates = smartSubstituteFormCommand(actionConfiguration.getPluginSpecifiedTemplates(),
executeActionDTO.getParams(), parameters);
actionConfiguration.setPluginSpecifiedTemplates(updatedTemplates);
} else {
// For raw queries do smart replacements in BSON body
if (actionConfiguration.getBody() != null) {
try {
String updatedRawQuery = smartSubstituteBSON(actionConfiguration.getBody(),
executeActionDTO.getParams(), parameters);
actionConfiguration.setBody(updatedRawQuery);
} catch (AppsmithPluginException e) {
ActionExecutionResult errorResult = new ActionExecutionResult();
errorResult.setStatusCode(AppsmithPluginError.PLUGIN_ERROR.getAppErrorCode().toString());
errorResult.setIsExecutionSuccess(false);
errorResult.setBody(e.getMessage());
return Mono.just(errorResult);
}
}
}
}

prepareConfigurationsForExecution(executeActionDTO, actionConfiguration, datasourceConfiguration);

// In case the input type is form instead of raw, parse the same into BSON command
String parsedRawCommand = convertMongoFormInputToRawCommand(actionConfiguration);
if (parsedRawCommand != null) {
Expand Down Expand Up @@ -258,7 +288,7 @@ public Mono<ActionExecutionResult> executeCommon(MongoClient mongoClient,

Mono<Document> mongoOutputMono = Mono.from(database.runCommand(command));
ActionExecutionResult result = new ActionExecutionResult();
List<RequestParamDTO> requestParams = List.of(new RequestParamDTO(ACTION_CONFIGURATION_BODY, query, null
List<RequestParamDTO> requestParams = List.of(new RequestParamDTO(ACTION_CONFIGURATION_BODY, query, null
, null, null));

return mongoOutputMono
Expand Down Expand Up @@ -382,15 +412,22 @@ public Mono<ActionExecutionResult> executeCommon(MongoClient mongoClient,
.subscribeOn(scheduler);
}

private Boolean isFormInput(List<Property> templates) {
if ((templates.size() >= (1 + INPUT_TYPE)) &&
(templates.get(INPUT_TYPE) != null) &&
("FORM".equals(templates.get(INPUT_TYPE).getValue())) &&
(templates.size() >= (1 + COMMAND)) &&
(templates.get(COMMAND) != null) &&
(templates.get(COMMAND).getValue() != null)) {
return TRUE;
}
return FALSE;
}

private String convertMongoFormInputToRawCommand(ActionConfiguration actionConfiguration) {
List<Property> templates = actionConfiguration.getPluginSpecifiedTemplates();
if (templates != null) {
if ((templates.size() >= (1 + INPUT_TYPE)) &&
(templates.get(INPUT_TYPE) != null) &&
("FORM".equals(templates.get(INPUT_TYPE).getValue())) &&
(templates.size() >= (1 + COMMAND)) &&
(templates.get(COMMAND) != null) &&
(templates.get(COMMAND).getValue() != null)) {
if (isFormInput(templates)) {
// The user has configured FORM for command input. Parse the commands appropriately

MongoCommand command = null;
Expand Down Expand Up @@ -435,6 +472,38 @@ private String convertMongoFormInputToRawCommand(ActionConfiguration actionConfi
return actionConfiguration.getBody();
}

private String smartSubstituteBSON(String rawQuery,
List<Param> params,
List<Map.Entry<String, String>> parameters) throws AppsmithPluginException {

// First extract all the bindings in order
List<String> mustacheKeysInOrder = MustacheHelper.extractMustacheKeysInOrder(rawQuery);
// Replace all the bindings with a ? as expected in a prepared statement.
String updatedQuery = MustacheHelper.replaceMustacheWithQuestionMark(rawQuery, mustacheKeysInOrder);

updatedQuery = (String) smartSubstitutionOfBindings(updatedQuery,
mustacheKeysInOrder,
params,
parameters);

return updatedQuery;
}

private List<Property> smartSubstituteFormCommand(List<Property> templates,
List<Param> params,
List<Map.Entry<String, String>> parameters) throws AppsmithPluginException {

for (int i = 0; i < templates.size(); i++) {
if (validConfigurationPresent(templates, i) && bsonFields.contains(i)) {
Property configuration = templates.get(i);
// Do Smart Substitution for each BSON field
configuration.setValue(smartSubstituteBSON((String) configuration.getValue(), params, parameters));
}
}

return templates;
}

private String getDatabaseName(DatasourceConfiguration datasourceConfiguration) {
// Explicitly set default database.
String databaseName = datasourceConfiguration.getConnection().getDefaultDatabaseName();
Expand Down Expand Up @@ -706,7 +775,7 @@ public Set<String> validateDatasource(DatasourceConfiguration datasourceConfigur
} else if (!isAuthenticated(authentication, mongoUri)) {
String mongoUriWithHiddenPassword = buildURIfromExtractedInfo(extractedInfo, "****");
properties.get(DATASOURCE_CONFIG_MONGO_URI_PROPERTY_INDEX).setValue(mongoUriWithHiddenPassword);
authentication = (authentication == null) ? new DBAuth(): authentication;
authentication = (authentication == null) ? new DBAuth() : authentication;
authentication.setUsername((String) extractedInfo.get(KEY_USERNAME));
authentication.setPassword((String) extractedInfo.get(KEY_PASSWORD));
authentication.setDatabaseName((String) extractedInfo.get(KEY_URI_DBNAME));
Expand Down Expand Up @@ -746,7 +815,7 @@ public Set<String> validateDatasource(DatasourceConfiguration datasourceConfigur
" directly.");
}
}

if (authentication != null) {
DBAuth.Type authType = authentication.getAuthType();

Expand Down Expand Up @@ -1012,11 +1081,11 @@ private static Object cleanUp(Object object) {

return object;
}

private static boolean isAuthenticated(DBAuth authentication, String mongoUri) {
if (authentication != null && authentication.getUsername() != null
&& authentication.getPassword() != null && mongoUri.contains("****")) {
&& authentication.getPassword() != null && mongoUri.contains("****")) {

return true;
}
return false;
Expand Down
4 changes: 2 additions & 2 deletions ...rver/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/commands/Delete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import static com.external.plugins.MongoPluginUtils.generateMongoFormConfigTemplates;
import static com.external.plugins.MongoPluginUtils.parseSafely;
import static com.external.plugins.MongoPluginUtils.validConfigurationPresent;
import static com.external.plugins.constants.ConfigurationIndex.BSON;
import static com.external.plugins.constants.ConfigurationIndex.SMART_BSON_SUBSTITUTION;
import static com.external.plugins.constants.ConfigurationIndex.COLLECTION;
import static com.external.plugins.constants.ConfigurationIndex.COMMAND;
import static com.external.plugins.constants.ConfigurationIndex.DELETE_LIMIT;
Expand Down Expand Up @@ -88,7 +88,7 @@ public List<DatasourceStructure.Template> generateTemplate(Map<String, Object> t

Map<Integer, Object> configMap = new HashMap<>();

configMap.put(BSON, Boolean.FALSE);
configMap.put(SMART_BSON_SUBSTITUTION, Boolean.TRUE);
configMap.put(INPUT_TYPE, "FORM");
configMap.put(COMMAND, "DELETE");
configMap.put(COLLECTION, collectionName);
Expand Down
6 changes: 3 additions & 3 deletions ...server/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/commands/Find.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import static com.external.plugins.MongoPluginUtils.generateMongoFormConfigTemplates;
import static com.external.plugins.MongoPluginUtils.parseSafely;
import static com.external.plugins.MongoPluginUtils.validConfigurationPresent;
import static com.external.plugins.constants.ConfigurationIndex.BSON;
import static com.external.plugins.constants.ConfigurationIndex.SMART_BSON_SUBSTITUTION;
import static com.external.plugins.constants.ConfigurationIndex.COLLECTION;
import static com.external.plugins.constants.ConfigurationIndex.COMMAND;
import static com.external.plugins.constants.ConfigurationIndex.FIND_LIMIT;
Expand Down Expand Up @@ -125,7 +125,7 @@ public List<DatasourceStructure.Template> generateTemplate(Map<String, Object> t
private DatasourceStructure.Template generateFindTemplate(String collectionName, String filterFieldName, String filterFieldValue) {
Map<Integer, Object> configMap = new HashMap<>();

configMap.put(BSON, Boolean.FALSE);
configMap.put(SMART_BSON_SUBSTITUTION, Boolean.TRUE);
configMap.put(INPUT_TYPE, "FORM");
configMap.put(COMMAND, "FIND");
configMap.put(COLLECTION, collectionName);
Expand Down Expand Up @@ -162,7 +162,7 @@ private DatasourceStructure.Template generateFindTemplate(String collectionName,
private DatasourceStructure.Template generateFindByIdTemplate(String collectionName) {
Map<Integer, Object> configMap = new HashMap<>();

configMap.put(BSON, Boolean.FALSE);
configMap.put(SMART_BSON_SUBSTITUTION, Boolean.TRUE);
configMap.put(INPUT_TYPE, "FORM");
configMap.put(COMMAND, "FIND");
configMap.put(FIND_QUERY, "{\"_id\": ObjectId(\"id_to_query_with\")}");
Expand Down
4 changes: 2 additions & 2 deletions ...rver/appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/commands/Insert.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import static com.external.plugins.MongoPluginUtils.generateMongoFormConfigTemplates;
import static com.external.plugins.MongoPluginUtils.parseSafely;
import static com.external.plugins.MongoPluginUtils.validConfigurationPresent;
import static com.external.plugins.constants.ConfigurationIndex.BSON;
import static com.external.plugins.constants.ConfigurationIndex.SMART_BSON_SUBSTITUTION;
import static com.external.plugins.constants.ConfigurationIndex.COLLECTION;
import static com.external.plugins.constants.ConfigurationIndex.COMMAND;
import static com.external.plugins.constants.ConfigurationIndex.INPUT_TYPE;
Expand Down Expand Up @@ -100,7 +100,7 @@ public List<DatasourceStructure.Template> generateTemplate(Map<String, Object> t

Map<Integer, Object> configMap = new HashMap<>();

configMap.put(BSON, Boolean.FALSE);
configMap.put(SMART_BSON_SUBSTITUTION, Boolean.TRUE);
configMap.put(INPUT_TYPE, "FORM");
configMap.put(COMMAND, "INSERT");
configMap.put(INSERT_DOCUMENT, "[{" + sampleInsertDocuments + "}]");
Expand Down
4 changes: 2 additions & 2 deletions .../appsmith-plugins/mongoPlugin/src/main/java/com/external/plugins/commands/UpdateMany.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import static com.external.plugins.MongoPluginUtils.generateMongoFormConfigTemplates;
import static com.external.plugins.MongoPluginUtils.parseSafely;
import static com.external.plugins.MongoPluginUtils.validConfigurationPresent;
import static com.external.plugins.constants.ConfigurationIndex.BSON;
import static com.external.plugins.constants.ConfigurationIndex.SMART_BSON_SUBSTITUTION;
import static com.external.plugins.constants.ConfigurationIndex.COLLECTION;
import static com.external.plugins.constants.ConfigurationIndex.COMMAND;
import static com.external.plugins.constants.ConfigurationIndex.INPUT_TYPE;
Expand Down Expand Up @@ -95,7 +95,7 @@ public List<DatasourceStructure.Template> generateTemplate(Map<String, Object> t

Map<Integer, Object> configMap = new HashMap<>();

configMap.put(BSON, Boolean.FALSE);
configMap.put(SMART_BSON_SUBSTITUTION, Boolean.TRUE);
configMap.put(INPUT_TYPE, "FORM");
configMap.put(COMMAND, "UPDATE_MANY");
configMap.put(COLLECTION, collectionName);
Expand Down
2 changes: 1 addition & 1 deletion ...-plugins/mongoPlugin/src/main/java/com/external/plugins/constants/ConfigurationIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.external.plugins.constants;

public class ConfigurationIndex {
public static final int BSON = 0;
public static final int SMART_BSON_SUBSTITUTION = 0;
public static final int INPUT_TYPE = 1;
public static final int COMMAND = 2;
public static final int COLLECTION = 19;
Expand Down
6 changes: 3 additions & 3 deletions app/server/appsmith-plugins/mongoPlugin/src/main/resources/editor.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
"initialValue": "FIND",
"options": [
{
"label": "Insert one or more documents",
"label": "Insert Document(s)",
"value": "INSERT"
},
{
"label": "Find one or more documents",
"label": "Find Document(s)",
"value": "FIND"
},
{
Expand All @@ -43,7 +43,7 @@
"value": "UPDATE_MANY"
},
{
"label": "Delete one or more documents",
"label": "Delete Document(s)",
"value": "DELETE"
},
{
Expand Down
Loading