Http sink: Empty String if the value does not exist in json body Template (#89)

* fix: add jsonpath Option while parsing template

* fix: * imports are failing the builds

* fix: code cleanup and test cases

* fix: code cleanup

* add: test cases to comprehensively test the new functionality

* fix: code cleaning

* add: documentation

* update: firehose version

---------

Co-authored-by: Mayank Rai <mayank.rai@gojek.com>
Co-authored-by: deepanshgoyal <deepansh.goyal@gojek.com>

Author: 3 people

build.gradle

@@ -33,7 +33,7 @@ lombok {
 }
 
 group 'com.gotocompany'
-version '0.12.16'
+version '0.12.17'
 
 def projName = "firehose"
 

docs/docs/sinks/http-sink.md

@@ -159,4 +159,26 @@ Currently supported typecasting target: DOUBLE, INTEGER, LONG, STRING
 
 - Example value: `[{"jsonPath": "$.root.someIntegerField", "type": "INTEGER"}, {"jsonPath": "$..[*].doubleField", "type": "DOUBLE"}]`
 - Type: `optional`
-- Default Value: `[]`
+- Default Value: `[]`
+
+### `SINK_HTTP_JSON_BODY_TEMPLATE_PARSE_OPTION`
+
+Defines the parsing options for the JSON body template. This configuration controls how JsonPath expressions behave when parsing the protobuf message to extract values for the template.
+**Available Options:**
+
+- **`DEFAULT_PATH_LEAF_TO_NULL`**: Returns `null` for missing leaf properties instead of throwing exceptions. Only affects missing final properties in a path (e.g., `$.user.name` on `{"user": {}}` returns `null`). Recommended for graceful handling of optional fields.
+
+- **`ALWAYS_RETURN_LIST`**: Forces all JsonPath queries to return results as a List, even for single values. For example, `$.name` returns `["John"]` instead of `"John"`. Useful when you need consistent List return types.
+
+- **`AS_PATH_LIST`**: Returns the JsonPath expressions pointing to matching nodes instead of their actual values. For example, returns `["$['users'][0]['name']"]` instead of `"John"`. Useful when you need to know data location rather than the data itself.
+
+- **`SUPPRESS_EXCEPTIONS`**: Suppresses ALL exceptions during JsonPath evaluation and returns `null` or empty collections for any errors (syntax errors, missing paths, type errors). Provides complete fault tolerance but less specific error handling.
+
+- **`REQUIRE_PROPERTIES`**: Throws exceptions if any property referenced in the path doesn't exist. Opposite of `DEFAULT_PATH_LEAF_TO_NULL` - enforces strict validation that all path components exist.
+
+**Key Differences:**
+- `SUPPRESS_EXCEPTIONS` vs `DEFAULT_PATH_LEAF_TO_NULL`: The former suppresses all types of exceptions broadly, while the latter only handles missing leaf properties specifically, allowing other exceptions (like syntax errors) to still be thrown for better debugging.
+
+- Example value: `DEFAULT_PATH_LEAF_TO_NULL`
+- Type: `optional`
+- Default Value: `""`

src/main/java/com/gotocompany/firehose/config/HttpSinkConfig.java

@@ -1,15 +1,17 @@
 package com.gotocompany.firehose.config;
 
+import com.gotocompany.firehose.config.converter.RangeToHashMapConverter;
+import com.gotocompany.firehose.config.converter.HttpSinkRequestMethodConverter;
+import com.gotocompany.firehose.config.converter.HttpSinkParameterSourceTypeConverter;
+import com.gotocompany.firehose.config.converter.HttpSinkParameterDataFormatConverter;
+import com.gotocompany.firehose.config.converter.HttpJsonBodyTemplateParseOptionConverter;
+import com.gotocompany.firehose.config.converter.HttpSinkParameterPlacementTypeConverter;
 import com.gotocompany.firehose.config.converter.HttpSinkSerializerJsonTypecastConfigConverter;
 import com.gotocompany.firehose.config.enums.HttpSinkDataFormatType;
 import com.gotocompany.firehose.config.enums.HttpSinkParameterPlacementType;
 import com.gotocompany.firehose.config.enums.HttpSinkParameterSourceType;
 import com.gotocompany.firehose.config.enums.HttpSinkRequestMethodType;
-import com.gotocompany.firehose.config.converter.HttpSinkRequestMethodConverter;
-import com.gotocompany.firehose.config.converter.HttpSinkParameterDataFormatConverter;
-import com.gotocompany.firehose.config.converter.HttpSinkParameterPlacementTypeConverter;
-import com.gotocompany.firehose.config.converter.HttpSinkParameterSourceTypeConverter;
-import com.gotocompany.firehose.config.converter.RangeToHashMapConverter;
+import com.jayway.jsonpath.Option;
 
 import java.util.Map;
 import java.util.function.Function;
@@ -80,6 +82,11 @@ public interface HttpSinkConfig extends AppConfig {
     @DefaultValue("")
     String getSinkHttpJsonBodyTemplate();
 
+    @Key("SINK_HTTP_JSON_BODY_TEMPLATE_PARSE_OPTION")
+    @DefaultValue("")
+    @ConverterClass(HttpJsonBodyTemplateParseOptionConverter.class)
+    Option getSinkHttpJsonBodyTemplateParseOption();
+
     @Key("SINK_HTTP_PARAMETER_PLACEMENT")
     @DefaultValue("header")
     @ConverterClass(HttpSinkParameterPlacementTypeConverter.class)

src/main/java/com/gotocompany/firehose/config/converter/HttpJsonBodyTemplateParseOptionConverter.java

@@ -0,0 +1,35 @@
+package com.gotocompany.firehose.config.converter;
+
+import com.jayway.jsonpath.Option;
+import org.aeonbits.owner.Converter;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+public class HttpJsonBodyTemplateParseOptionConverter implements Converter<Option> {
+    @Override
+    public Option convert(Method method, String input) {
+        if (isNullOrBlank(input)) {
+            return null;
+        }
+        String normalizedInput = input.trim().toUpperCase();
+        try {
+            return Option.valueOf(normalizedInput);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException(
+                    String.format("Invalid JSONPath option: '%s'. Valid options are: %s",
+                            input, getValidOptionsString()), e);
+        }
+    }
+
+    private boolean isNullOrBlank(String input) {
+        return input == null || input.trim().isEmpty();
+    }
+
+    private String getValidOptionsString() {
+        return Arrays.stream(Option.values())
+                .map(Enum::name)
+                .collect(Collectors.joining(", "));
+    }
+}

src/main/java/com/gotocompany/firehose/serializer/MessageToTemplatizedJson.java

@@ -9,7 +9,9 @@
 import com.google.protobuf.DynamicMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
+import com.jayway.jsonpath.Configuration;
 import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.Option;
 import com.jayway.jsonpath.PathNotFoundException;
 import com.gotocompany.stencil.Parser;
 import org.json.simple.parser.JSONParser;
@@ -32,24 +34,26 @@ public class MessageToTemplatizedJson implements MessageSerializer {
     private Parser protoParser;
     private HashSet<String> pathsToReplace;
     private JSONParser jsonParser;
+    private Configuration jsonPathConfig;
     private FirehoseInstrumentation firehoseInstrumentation;
 
-    public static MessageToTemplatizedJson create(FirehoseInstrumentation firehoseInstrumentation, String httpSinkJsonBodyTemplate, Parser protoParser) {
-        MessageToTemplatizedJson messageToTemplatizedJson = new MessageToTemplatizedJson(firehoseInstrumentation, httpSinkJsonBodyTemplate, protoParser);
+    public static MessageToTemplatizedJson create(FirehoseInstrumentation firehoseInstrumentation, String httpSinkJsonBodyTemplate, Parser protoParser, Option option) {
+        MessageToTemplatizedJson messageToTemplatizedJson = new MessageToTemplatizedJson(firehoseInstrumentation, httpSinkJsonBodyTemplate, protoParser, option);
         if (messageToTemplatizedJson.isInvalidJson()) {
-            throw new ConfigurationException("Given HTTPSink JSON body template :"
-                    + httpSinkJsonBodyTemplate
-                    + ", must be a valid JSON.");
+            throw new ConfigurationException("Given HTTPSink JSON body template: " + httpSinkJsonBodyTemplate + " must be a valid JSON.");
         }
         messageToTemplatizedJson.setPathsFromTemplate();
         return messageToTemplatizedJson;
     }
 
-    public MessageToTemplatizedJson(FirehoseInstrumentation firehoseInstrumentation, String httpSinkJsonBodyTemplate, Parser protoParser) {
+    public MessageToTemplatizedJson(FirehoseInstrumentation firehoseInstrumentation, String httpSinkJsonBodyTemplate, Parser protoParser, Option option) {
         this.httpSinkJsonBodyTemplate = httpSinkJsonBodyTemplate;
         this.protoParser = protoParser;
         this.jsonParser = new JSONParser();
         this.gson = new Gson();
+        this.jsonPathConfig = option == null
+                ? Configuration.defaultConfiguration()
+                : Configuration.defaultConfiguration().addOptions(option);
         this.firehoseInstrumentation = firehoseInstrumentation;
     }
 
@@ -81,15 +85,23 @@ public String serialize(Message message) throws DeserializerException {
             DynamicMessage msg = protoParser.parse(message.getLogMessage());
             jsonMessage = JsonFormat.printer().includingDefaultValueFields().preservingProtoFieldNames().print(msg);
             String finalMessage = httpSinkJsonBodyTemplate;
+
             for (String path : pathsToReplace) {
                 if (path.equals(ALL_FIELDS_FROM_TEMPLATE)) {
                     jsonString = jsonMessage;
                 } else {
-                    Object element = JsonPath.read(jsonMessage, path.replaceAll("\"", ""));
-                    jsonString = gson.toJson(element);
+                    Object element = JsonPath.using(jsonPathConfig).parse(jsonMessage).read(path.replaceAll("\"", ""));
+                    if (element == null && (jsonPathConfig.getOptions().contains(Option.DEFAULT_PATH_LEAF_TO_NULL)
+                            || jsonPathConfig.getOptions().contains(Option.SUPPRESS_EXCEPTIONS))) {
+                        firehoseInstrumentation.logWarn("Missing value for path: {}", path);
+                        jsonString = "";
+                    } else {
+                        jsonString = gson.toJson(element);
+                    }
                 }
                 finalMessage = finalMessage.replace(path, jsonString);
             }
+
             return finalMessage;
         } catch (InvalidProtocolBufferException | PathNotFoundException e) {
             throw new DeserializerException(e.getMessage());

src/main/java/com/gotocompany/firehose/sink/http/factory/SerializerFactory.java

@@ -39,7 +39,7 @@ public MessageSerializer build() {
             } else {
                 firehoseInstrumentation.logDebug("Serializer type: EsbMessageToTemplatizedJson");
                 return getTypecastedJsonSerializer(
-                        MessageToTemplatizedJson.create(new FirehoseInstrumentation(statsDReporter, MessageToTemplatizedJson.class), httpSinkConfig.getSinkHttpJsonBodyTemplate(), protoParser));
+                        MessageToTemplatizedJson.create(new FirehoseInstrumentation(statsDReporter, MessageToTemplatizedJson.class), httpSinkConfig.getSinkHttpJsonBodyTemplate(), protoParser, httpSinkConfig.getSinkHttpJsonBodyTemplateParseOption()));
             }
         }