RD-12747 - secret scrubbing (#66)

Author: harelmo-lumigo

.gitignore

@@ -221,3 +221,4 @@ _meta
 .serverless
 package-lock.json
 src/main/resources/lumigo-agent.jar
+.vscode/

pom.xml

@@ -1,25 +1,19 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <groupId>io.lumigo</groupId>
     <artifactId>java-tracer</artifactId>
     <version>1.0.42</version>
     <packaging>jar</packaging>
-
     <name>Lumigo java tracer</name>
     <description>The Lumigo java tracer for serverless functions</description>
     <url>https://lumigo.io/</url>
-
     <scm>
         <url>https://github.com/lumigo-io/java-tracer</url>
         <connection>scm:git:https://github.com:lumigo-io/java-tracer.git</connection>
         <developerConnection>scm:git:https://github.com:lumigo-io/java-tracer.git</developerConnection>
         <tag>1.0.42</tag>
     </scm>
-
     <developers>
         <developer>
             <name>Lumigo Dev Team</name>
@@ -28,21 +22,17 @@
             <organizationUrl>https://lumigo.io/</organizationUrl>
         </developer>
     </developers>
-
     <organization>
         <name>Lumigo</name>
         <url>https://lumigo.io/</url>
     </organization>
-
     <licenses>
         <license>
             <name>Apache License, Version 2.0</name>
             <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
             <distribution>repo</distribution>
         </license>
     </licenses>
-
-
     <distributionManagement>
         <snapshotRepository>
             <id>ossrh</id>
@@ -53,8 +43,6 @@
             <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
         </repository>
     </distributionManagement>
-
-
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
@@ -159,7 +147,7 @@
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
-            <version>1.18.22</version>
+            <version>1.18.32</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
@@ -182,7 +170,11 @@
             <artifactId>byte-buddy-agent</artifactId>
             <version>1.14.14</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+            <version>20210307</version>
+        </dependency>
         <!-- Testing dependencies -->
         <dependency>
             <groupId>org.junit.jupiter</groupId>
@@ -233,7 +225,7 @@
                 <includes>
                     <include>**/lumigo-version.txt</include>
                 </includes>
-        </resource>
+            </resource>
         </resources>
         <plugins>
             <!-- Used to make it so the JavaDoc and sources are included. -->
@@ -267,8 +259,7 @@
                 </executions>
             </plugin>
             <!-- End of sources/javadoc inclusion. -->
-
-            <!-- Use for running tests-->
+            <!-- Use for running tests -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -297,12 +288,11 @@
                 <version>3.0.5</version>
                 <configuration>
                     <!--
-                        Enables analysis which takes more memory but finds more bugs.
-                        If you run out of memory, changes the value of the effort element
-                        to 'Low'.
-                    -->
+                         Enables analysis which takes more memory but finds more bugs.
+                         If you run out of memory, changes the value of the effort element
+                         to 'Low'.
+                         -->
                     <effort>Max</effort>
-
                     <!-- Reports all bugs (other values are medium and max) -->
                     <threshold>Low</threshold>
                     <!-- Produces XML report -->
@@ -312,9 +302,7 @@
                     <excludeFilterFile>findbugs/findbugs-exclude.xml</excludeFilterFile>
                 </configuration>
                 <executions>
-                    <!--
-                        Ensures that FindBugs inspects source code when project is compiled.
-                    -->
+                    <!-- Ensures that FindBugs inspects source code when project is compiled. -->
                     <execution>
                         <id>analyze-compile</id>
                         <phase>compile</phase>
@@ -327,7 +315,7 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.3</version>
+                <version>0.8.11</version>
                 <executions>
                     <execution>
                         <goals>
@@ -377,4 +365,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>

scripts/checks.sh

@@ -2,5 +2,5 @@
 set -eo pipefail
 
 java -jar libs/google-java-format-1.7-all-deps.jar --set-exit-if-changed -i -a $(find . -type f -name "*.java" | grep ".*/src/.*java")
-mvn -f agent/pom.xml clean package
-mvn clean package
+mvn -Djava.security.manager=allow -f agent/pom.xml clean package
+mvn -Djava.security.manager=allow clean package

src/main/java/io/lumigo/core/SpansContainer.java

@@ -9,9 +9,12 @@
 import io.lumigo.core.parsers.v1.AwsSdkV1ParserFactory;
 import io.lumigo.core.parsers.v2.AwsSdkV2ParserFactory;
 import io.lumigo.core.utils.AwsUtils;
+import io.lumigo.core.utils.EnvUtil;
 import io.lumigo.core.utils.JsonUtils;
+import io.lumigo.core.utils.SecretScrubber;
 import io.lumigo.core.utils.StringUtils;
 import io.lumigo.models.HttpSpan;
+import io.lumigo.models.Reportable;
 import io.lumigo.models.Span;
 import java.io.*;
 import java.util.*;
@@ -38,14 +41,14 @@ public class SpansContainer {
     private static final String AMZN_TRACE_ID = "_X_AMZN_TRACE_ID";
     private static final String FUNCTION_SPAN_TYPE = "function";
     private static final String HTTP_SPAN_TYPE = "http";
+    private static final SecretScrubber secretScrubber = new SecretScrubber(new EnvUtil().getEnv());
 
     private Span baseSpan;
     private Span startFunctionSpan;
     private Long rttDuration;
     private Span endFunctionSpan;
     private Reporter reporter;
     private List<HttpSpan> httpSpans = new LinkedList<>();
-
     private static final SpansContainer ourInstance = new SpansContainer();
 
     public static SpansContainer getInstance() {
@@ -68,6 +71,7 @@ private SpansContainer() {}
     public void init(Map<String, String> env, Reporter reporter, Context context, Object event) {
         this.clear();
         this.reporter = reporter;
+
         int javaVersion = AwsUtils.parseJavaVersion(System.getProperty("java.version"));
         if (javaVersion > 11) {
             awsTracerId = System.getProperty("com.amazonaws.xray.traceHeader");
@@ -214,8 +218,8 @@ public Span getStartFunctionSpan() {
         return startFunctionSpan;
     }
 
-    public List<Object> getAllCollectedSpans() {
-        List<Object> spans = new LinkedList<>();
+    public List<Reportable> getAllCollectedSpans() {
+        List<Reportable> spans = new LinkedList<>();
         spans.add(endFunctionSpan);
         spans.addAll(httpSpans);
         return spans;
@@ -518,63 +522,22 @@ protected static <T> T callIfVerbose(Callable<T> method) {
         }
     }
 
-    private Object prepareToSend(Object span, boolean hasError) {
-        return reduceSpanSize(span, hasError);
+    private Reportable prepareToSend(Reportable span, boolean hasError) {
+        return reduceSpanSize(span.scrub(secretScrubber), hasError);
     }
 
-    private List<Object> prepareToSend(List<Object> spans, boolean hasError) {
-        for (Object span : spans) {
-            reduceSpanSize(span, hasError);
+    private List<Reportable> prepareToSend(List<Reportable> spans, boolean hasError) {
+        for (Reportable span : spans) {
+            reduceSpanSize(span.scrub(secretScrubber), hasError);
         }
         return spans;
     }
 
-    public Object reduceSpanSize(Object span, boolean hasError) {
+    public Reportable reduceSpanSize(Reportable span, boolean hasError) {
         int maxFieldSize =
                 hasError
                         ? Configuration.getInstance().maxSpanFieldSizeWhenError()
                         : Configuration.getInstance().maxSpanFieldSize();
-        if (span instanceof Span) {
-            Span functionSpan = (Span) span;
-            functionSpan.setEnvs(
-                    StringUtils.getMaxSizeString(
-                            functionSpan.getEnvs(),
-                            Configuration.getInstance().maxSpanFieldSize()));
-            functionSpan.setReturn_value(
-                    StringUtils.getMaxSizeString(functionSpan.getReturn_value(), maxFieldSize));
-            functionSpan.setEvent(
-                    StringUtils.getMaxSizeString(functionSpan.getEvent(), maxFieldSize));
-        } else if (span instanceof HttpSpan) {
-            HttpSpan httpSpan = (HttpSpan) span;
-            httpSpan.getInfo()
-                    .getHttpInfo()
-                    .getRequest()
-                    .setHeaders(
-                            StringUtils.getMaxSizeString(
-                                    httpSpan.getInfo().getHttpInfo().getRequest().getHeaders(),
-                                    maxFieldSize));
-            httpSpan.getInfo()
-                    .getHttpInfo()
-                    .getRequest()
-                    .setBody(
-                            StringUtils.getMaxSizeString(
-                                    httpSpan.getInfo().getHttpInfo().getRequest().getBody(),
-                                    maxFieldSize));
-            httpSpan.getInfo()
-                    .getHttpInfo()
-                    .getResponse()
-                    .setHeaders(
-                            StringUtils.getMaxSizeString(
-                                    httpSpan.getInfo().getHttpInfo().getResponse().getHeaders(),
-                                    maxFieldSize));
-            httpSpan.getInfo()
-                    .getHttpInfo()
-                    .getResponse()
-                    .setBody(
-                            StringUtils.getMaxSizeString(
-                                    httpSpan.getInfo().getHttpInfo().getResponse().getBody(),
-                                    maxFieldSize));
-        }
-        return span;
+        return span.reduceSize(maxFieldSize);
     }
 }

src/main/java/io/lumigo/core/configuration/Configuration.java

@@ -27,6 +27,7 @@ public class Configuration {
     public static final String LUMIGO_MAX_RESPONSE_SIZE = "LUMIGO_MAX_RESPONSE_SIZE";
     public static final String LUMIGO_MAX_SIZE_FOR_REQUEST = "LUMIGO_MAX_SIZE_FOR_REQUEST";
     public static final String LUMIGO_INSTRUMENTATION = "LUMIGO_INSTRUMENTATION";
+    public static final String LUMIGO_SECRET_MASKING_REGEX = "LUMIGO_SECRET_MASKING_REGEX";
 
     private static Configuration instance;
     private LumigoConfiguration inlineConf;

src/main/java/io/lumigo/core/network/Reporter.java

@@ -3,6 +3,7 @@
 import io.lumigo.core.configuration.Configuration;
 import io.lumigo.core.utils.JsonUtils;
 import io.lumigo.core.utils.StringUtils;
+import io.lumigo.models.Reportable;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -21,11 +22,11 @@ public Reporter() {
                         .build();
     }
 
-    public long reportSpans(Object span, int maxSize) throws IOException {
+    public long reportSpans(Reportable span, int maxSize) throws IOException {
         return reportSpans(Collections.singletonList(span), maxSize);
     }
 
-    public long reportSpans(List<Object> spans, int maxSize) throws IOException {
+    public long reportSpans(List<Reportable> spans, int maxSize) throws IOException {
         long time = System.currentTimeMillis();
         List<String> spansAsStringList = new LinkedList<>();
         int sizeCount = 0;

src/main/java/io/lumigo/core/utils/SecretScrubber.java

@@ -0,0 +1,66 @@
+package io.lumigo.core.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+public class SecretScrubber {
+    private List<Pattern> scrubbingPatterns;
+
+    private static final String SECRET_PLACEHOLDER = "****";
+
+    public SecretScrubber(Map<String, String> env) {
+        this.scrubbingPatterns = new SecretScrubbingPatternProvider(env).getScrubbingPatterns();
+    }
+
+    public String scrubStringifiedObject(String stringifiedObject) {
+        try {
+            JSONObject jsonObject = new JSONObject(stringifiedObject);
+            return scrubJsonObject(jsonObject, this.scrubbingPatterns).toString();
+        } catch (Exception e) {
+            return stringifiedObject;
+        }
+    }
+
+    private JSONObject scrubJsonObject(JSONObject jsonObject, List<Pattern> patterns) {
+        for (String key : jsonObject.keySet()) {
+            Object value = jsonObject.get(key);
+
+            if (value instanceof String && isSecret(key, patterns)) {
+                jsonObject.put(key, SECRET_PLACEHOLDER);
+            } else if (value instanceof JSONArray) {
+                ArrayList<Object> scrubbedArray = new ArrayList<>();
+
+                for (Object item : (JSONArray) value) {
+                    if (item instanceof String && isSecret(key, patterns)) {
+                        scrubbedArray.add(SECRET_PLACEHOLDER);
+                    } else if (item instanceof JSONObject) {
+                        scrubbedArray.add(scrubJsonObject((JSONObject) item, patterns));
+                    } else {
+                        scrubbedArray.add(item);
+                    }
+                }
+
+                jsonObject.put(key, scrubbedArray.toArray());
+
+            } else if (value instanceof JSONObject) {
+                jsonObject.put(key, scrubJsonObject((JSONObject) value, patterns));
+            }
+        }
+
+        return jsonObject;
+    }
+
+    private boolean isSecret(String value, List<Pattern> patterns) {
+        for (Pattern pattern : patterns) {
+            if (pattern.matcher(value).matches()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}