BMP 2.1.33 (#25)

* Extend browsermob proxy add headers with additional field in json `headersFilterRegexp` which allows you to add headers only to specific domain

* update to openjdk8

* update to oraclejdk8

* Update litle proxy

* Update little proxy

* Update snapshot version

Author: kirillbilchenko

.travis.yml

@@ -1,6 +1,7 @@
 sudo: false
 
 language: java
+dist: trusty
 jdk:
     # Not running tests against openjdk7, since the SunEC is not included in travis-ci's version of openjdk7.
     # Not running tests against oraclejdk7, since travis-ci no longer provides it.

README.md

@@ -144,7 +144,7 @@ Set a URL to blacklist | PUT | */proxy/[port]/blacklist* |<p>*regex* - The black
 Clears all URL patterns from the blacklist | DELETE | */proxy/[port]/blacklist* ||
 Limit the bandwidth through the proxy on the *[port]* | PUT | */proxy/[port]/limit* |<p>*downstreamKbps* - Sets the downstream bandwidth limit in kbps. Optional.</p><p>*upstreamKbps* - Sets the upstream bandwidth limit kbps. Optional, by default unlimited.</p><p>*downstreamMaxKB* - Specifies how many kilobytes in total the client is allowed to download through the proxy. Optional, by default unlimited.</p><p>*upstreamMaxKB* - Specifies how many kilobytes in total the client is allowed to upload through the proxy. Optional, by default unlimited.</p><p>*latency* - Add the given latency to each HTTP request. Optional, by default all requests are invoked without latency.</p><p>*enable* - A boolean that enable bandwidth limiter. Optional, by default to "false", but setting any of the properties above will implicitly enable throttling</p><p>*payloadPercentage* - Specifying what percentage of data sent is payload, e.g. use this to take into account overhead due to tcp/ip. Optional.</p><p>*maxBitsPerSecond* - The max bits per seconds you want this instance of StreamManager to respect. Optional.</p>
 Displays the amount of data remaining to be uploaded/downloaded until the limit is reached | GET | */proxy/[port]/limit* ||
-Set and override HTTP Request headers | POST | */proxy/[port]/headers* | Payload data should be **JSON** encoded set of headers. Where key is a header name (such as "User-Agent") and  value is a value of HTTP header to setup (such as "BrowserMob-Agent"). Example: `{"User-Agent": "BrowserMob-Agent"}`|
+Set and override HTTP Request headers | POST | */proxy/[port]/headers* | Payload data should be **JSON** encoded set of headers and optionally `headersFilterRegexp: .*.*` . Where key is a header name (such as "User-Agent") and  value is a value of HTTP header to setup (such as "BrowserMob-Agent"). Example: `{"User-Agent": "BrowserMob-Agent"}`|
 Overrides normal DNS lookups and remaps the given hosts with the associated IP address | POST | */proxy/[port]/hosts* | Payload data should be **JSON** encoded set of hosts. Where key is a host name (such as "example.com") and value is a IP address which associatied with host hame (such as "1.2.3.4"'). Example: `{"example.com": "1.2.3.4"}`|
 Sets automatic basic authentication for the specified domain | POST | */proxy/[port]/auth/basic/[domain]* | Payload data should be **JSON** encoded username and password name/value pairs. Example: `{"username": "myUsername", "password": "myPassword"}`|
 Wait till all request are being made | PUT | */proxy/[port]/wait* |<p>*quietPeriodInMs* - Wait till all request are being made. Optional.</p><p>*timeoutInMs* - Sets quiet period in milliseconds. Optional.</p>|
@@ -396,7 +396,7 @@ When you build the latest code from source, you'll have access to the latest sna
     <dependency>
         <groupId>net.lightbody.bmp</groupId>
         <artifactId>browsermob-core</artifactId>
-        <version>2.1.33-SNAPSHOT</version>
+        <version>2.1.34-SNAPSHOT</version>
         <scope>test</scope>
     </dependency>
 ```

browsermob-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.33-SNAPSHOT</version>
+        <version>2.1.34-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxy.java

@@ -39,7 +39,7 @@ public interface BrowserMobProxy {
      * Starts the proxy on the specified port. The proxy will listen for connections on the network interface specified by the bindAddress, and will
      * also initiate connections to upstream servers on the same network interface.
      *
-     * @param port port to listen on
+     * @param port        port to listen on
      * @param bindAddress address of the network interface on which the proxy will listen for connections and also attempt to connect to upstream servers.
      * @throws java.lang.IllegalStateException if the proxy has already been started
      */
@@ -49,7 +49,7 @@ public interface BrowserMobProxy {
      * Starts the proxy on the specified port. The proxy will listen for connections on the network interface specified by the clientBindAddress, and will
      * initiate connections to upstream servers from the network interface specified by the serverBindAddress.
      *
-     * @param port port to listen on
+     * @param port              port to listen on
      * @param clientBindAddress address of the network interface on which the proxy will listen for connections
      * @param serverBindAddress address of the network interface on which the proxy will connect to upstream servers
      * @throws java.lang.IllegalStateException if the proxy has already been started
@@ -68,7 +68,7 @@ public interface BrowserMobProxy {
      * @throws java.lang.IllegalStateException if the proxy has not been started.
      */
     void stop();
-    
+
     /**
      * Like {@link #stop()}, shuts down the proxy server and no longer accepts incoming connections, but does not wait for any existing
      * network traffic to cease. Any existing connections to clients or to servers may be force-killed immediately.
@@ -125,7 +125,7 @@ public interface BrowserMobProxy {
     /**
      * Starts a new HAR file with the specified page name and page title. Enables HAR capture if it was not previously enabled.
      *
-     * @param initialPageRef initial page name of the new HAR file
+     * @param initialPageRef   initial page name of the new HAR file
      * @param initialPageTitle initial page title of the new HAR file
      * @return existing HAR file, or null if none exists or HAR capture was disabled
      */
@@ -215,7 +215,7 @@ public interface BrowserMobProxy {
      * Starts a new HAR page using the specified pageRef as the page name and the pageTitle as the page title. Populates the
      * {@link net.lightbody.bmp.core.har.HarPageTimings#onLoad} value based on the amount of time the current page has been captured.
      *
-     * @param pageRef name of the new page
+     * @param pageRef   name of the new page
      * @param pageTitle title of the new page
      * @return the HAR as it existed immediately after ending the current page
      * @throws java.lang.IllegalStateException if HAR capture has not been enabled via {@link #newHar()} or {@link #newHar(String)}
@@ -258,7 +258,7 @@ public interface BrowserMobProxy {
      * The minimum amount of time that will elapse between the time the proxy begins receiving a response from the server and the time the
      * proxy begins sending the response to the client.
      *
-     * @param latency minimum latency, or 0 for no minimum
+     * @param latency  minimum latency, or 0 for no minimum
      * @param timeUnit TimeUnit for the latency
      */
     void setLatency(long latency, TimeUnit timeUnit);
@@ -268,7 +268,7 @@ public interface BrowserMobProxy {
      * specified time, the proxy will respond with an HTTP 502 Bad Gateway. The default value is 60 seconds.
      *
      * @param connectionTimeout maximum time to wait to establish a connection to a server, or 0 to wait indefinitely
-     * @param timeUnit TimeUnit for the connectionTimeout
+     * @param timeUnit          TimeUnit for the connectionTimeout
      */
     void setConnectTimeout(int connectionTimeout, TimeUnit timeUnit);
 
@@ -279,7 +279,7 @@ public interface BrowserMobProxy {
      * connection to the client <i>may</i> be closed abruptly. The default value is 60 seconds.
      *
      * @param idleConnectionTimeout maximum time to allow a connection to remain idle, or 0 to wait indefinitely.
-     * @param timeUnit TimeUnit for the idleConnectionTimeout
+     * @param timeUnit              TimeUnit for the idleConnectionTimeout
      */
     void setIdleConnectionTimeout(int idleConnectionTimeout, TimeUnit timeUnit);
 
@@ -290,15 +290,15 @@ public interface BrowserMobProxy {
      * connection to the client <i>may</i> be closed abruptly. The default value is 0 (wait indefinitely).
      *
      * @param requestTimeout maximum time to wait for an HTTP response, or 0 to wait indefinitely
-     * @param timeUnit TimeUnit for the requestTimeout
+     * @param timeUnit       TimeUnit for the requestTimeout
      */
     void setRequestTimeout(int requestTimeout, TimeUnit timeUnit);
 
     /**
      * Enables automatic authorization for the specified domain and auth type. Every request sent to the specified domain will contain the
      * specified authorization information.
      *
-     * @param domain domain automatically send authorization information to
+     * @param domain   domain automatically send authorization information to
      * @param username authorization username
      * @param password authorization password
      * @param authType authorization type
@@ -340,7 +340,7 @@ public interface BrowserMobProxy {
      * For example, the following rewrite rule:
      *
      * <pre>   {@code proxy.rewriteUrl("http://www\\.(yahoo|bing)\\.com/\\?(\\w+)=(\\w+)", "http://www.google.com/?originalDomain=$1&$2=$3");}</pre>
-     *
+     * <p>
      * will match an HTTP request (but <i>not</i> HTTPS!) to www.yahoo.com or www.bing.com with exactly 1 query parameter,
      * and replace it with a call to www.google.com with an 'originalDomain' query parameter, as well as the original query parameter.
      * <p/>
@@ -353,7 +353,7 @@ public interface BrowserMobProxy {
      * will result in the proxy making a request to:
      * <pre>   {@code http://www.google.com?originalDomain=bing&anotherParam=anotherValue}</pre>
      *
-     * @param urlPattern URL-matching regular expression
+     * @param urlPattern            URL-matching regular expression
      * @param replacementExpression an expression, which may optionally contain capture groups, which will replace any URL which matches urlPattern
      */
     void rewriteUrl(String urlPattern, String replacementExpression);
@@ -410,8 +410,8 @@ public interface BrowserMobProxy {
      * <p/>
      * See {@link #blacklistRequests(String, int)} for details on the URL the urlPattern will match.
      *
-     * @param urlPattern URL-matching regular expression to blacklist
-     * @param statusCode HTTP status code to return
+     * @param urlPattern        URL-matching regular expression to blacklist
+     * @param statusCode        HTTP status code to return
      * @param httpMethodPattern regular expression matching a request's HTTP method
      */
     void blacklistRequests(String urlPattern, int statusCode, String httpMethodPattern);
@@ -447,7 +447,7 @@ public interface BrowserMobProxy {
      * whitelist response code.
      *
      * @param urlPatterns URL-matching regular expressions to whitelist; null or an empty collection will enable an empty whitelist
-     * @param statusCode HTTP status code to return to clients when a URL matches a pattern
+     * @param statusCode  HTTP status code to return to clients when a URL matches a pattern
      */
     void whitelistRequests(Collection<String> urlPatterns, int statusCode);
 
@@ -501,11 +501,18 @@ public interface BrowserMobProxy {
     /**
      * Adds a new HTTP header to every request. If the header already exists on the request, it will be replaced with the specified header.
      *
-     * @param name name of the header to add
+     * @param name  name of the header to add
      * @param value new header's value
      */
     void addHeader(String name, String value);
 
+    /**
+     * Header filter regexp.
+     *
+     * @param headerFilterRegexp the header filter regexp
+     */
+    void headerFilterRegexp(String headerFilterRegexp);
+
     /**
      * Removes a header previously added with {@link #addHeader(String name, String value)}.
      *
@@ -545,8 +552,8 @@ public interface BrowserMobProxy {
      * for the quiet period within the specified timeout, otherwise returns false.
      *
      * @param quietPeriod amount of time after which network traffic will be considered "stopped"
-     * @param timeout maximum amount of time to wait for network traffic to stop
-     * @param timeUnit TimeUnit for the quietPeriod and timeout
+     * @param timeout     maximum amount of time to wait for network traffic to stop
+     * @param timeUnit    TimeUnit for the quietPeriod and timeout
      * @return true if network traffic is stopped, otherwise false
      */
     boolean waitForQuiescence(long quietPeriod, long timeout, TimeUnit timeUnit);
@@ -587,7 +594,7 @@ public interface BrowserMobProxy {
      * {@link org.littleshoot.proxy.HttpFilters} instance (typically, a subclass of {@link org.littleshoot.proxy.HttpFiltersAdapter}).
      * To disable or bypass a filter on a per-request basis, the filterRequest() method may return null.
      *
-     *  @param filterFactory factory to generate HttpFilters
+     * @param filterFactory factory to generate HttpFilters
      */
     void addLastHttpFilterFactory(HttpFiltersSource filterFactory);
 

browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java

@@ -145,6 +145,11 @@ public class BrowserMobProxyServer implements BrowserMobProxy {
      */
     private volatile int connectTimeoutMs;
 
+    /**
+     * Regexp to check request url and inject headers if url match regexp.
+     */
+    private static String headersFilterRegexp;
+
     /**
      * The amount of time a connection to a server can remain idle while receiving data from the server.
      */
@@ -592,6 +597,11 @@ public void addHeaders(Map<String, String> headers) {
         this.additionalHeaders = newHeaders;
     }
 
+    @Override
+    public void headerFilterRegexp(String headerFilterRegexp) {
+        this.headersFilterRegexp = headerFilterRegexp;
+    }
+
     @Override
     public void setLatency(long latency, TimeUnit timeUnit) {
         this.latencyMs = (int) TimeUnit.MILLISECONDS.convert(latency, timeUnit);
@@ -1052,8 +1062,8 @@ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerCont
 
         addHttpFilterFactory(new HttpFiltersSourceAdapter() {
             @Override
-            public HttpFilters filterRequest(HttpRequest originalRequest) {
-                return new AddHeadersFilter(originalRequest, additionalHeaders);
+            public HttpFilters filterRequest(HttpRequest originalRequest,ChannelHandlerContext ctx) {
+                return new AddHeadersFilter(originalRequest,ctx, additionalHeaders, headersFilterRegexp);
             }
         });
 

browsermob-core/src/main/java/net/lightbody/bmp/filters/AddHeadersFilter.java

@@ -1,9 +1,10 @@
 package net.lightbody.bmp.filters;
 
+import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
-import org.littleshoot.proxy.HttpFiltersAdapter;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.Collections;
 import java.util.Map;
@@ -12,29 +13,48 @@
  * Adds the headers specified in the constructor to this request. The filter does not make a defensive copy of the map, so there is no guarantee
  * that the map at the time of construction will contain the same values when the filter is actually invoked, if the map is modified concurrently.
  */
-public class AddHeadersFilter extends HttpFiltersAdapter {
+public class AddHeadersFilter extends HttpsAwareFiltersAdapter {
     private final Map<String, String> additionalHeaders;
+    private static String headersSpecificFilter;
 
-    public AddHeadersFilter(HttpRequest originalRequest, Map<String, String> additionalHeaders) {
-        super(originalRequest);
+    public AddHeadersFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Map<String, String> additionalHeaders, String headersSpecificFilter) {
+        super(originalRequest, ctx);
 
         if (additionalHeaders != null) {
             this.additionalHeaders = additionalHeaders;
         } else {
             this.additionalHeaders = Collections.emptyMap();
         }
+        if (StringUtils.isNotEmpty(headersSpecificFilter)) {
+            setHeadersSpecificFilter(headersSpecificFilter);
+        }
     }
 
     @Override
     public HttpResponse clientToProxyRequest(HttpObject httpObject) {
         if (httpObject instanceof HttpRequest) {
             HttpRequest httpRequest = (HttpRequest) httpObject;
-
-            for (Map.Entry<String, String> header : additionalHeaders.entrySet()) {
-                httpRequest.headers().add(header.getKey(), header.getValue());
+            if (StringUtils.isNotEmpty(headersSpecificFilter)) {
+                if (getFullUrl(httpRequest).matches(headersSpecificFilter)) {
+                    for (Map.Entry<String, String> header : additionalHeaders.entrySet()) {
+                        httpRequest.headers().add(header.getKey(), header.getValue());
+                    }
+                }
+            } else {
+                for (Map.Entry<String, String> header : additionalHeaders.entrySet()) {
+                    httpRequest.headers().add(header.getKey(), header.getValue());
+                }
             }
         }
 
         return null;
     }
+
+    public String getHeadersSpecificFilter() {
+        return headersSpecificFilter;
+    }
+
+    private synchronized void setHeadersSpecificFilter(String headersSpecificFilter) {
+        this.headersSpecificFilter = headersSpecificFilter;
+    }
 }

browsermob-dist/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.33-SNAPSHOT</version>
+        <version>2.1.34-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-legacy/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.33-SNAPSHOT</version>
+        <version>2.1.34-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/LegacyProxyServer.java

@@ -18,7 +18,7 @@
 
 /**
  * Describes the legacy BrowserMob Proxy 2.0 interface. Clients <b>should not</b> implement or use this interface.
- *
+ * <p>
  * Use {@link BrowserMobProxy}.
  */
 public interface LegacyProxyServer {
@@ -122,6 +122,8 @@ public interface LegacyProxyServer {
 
     void addHeader(String name, String value);
 
+    void headerFilterRegexp(String headerFilterRegexp);
+
     void setCaptureHeaders(boolean captureHeaders);
 
     void setCaptureContent(boolean captureContent);