Merge pull request lightbody#576 from jekh/remove-uadetector

Remove uadetector

Author: jekh

browsermob-core/pom.xml

@@ -78,12 +78,6 @@
             <artifactId>jackson-annotations</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>net.sf.uadetector</groupId>
-            <artifactId>uadetector-resources</artifactId>
-            <version>2014.10</version>
-        </dependency>
-
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java

@@ -43,8 +43,8 @@
 import net.lightbody.bmp.proxy.auth.AuthType;
 import net.lightbody.bmp.proxy.dns.AdvancedHostResolver;
 import net.lightbody.bmp.proxy.dns.DelegatingHostResolver;
-import net.lightbody.bmp.util.BrowserMobProxyUtil;
 import net.lightbody.bmp.util.BrowserMobHttpUtil;
+import net.lightbody.bmp.util.BrowserMobProxyUtil;
 import org.littleshoot.proxy.ChainedProxy;
 import org.littleshoot.proxy.ChainedProxyAdapter;
 import org.littleshoot.proxy.ChainedProxyManager;
@@ -461,9 +461,6 @@ public Har newHar(String initialPageRef) {
 
     @Override
     public Har newHar(String initialPageRef, String initialPageTitle) {
-        // eagerly initialize the User Agent String Parser, since it will be needed for the HAR
-        BrowserMobProxyUtil.getUserAgentStringParser();
-
         Har oldHar = getHar();
 
         addHarCaptureFilter();

browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java

@@ -17,7 +17,6 @@
 import net.lightbody.bmp.core.har.HarCookie;
 import net.lightbody.bmp.core.har.HarEntry;
 import net.lightbody.bmp.core.har.HarNameValuePair;
-import net.lightbody.bmp.core.har.HarNameVersion;
 import net.lightbody.bmp.core.har.HarPostData;
 import net.lightbody.bmp.core.har.HarPostDataParam;
 import net.lightbody.bmp.core.har.HarRequest;
@@ -27,8 +26,6 @@
 import net.lightbody.bmp.filters.util.HarCaptureUtil;
 import net.lightbody.bmp.proxy.CaptureType;
 import net.lightbody.bmp.util.BrowserMobHttpUtil;
-import net.lightbody.bmp.util.BrowserMobProxyUtil;
-import net.sf.uadetector.ReadableUserAgent;
 import org.littleshoot.proxy.impl.ProxyUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -204,7 +201,9 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
             harEntry.setResponse(defaultHarResponse);
 
             captureQueryParameters(httpRequest);
-            captureUserAgent(httpRequest);
+            // not capturing user agent: in many cases, it doesn't make sense to capture at the HarLog level, since the proxy could be
+            // serving requests from many different clients with various user agents. clients can turn on the REQUEST_HEADERS capture type
+            // in order to capture the User-Agent header, if desired.
             captureRequestHeaderSize(httpRequest);
 
             if (dataToCapture.contains(CaptureType.REQUEST_COOKIES)) {
@@ -337,23 +336,6 @@ protected void captureQueryParameters(HttpRequest httpRequest) {
         }
     }
 
-    protected void captureUserAgent(HttpRequest httpRequest) {
-        // save the browser and version if it's not yet been set
-        if (har.getLog().getBrowser() == null) {
-            String userAgentHeader = HttpHeaders.getHeader(httpRequest, HttpHeaders.Names.USER_AGENT);
-            if (userAgentHeader != null && userAgentHeader.length() > 0) {
-                try {
-                    ReadableUserAgent uai = BrowserMobProxyUtil.getUserAgentStringParser().parse(userAgentHeader);
-                    String browser = uai.getName();
-                    String version = uai.getVersionNumber().toVersionString();
-                    har.getLog().setBrowser(new HarNameVersion(browser, version));
-                } catch (RuntimeException e) {
-                    log.warn("Failed to parse user agent string", e);
-                }
-            }
-        }
-    }
-
     protected void captureRequestHeaderSize(HttpRequest httpRequest) {
         String requestLine = httpRequest.getMethod().toString() + ' ' + httpRequest.getUri() + ' ' + httpRequest.getProtocolVersion().toString();
         // +2 => CRLF after status line, +4 => header/data separation

browsermob-core/src/main/java/net/lightbody/bmp/util/BrowserMobProxyUtil.java

@@ -7,8 +7,6 @@
 import net.lightbody.bmp.core.har.HarLog;
 import net.lightbody.bmp.core.har.HarPage;
 import net.lightbody.bmp.mitm.exception.UncheckedIOException;
-import net.sf.uadetector.UserAgentStringParser;
-import net.sf.uadetector.service.UADetectorServiceFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,11 +30,6 @@ public class BrowserMobProxyUtil {
      */
     private static final String UNKNOWN_VERSION_STRING = "UNKNOWN-VERSION";
 
-    /**
-     * Singleton User Agent parser.
-     */
-    private static volatile UserAgentStringParser parser;
-
     /**
      * Singleton version string loader.
      */
@@ -47,27 +40,6 @@ public String get() {
         }
     });
 
-    private static final Object PARSER_INIT_LOCK = new Object();
-
-    /**
-     * Retrieve the User Agent String Parser. Create the parser if it has not yet been initialized.
-     * 
-     * @return singleton UserAgentStringParser object
-     */
-    public static UserAgentStringParser getUserAgentStringParser() {
-        if (parser == null) {
-            synchronized (PARSER_INIT_LOCK) {
-                if (parser == null) {
-                    // using resourceModuleParser for now because user-agent-string.info no longer exists. the updating
-                    // parser will get incorrect data and wipe out its entire user agent repository.
-                    parser = UADetectorServiceFactory.getResourceModuleParser();
-                }
-            }
-        }
-
-        return parser;
-    }
-
     /**
      * Copies {@link HarEntry} and {@link HarPage} references from the specified har to a new har copy, up to and including
      * the specified pageRef. Does not perform a "deep copy", so any subsequent modification to the entries or pages will

browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java

@@ -7,7 +7,6 @@
 import net.lightbody.bmp.core.har.HarCookie;
 import net.lightbody.bmp.core.har.HarEntry;
 import net.lightbody.bmp.core.har.HarNameValuePair;
-import net.lightbody.bmp.core.har.HarNameVersion;
 import net.lightbody.bmp.core.har.HarPostData;
 import net.lightbody.bmp.core.har.HarPostDataParam;
 import net.lightbody.bmp.core.har.HarRequest;
@@ -18,11 +17,9 @@
 import net.lightbody.bmp.proxy.dns.AdvancedHostResolver;
 import net.lightbody.bmp.proxy.jetty.util.MultiMap;
 import net.lightbody.bmp.proxy.jetty.util.UrlEncoded;
-import net.lightbody.bmp.util.BrowserMobProxyUtil;
 import net.lightbody.bmp.proxy.util.CappedByteArrayOutputStream;
 import net.lightbody.bmp.proxy.util.ClonedOutputStream;
 import net.lightbody.bmp.proxy.util.IOUtils;
-import net.sf.uadetector.ReadableUserAgent;
 import org.apache.http.Header;
 import org.apache.http.HeaderElement;
 import org.apache.http.HttpClientConnection;
@@ -220,8 +217,8 @@ public class BrowserMobHttpClient {
 
     /**
      * Hostname resolver that wraps a {@link net.lightbody.bmp.proxy.dns.HostResolver}. The wrapped HostResolver can be replaced safely at
-     * runtime using {@link LegacyHostResolverAdapter#setResolver(net.lightbody.bmp.proxy.dns.HostResolver)}.
-     * See {@link #setResolver(net.lightbody.bmp.proxy.dns.HostResolver)}.
+     * runtime using {@link LegacyHostResolverAdapter#setResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
+     * See {@link #setResolver(net.lightbody.bmp.proxy.dns.AdvancedHostResolver)}.
      */
     private final LegacyHostResolverAdapter resolverWrapper = new LegacyHostResolverAdapter(ClientUtil.createDnsJavaWithNativeFallbackResolver());
 
@@ -636,23 +633,6 @@ private BrowserMobHttpResponse execute(BrowserMobHttpRequest req, int depth) {
         HttpRequestBase method = req.getMethod();
         String url = method.getURI().toString();
 
-        // save the browser and version if it's not yet been set
-        if (har != null && har.getLog().getBrowser() == null) {
-            Header[] uaHeaders = method.getHeaders("User-Agent");
-            if (uaHeaders != null && uaHeaders.length > 0) {
-                String userAgent = uaHeaders[0].getValue();
-                try {
-                    // note: this doesn't work for 'Fandango/4.5.1 CFNetwork/548.1.4 Darwin/11.0.0'
-                    ReadableUserAgent uai = BrowserMobProxyUtil.getUserAgentStringParser().parse(userAgent);
-                    String browser = uai.getName();
-                    String version = uai.getVersionNumber().toVersionString();
-                    har.getLog().setBrowser(new HarNameVersion(browser, version));
-                } catch (RuntimeException e) {
-                	LOG.warn("Failed to parse user agent string", e);
-                }
-            }
-        }
-
         // process any rewrite requests
         boolean rewrote = false;
         String newUrl = url;
@@ -1155,9 +1135,6 @@ public void abortActiveRequests() {
 
     public void setHar(Har har) {
         this.har = har;
-        
-        // eagerly initialize the User Agent String Parser, since it will be needed for the HAR
-        BrowserMobProxyUtil.getUserAgentStringParser();
     }
 
     public void setHarPageRef(String harPageRef) {

browsermob-legacy/src/test/java/net/lightbody/bmp/proxy/HarTest.java

@@ -84,27 +84,6 @@ public void testRequestAndResponseSizesAreSet() throws Exception {
         assertEquals(13, entry.getResponse().getBodySize());
     }
 
-	@Test
-	public void testHarContainsUserAgent() throws IOException, InterruptedException {
-		proxy.setCaptureHeaders(true);
-		proxy.newHar("testHarContainsUserAgent");
-
-		HttpGet httpGet = new HttpGet(getLocalServerHostnameAndPort() + "/echo");
-		httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0");
-		EntityUtils.consumeQuietly(client.execute(httpGet).getEntity());
-
-        Thread.sleep(500);
-		Har har = proxy.getHar();
-		assertNotNull("Har is null", har);
-		HarLog log = har.getLog();
-		assertNotNull("Log is null", log);
-		HarNameVersion harNameVersion = log.getBrowser();
-		assertNotNull("HarNameVersion is null", harNameVersion);
-
-		assertEquals("Expected browser to be Firefox", "Firefox", harNameVersion.getName());
-		assertEquals("Expected browser version to be 31.0", "31.0", harNameVersion.getVersion());
-	}
-
 	@Test
 	public void testThatProxyCanCaptureBodyInHar() throws IOException, InterruptedException {
 		proxy.setCaptureContent(true);