domain hunter v1.4

1.use multiple thread to improve search speed 2.use regex to find more domain in every response
devsecops-SRC · Jul 6, 2019 · 01ec1bf · 01ec1bf
1 parent fc8001d
commit 01ec1bf
Show file tree

Hide file tree

Showing 10 changed files with 570 additions and 188 deletions.
diff --git a/.classpath b/.classpath
diff --git a/.fatjar b/.fatjar
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,5 @@
 .project
 /bin/
 /target/
+/.settings/
+.fatjar
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 **author**
 
-[bit4](https://github.com/bit4woo)
+[bit4woo](https://github.com/bit4woo)
 
 **domain_hunter**
 
@@ -46,6 +46,8 @@ Some times similar domain and related domains give you surprise^_^. that's why I
 
 2018-11-01: Add "Add to domain hunter"  menu  in site map tree.
 
+2019-07-06: Use multiple thread to improve search speed. Use regex to find more domain in every response.
+
 **xmind of domain collection**
 
 ![xmind](doc/xmind.png)

diff --git a/domain_hunter-1.3.jar b/domain_hunter-1.3.jar
diff --git a/pom.xml b/pom.xml
@@ -1,61 +1,69 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>com.bit4woo.burp</groupId>
-  <artifactId>domain_hunter</artifactId>
-  <version>1.3</version>
-  <build>
-    <sourceDirectory>src</sourceDirectory>
-    <plugins>
-      <plugin>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.7.0</version>
-        <configuration>
-          <source>1.8</source>
-          <target>1.8</target>
-        </configuration>
-      </plugin>
-
-      <plugin>
-	    <artifactId>maven-assembly-plugin</artifactId>
-	    <configuration>
-	        <descriptorRefs>
-	            <descriptorRef>jar-with-dependencies</descriptorRef>
-	        </descriptorRefs>
-	    </configuration>
-	    <executions>
-	        <execution>
-	            <id>make-assembly</id>
-	            <phase>package</phase>
-	            <goals>
-	                <goal>single</goal>
-	            </goals>
-	        </execution>
-	    </executions>
-	</plugin>
-
-    </plugins>
-  </build>
-  <dependencies>
-	<!-- https://mvnrepository.com/artifact/net.portswigger.burp.extender/burp-extender-api -->
-	<dependency>
-	    <groupId>net.portswigger.burp.extender</groupId>
-	    <artifactId>burp-extender-api</artifactId>
-	    <version>1.7.22</version>
-	</dependency>
-
-	<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
-	<dependency>
-	    <groupId>com.alibaba</groupId>
-	    <artifactId>fastjson</artifactId>
-	    <version>1.2.51</version>
-	</dependency>
-
-	<!-- to get root domain -->
-	<dependency>
-     <groupId>com.google.guava</groupId>
-     <artifactId>guava</artifactId>
-     <version>19.0</version>
-	</dependency>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>com.bit4woo.burp</groupId>
+	<artifactId>domain_hunter</artifactId>
+	<version>1.4</version>
+	<build>
+		<sourceDirectory>src</sourceDirectory>
+		<plugins>
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.7.0</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
 
+			<plugin>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<configuration>
+					<descriptorRefs>
+						<descriptorRef>jar-with-dependencies</descriptorRef>
+					</descriptorRefs>
+				</configuration>
+				<executions>
+					<execution>
+						<id>make-assembly</id>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
+		</plugins>
+	</build>
+	<dependencies>
+		<!-- https://mvnrepository.com/artifact/net.portswigger.burp.extender/burp-extender-api -->
+		<dependency>
+			<groupId>net.portswigger.burp.extender</groupId>
+			<artifactId>burp-extender-api</artifactId>
+			<version>1.7.22</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
+		<dependency>
+			<groupId>com.alibaba</groupId>
+			<artifactId>fastjson</artifactId>
+			<version>1.2.51</version>
+		</dependency>
+
+		<!-- to get root domain -->
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>19.0</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-text -->
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-text</artifactId>
+			<version>1.6</version>
+		</dependency>
 	</dependencies>
 </project>
diff --git a/src/burp/BurpExtender.java b/src/burp/BurpExtender.java
@@ -1,29 +1,24 @@
 package burp;
 
+import java.awt.Component;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.io.PrintWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 
 import javax.swing.JMenuItem;
 import javax.swing.SwingUtilities;
-import java.awt.event.ActionEvent;
-import java.awt.event.ActionListener;
-import java.awt.Component;
-
-import java.io.PrintWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
 
 
 public class BurpExtender extends GUI implements IBurpExtender, ITab, IExtensionStateListener,IContextMenuFactory{
-	private IBurpExtenderCallbacks callbacks;
+	private static IBurpExtenderCallbacks callbacks;
     private IExtensionHelpers helpers;
 
 
@@ -58,86 +53,11 @@ public void extensionUnloaded() {
 
     @Override
 	public Map<String, Set<String>> search(Set<String> rootdomains, Set<String> keywords){
-
-		Set<String> httpsURLs = new HashSet<String>();
-		Set<IHttpService> httpServiceSet = getHttpServiceFromSiteMap();
-	    for (IHttpService httpservice:httpServiceSet){
-
-	    	String shortURL = httpservice.toString();
-	    	String protocol =  httpservice.getProtocol();
-			String Host = httpservice.getHost();
-
-			//callbacks.printOutput(rootdomains.toString());
-			//callbacks.printOutput(keywords.toString());
-			int type = domainResult.domainType(Host);
-			//callbacks.printOutput(Host+":"+type);
-			if (type == DomainObject.SUB_DOMAIN)
-			{
-				domainResult.subDomainSet.add(Host);
-			}else if (type == DomainObject.SIMILAR_DOMAIN) {
-				domainResult.similarDomainSet.add(Host);
-			}
-
-			if (type !=DomainObject.USELESS && protocol.equalsIgnoreCase("https")){
-				httpsURLs.add(shortURL);
-			}
-	    }
-
-	    stdout.println("sub-domains and similar-domains search finished,starting get related-domains");
-	    //stdout.println(httpsURLs);
-
-	    //多线程获取
-	    //Set<Future<Set<String>>> set = new HashSet<Future<Set<String>>>();
-    	Map<String,Future<Set<String>>> urlResultmap = new HashMap<String,Future<Set<String>>>();
-        ExecutorService pool = Executors.newFixedThreadPool(10);
-
-        for (String url:httpsURLs) {
-          Callable<Set<String>> callable = new ThreadCertInfo(url,keywords);
-          Future<Set<String>> future = pool.submit(callable);
-          //set.add(future);
-          urlResultmap.put(url, future);
-        }
-
-        Set<String> tmpRelatedDomainSet = new HashSet<String>();
-        for(String url:urlResultmap.keySet()) {
-        	Future<Set<String>> future = urlResultmap.get(url);
-        //for (Future<Set<String>> future : set) {
-          try {
-        	  stdout.println("founded related-domains :"+future.get() +" from "+url);
-        	  if (future.get()!=null) {
-        		  tmpRelatedDomainSet.addAll(future.get());
-        	  }
-		} catch (Exception e) {
-			//e.printStackTrace(stderr);
-			stderr.println(e.getMessage());
-        }
-        }
-        domainResult.relatedDomainSet =tmpRelatedDomainSet;
-        /* 单线程获取方式
-	    Set<String> tmpRelatedDomainSet = new HashSet<String>();
-	    //begin get related domains
-	    for(String url:httpsURLs) {
-	    	try {
-	    		tmpRelatedDomainSet.addAll(CertInfo.getSANs(url));
-			}catch(UnknownHostException e) {
-				stderr.println("UnknownHost "+ url);
-				continue;
-			}catch(ConnectException e) {
-				stderr.println("Connect Failed "+ url);
-				continue;
-			}
-	    	catch (Exception e) {
-				e.printStackTrace(stderr);
-				continue;
-			}
-	    }
-	    */
-        //to save domain result to extensionSetting
-        String content= domainResult.Save();
-        callbacks.saveExtensionSetting("content", content);
-
-	    return null;
-    }
+		IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();
+		IHttpRequestResponse[] messages = callbacks.getSiteMap(null);
+		new ThreadSearhDomain(Arrays.asList(messages)).Do();
+		return null;
+	}
 
 	@Override
 	public Map<String, Set<String>> crawl (Set<String> rootdomains, Set<String> keywords) {
@@ -309,6 +229,11 @@ public void actionPerformed(ActionEvent e)
 	        }
 	    }
 	}
+
+	public static IBurpExtenderCallbacks getCallbacks() {
+		// TODO Auto-generated method stub
+		return callbacks;
+	}
 
 
 /*	public static void main(String[] args) {

diff --git a/src/burp/GUI.java b/src/burp/GUI.java
@@ -67,10 +67,10 @@
 
 public class GUI extends JFrame {
 
-    public String ExtenderName = "Domain Hunter v1.3 by bit4";
+    public String ExtenderName = "Domain Hunter v1.4 by bit4woo";
     public String github = "https://github.com/bit4woo/domain_hunter";
 
-    public DomainObject domainResult = new DomainObject("");
+    public static DomainObject domainResult = new DomainObject("");
 
     public PrintWriter stdout;
     public PrintWriter stderr;
@@ -811,4 +811,10 @@ public boolean accept(File file) {
 	    }
 	}
 
+
+	public static DomainObject getDomainResult() {
+		// TODO Auto-generated method stub
+		return domainResult;
+	}
+
 }