diff --git a/.classpath b/.classpath deleted file mode 100644 index 149cb3c..0000000 --- a/.classpath +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/.fatjar b/.fatjar deleted file mode 100644 index 5eada3a..0000000 --- a/.fatjar +++ /dev/null @@ -1,14 +0,0 @@ -#Fat Jar Configuration File -#Fri Oct 26 14:20:32 CST 2018 -onejar.license.required=true -manifest.classpath= -manifest.removesigners=true -onejar.checkbox=false -jarname=D\:\\github\\domain_hunter\\domain_hunter.v1.2.jar -manifest.mergeall=true -manifest.mainclass= -manifest.file= -jarname.isextern=true -onejar.expand= -excludes=~test/ -includes= diff --git a/.gitignore b/.gitignore index eea009b..624e8e8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .project /bin/ /target/ +/.settings/ +.fatjar diff --git a/README.md b/README.md index 84de272..7003aac 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **author** -[bit4](https://github.com/bit4woo) +[bit4woo](https://github.com/bit4woo) **domain_hunter** @@ -46,6 +46,8 @@ Some times similar domain and related domains give you surprise^_^. that's why I 2018-11-01: Add "Add to domain hunter" menu in site map tree. +2019-07-06: Use multiple thread to improve search speed. Use regex to find more domain in every response. + **xmind of domain collection** ![xmind](doc/xmind.png) diff --git a/domain_hunter-1.3.jar b/domain_hunter-1.3.jar deleted file mode 100644 index 7e869e2..0000000 Binary files a/domain_hunter-1.3.jar and /dev/null differ diff --git a/pom.xml b/pom.xml index 3db1b8d..7cd93e8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,61 +1,69 @@ - - 4.0.0 - com.bit4woo.burp - domain_hunter - 1.3 - - src - - - maven-compiler-plugin - 3.7.0 - - 1.8 - 1.8 - - - - - maven-assembly-plugin - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - - - - - net.portswigger.burp.extender - burp-extender-api - 1.7.22 - - - - - com.alibaba - fastjson - 1.2.51 - - - - - com.google.guava - guava - 19.0 - + + 4.0.0 + com.bit4woo.burp + domain_hunter + 1.4 + + src + + + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + maven-assembly-plugin + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + + + + net.portswigger.burp.extender + burp-extender-api + 1.7.22 + + + + + com.alibaba + fastjson + 1.2.51 + + + + + com.google.guava + guava + 19.0 + + + + + org.apache.commons + commons-text + 1.6 + \ No newline at end of file diff --git a/src/burp/BurpExtender.java b/src/burp/BurpExtender.java index 8acfc83..83baba1 100644 --- a/src/burp/BurpExtender.java +++ b/src/burp/BurpExtender.java @@ -1,29 +1,24 @@ package burp; +import java.awt.Component; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.io.PrintWriter; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import javax.swing.JMenuItem; import javax.swing.SwingUtilities; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.Component; - -import java.io.PrintWriter; -import java.net.MalformedURLException; -import java.net.URL; public class BurpExtender extends GUI implements IBurpExtender, ITab, IExtensionStateListener,IContextMenuFactory{ - private IBurpExtenderCallbacks callbacks; + private static IBurpExtenderCallbacks callbacks; private IExtensionHelpers helpers; @@ -58,86 +53,11 @@ public void extensionUnloaded() { @Override public Map> search(Set rootdomains, Set keywords){ - - Set httpsURLs = new HashSet(); - Set httpServiceSet = getHttpServiceFromSiteMap(); - for (IHttpService httpservice:httpServiceSet){ - - String shortURL = httpservice.toString(); - String protocol = httpservice.getProtocol(); - String Host = httpservice.getHost(); - - //callbacks.printOutput(rootdomains.toString()); - //callbacks.printOutput(keywords.toString()); - int type = domainResult.domainType(Host); - //callbacks.printOutput(Host+":"+type); - if (type == DomainObject.SUB_DOMAIN) - { - domainResult.subDomainSet.add(Host); - }else if (type == DomainObject.SIMILAR_DOMAIN) { - domainResult.similarDomainSet.add(Host); - } - - if (type !=DomainObject.USELESS && protocol.equalsIgnoreCase("https")){ - httpsURLs.add(shortURL); - } - } - - stdout.println("sub-domains and similar-domains search finished,starting get related-domains"); - //stdout.println(httpsURLs); - - //多线程获取 - //Set>> set = new HashSet>>(); - Map>> urlResultmap = new HashMap>>(); - ExecutorService pool = Executors.newFixedThreadPool(10); - - for (String url:httpsURLs) { - Callable> callable = new ThreadCertInfo(url,keywords); - Future> future = pool.submit(callable); - //set.add(future); - urlResultmap.put(url, future); - } - - Set tmpRelatedDomainSet = new HashSet(); - for(String url:urlResultmap.keySet()) { - Future> future = urlResultmap.get(url); - //for (Future> future : set) { - try { - stdout.println("founded related-domains :"+future.get() +" from "+url); - if (future.get()!=null) { - tmpRelatedDomainSet.addAll(future.get()); - } - } catch (Exception e) { - //e.printStackTrace(stderr); - stderr.println(e.getMessage()); - } - } - domainResult.relatedDomainSet =tmpRelatedDomainSet; - /* 单线程获取方式 - Set tmpRelatedDomainSet = new HashSet(); - //begin get related domains - for(String url:httpsURLs) { - try { - tmpRelatedDomainSet.addAll(CertInfo.getSANs(url)); - }catch(UnknownHostException e) { - stderr.println("UnknownHost "+ url); - continue; - }catch(ConnectException e) { - stderr.println("Connect Failed "+ url); - continue; - } - catch (Exception e) { - e.printStackTrace(stderr); - continue; - } - } - */ - //to save domain result to extensionSetting - String content= domainResult.Save(); - callbacks.saveExtensionSetting("content", content); - - return null; - } + IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks(); + IHttpRequestResponse[] messages = callbacks.getSiteMap(null); + new ThreadSearhDomain(Arrays.asList(messages)).Do(); + return null; + } @Override public Map> crawl (Set rootdomains, Set keywords) { @@ -309,6 +229,11 @@ public void actionPerformed(ActionEvent e) } } } + + public static IBurpExtenderCallbacks getCallbacks() { + // TODO Auto-generated method stub + return callbacks; + } /* public static void main(String[] args) { diff --git a/src/burp/GUI.java b/src/burp/GUI.java index 249bb1e..4d2310d 100644 --- a/src/burp/GUI.java +++ b/src/burp/GUI.java @@ -67,10 +67,10 @@ public class GUI extends JFrame { - public String ExtenderName = "Domain Hunter v1.3 by bit4"; + public String ExtenderName = "Domain Hunter v1.4 by bit4woo"; public String github = "https://github.com/bit4woo/domain_hunter"; - public DomainObject domainResult = new DomainObject(""); + public static DomainObject domainResult = new DomainObject(""); public PrintWriter stdout; public PrintWriter stderr; @@ -811,4 +811,10 @@ public boolean accept(File file) { } } + + public static DomainObject getDomainResult() { + // TODO Auto-generated method stub + return domainResult; + } + } diff --git a/src/burp/Getter.java b/src/burp/Getter.java new file mode 100644 index 0000000..6932709 --- /dev/null +++ b/src/burp/Getter.java @@ -0,0 +1,230 @@ +package burp; + +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; + +public class Getter { + private static IExtensionHelpers helpers; + private final static String Header_Spliter = ": "; + public Getter(IExtensionHelpers helpers) { + this.helpers = helpers; + } + + /* + * 获取header的字符串数组，是构造burp中请求需要的格式。 + */ + public List getHeaderList(boolean messageIsRequest,IHttpRequestResponse messageInfo) { + if(messageIsRequest) { + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + List headers = analyzeRequest.getHeaders(); + return headers; + }else { + IResponseInfo analyzeResponse = helpers.analyzeResponse(messageInfo.getResponse()); + List headers = analyzeResponse.getHeaders(); + return headers; + } + } + + public List getHeaderList(boolean IsRequest,byte[] requestOrResponse) { + if(IsRequest) { + IRequestInfo analyzeRequest = helpers.analyzeRequest(requestOrResponse); + List headers = analyzeRequest.getHeaders(); + return headers; + }else { + IResponseInfo analyzeResponse = helpers.analyzeResponse(requestOrResponse); + List headers = analyzeResponse.getHeaders(); + return headers; + } + } + + /* + * 获取所有headers，当做一个string看待。 + * 主要用于判断是否包含某个特殊字符串 + * List getHeaders 调用toString()方法，得到如下格式：[111111, 2222] + * 就能满足上面的场景了 + */ + public String getHeaderString(boolean messageIsRequest,IHttpRequestResponse messageInfo) { + List headers =null; + StringBuilder headerString = new StringBuilder(); + if(messageIsRequest) { + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + headers = analyzeRequest.getHeaders(); + }else { + IResponseInfo analyzeResponse = helpers.analyzeResponse(messageInfo.getResponse()); + headers = analyzeResponse.getHeaders(); + } + + for (String header : headers) { + headerString.append(header); + } + + return headerString.toString(); + } + + /* + * 获取header的map格式，key:value形式 + * 这种方式可以用put函数轻松实现：如果有则update，如果无则add。 + * ！！！注意：这个方法获取到的map，会少了协议头GET /cps.gec/limit/information.html HTTP/1.1 + */ + public HashMap getHeaderHashMap(boolean messageIsRequest,IHttpRequestResponse messageInfo) { + List headers=null; + HashMap result = new HashMap(); + if(messageIsRequest) { + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + headers = analyzeRequest.getHeaders(); + }else { + IResponseInfo analyzeResponse = helpers.analyzeResponse(messageInfo.getResponse()); + headers = analyzeResponse.getHeaders(); + } + + for (String header : headers) { + if(header.contains(Header_Spliter)) {//to void trigger the Exception + try { + String headerName = header.split(Header_Spliter, 0)[0]; + String headerValue = header.split(Header_Spliter, 0)[1]; + //POST /login.pub HTTP/1.1 the first line of header will tirgger error here + result.put(headerName, headerValue); + } catch (Exception e) { + //e.printStackTrace(); + } + } + } + return result; + } + + public List MapToList(HashMap Headers){ + List result = new ArrayList(); + for (Entry header:Headers.entrySet()) { + String item = header.getKey()+Header_Spliter+header.getValue(); + result.add(item); + } + return result; + } + + /* + * 获取某个header的值，如果没有此header，返回null。 + */ + public String getHeaderValueOf(boolean messageIsRequest,IHttpRequestResponse messageInfo, String headerName) { + List headers=null; + if(messageIsRequest) { + if (messageInfo.getRequest() == null) { + return null; + } + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + headers = analyzeRequest.getHeaders(); + }else { + if (messageInfo.getResponse() == null) { + return null; + } + IResponseInfo analyzeResponse = helpers.analyzeResponse(messageInfo.getResponse()); + headers = analyzeResponse.getHeaders(); + } + + + headerName = headerName.toLowerCase().replace(":", ""); + for (String header : headers) { + if (header.toLowerCase().startsWith(headerName)) { + return header.split(Header_Spliter, 2)[1];//分成2部分，Location: https://www.jd.com + } + } + return null; + } + + + public byte[] getBody(boolean messageIsRequest,IHttpRequestResponse messageInfo) { + if (messageInfo == null){ + return null; + } + if(messageIsRequest) { + if (messageInfo.getRequest() ==null) { + return null; + } + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + int bodyOffset = analyzeRequest.getBodyOffset(); + byte[] byte_Request = messageInfo.getRequest(); + + byte[] byte_body = Arrays.copyOfRange(byte_Request, bodyOffset, byte_Request.length);//not length-1 + //String body = new String(byte_body); //byte[] to String + return byte_body; + }else { + if (messageInfo.getResponse() ==null) { + return null; + } + IResponseInfo analyzeResponse = helpers.analyzeResponse(messageInfo.getResponse()); + int bodyOffset = analyzeResponse.getBodyOffset(); + byte[] byte_Request = messageInfo.getResponse(); + + byte[] byte_body = Arrays.copyOfRange(byte_Request, bodyOffset, byte_Request.length);//not length-1 + return byte_body; + } + } + + public byte[] getBody(boolean isRequest,byte[] requestOrResponse) { + if (requestOrResponse == null){ + return null; + } + int bodyOffset = -1; + if(isRequest) { + IRequestInfo analyzeRequest = helpers.analyzeRequest(requestOrResponse); + bodyOffset = analyzeRequest.getBodyOffset(); + }else { + IResponseInfo analyzeResponse = helpers.analyzeResponse(requestOrResponse); + bodyOffset = analyzeResponse.getBodyOffset(); + } + byte[] byte_body = Arrays.copyOfRange(requestOrResponse, bodyOffset, requestOrResponse.length);//not length-1 + //String body = new String(byte_body); //byte[] to String + return byte_body; + } + + + public String getShortUrl(IHttpRequestResponse messageInfo) { + return messageInfo.getHttpService().toString(); + } + + public URL getURL(IHttpRequestResponse messageInfo){ + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + return analyzeRequest.getUrl(); + + //callbacks.getHelpers().analyzeRequest(baseRequestResponse).getUrl(); + } + + public String getHost(IHttpRequestResponse messageInfo) { + return messageInfo.getHttpService().getHost(); + } + + public short getStatusCode(IHttpRequestResponse messageInfo) { + if (messageInfo == null || messageInfo.getResponse() == null) { + return -1; + } + IResponseInfo analyzedResponse = helpers.analyzeResponse(messageInfo.getResponse()); + return analyzedResponse.getStatusCode(); + } + + public List getParas(IHttpRequestResponse messageInfo){ + IRequestInfo analyzeRequest = helpers.analyzeRequest(messageInfo); + return analyzeRequest.getParameters(); + } + + + public String getHTTPBasicCredentials(IHttpRequestResponse messageInfo) throws Exception{ + String authHeader = getHeaderValueOf(true, messageInfo, "Authorization").trim(); + String[] parts = authHeader.split("\\s"); + + if (parts.length != 2) + throw new Exception("Wrong number of HTTP Authorization header parts"); + + if (!parts[0].equalsIgnoreCase("Basic")) + throw new Exception("HTTP authentication must be Basic"); + + return parts[1]; + } + + public static void main(String args[]) { + String a= "xxxxx%s%bxxxxxxx"; + System.out.println(String.format(a, "111")); + } +} diff --git a/src/burp/ThreadSearhDomain.java b/src/burp/ThreadSearhDomain.java new file mode 100644 index 0000000..cf6dfd6 --- /dev/null +++ b/src/burp/ThreadSearhDomain.java @@ -0,0 +1,243 @@ +package burp; + +import java.io.PrintWriter; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.text.StringEscapeUtils; + +//////////////////ThreadGetTitle block///////////// +//no need to pass BurpExtender object to these class, IBurpExtenderCallbacks object is enough +class ThreadSearhDomain{ + private List messages; + private List plist; + + private static IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();//静态变量，burp插件的逻辑中，是可以保证它被初始化的。; + public PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true); + public PrintWriter stderr = new PrintWriter(callbacks.getStderr(), true); + public IExtensionHelpers helpers = callbacks.getHelpers(); + + public ThreadSearhDomain(List messages) { + this.messages = messages; + } + + public void Do(){ + stdout.println("~~~~~~~~~~~~~Start Search Domain~~~~~~~~~~~~~"); + + BlockingQueue inputQueue = new LinkedBlockingQueue();//use to store messageInfo + BlockingQueue subDomainQueue = new LinkedBlockingQueue(); + BlockingQueue similarDomainQueue = new LinkedBlockingQueue(); + BlockingQueue relatedDomainQueue = new LinkedBlockingQueue(); + + inputQueue.addAll(messages); + + plist = new ArrayList(); + + for (int i=0;i<=50;i++) { + DomainProducer p = new DomainProducer(inputQueue,subDomainQueue, + similarDomainQueue,relatedDomainQueue,i); + p.start(); + plist.add(p); + } + + while(true) {//to wait all threads exit. + if (inputQueue.isEmpty() && isAllProductorFinished()) { + stdout.println("~~~~~~~~~~~~~Search Domain Done~~~~~~~~~~~~~"); + break; + }else { + try { + Thread.sleep(1*1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + continue; + } + } + int oldnumber = GUI.getDomainResult().getSubDomainSet().size(); + + GUI.getDomainResult().getSubDomainSet().addAll(subDomainQueue); + GUI.getDomainResult().getSimilarDomainSet().addAll(similarDomainQueue); + GUI.getDomainResult().getRelatedDomainSet().addAll(relatedDomainQueue); + + int newnumber = GUI.getDomainResult().getSubDomainSet().size(); + stdout.println(String.format("~~~~~~~~~~~~~%s subdomains added!~~~~~~~~~~~~~",newnumber-oldnumber)); + + return; + } + + boolean isAllProductorFinished(){ + for (DomainProducer p:plist) { + if(p.isAlive()) { + return false; + } + } + return true; + } + + public void stopThreads() { + for (DomainProducer p:plist) { + p.stopThread(); + } + stdout.println("threads stopped!"); + } + + public static void main(String args[]) {//test + System.out.println(DomainProducer.grepDomain("http://www.jd.com/usr/www.baidu.com/xss.jd.com")); + } +} + +/* + * do request use method of burp + * return IResponseInfo object Set + * + */ + +class DomainProducer extends Thread {//Producer do + private final BlockingQueue inputQueue;//use to store messageInfo + private final BlockingQueue subDomainQueue; + private final BlockingQueue similarDomainQueue; + private final BlockingQueue relatedDomainQueue; + private BlockingQueue httpsQueue = new LinkedBlockingQueue<>();//temp variable to identify checked https + + private int threadNo; + private boolean stopflag = false; + + private static IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();//静态变量，burp插件的逻辑中，是可以保证它被初始化的。; + public PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true); + public PrintWriter stderr = new PrintWriter(callbacks.getStderr(), true); + public IExtensionHelpers helpers = callbacks.getHelpers(); + + public DomainProducer(BlockingQueue inputQueue, + BlockingQueue subDomainQueue, + BlockingQueue similarDomainQueue, + BlockingQueue relatedDomainQueue, + int threadNo) { + this.threadNo = threadNo; + this.inputQueue = inputQueue; + this.subDomainQueue = subDomainQueue; + this.similarDomainQueue = similarDomainQueue; + this.relatedDomainQueue = relatedDomainQueue; + stopflag= false; + } + + public void stopThread() { + stopflag = true; + } + + @Override + public void run() { + while(true){ + Getter getter = new Getter(helpers); + try { + if (inputQueue.isEmpty() || stopflag) { + //stdout.println("Producer break"); + break; + } + + IHttpRequestResponse messageinfo = inputQueue.take(); + + IHttpService httpservice = messageinfo.getHttpService(); + String urlString = helpers.analyzeRequest(messageinfo).getUrl().toString(); + + String shortURL = httpservice.toString(); + String protocol = httpservice.getProtocol(); + String Host = httpservice.getHost(); + + //callbacks.printOutput(rootdomains.toString()); + //callbacks.printOutput(keywords.toString()); + int type = GUI.domainResult.domainType(Host); + //callbacks.printOutput(Host+":"+type); + if (type == DomainObject.SUB_DOMAIN) + { + if (!subDomainQueue.contains(Host)) { + subDomainQueue.add(Host); + } + }else if (type == DomainObject.SIMILAR_DOMAIN) { + if (!similarDomainQueue.contains(Host)) { + similarDomainQueue.add(Host); + } + } + + if (type !=DomainObject.USELESS && protocol.equalsIgnoreCase("https")){//get related domains + if (!httpsQueue.contains(shortURL)) {//httpService checked or not + Set tmpDomains = CertInfo.getSANs(shortURL,GUI.domainResult.fetchKeywordSet()); + for (String domain:tmpDomains) { + if (!relatedDomainQueue.contains(domain)) { + relatedDomainQueue.add(domain); + } + } + httpsQueue.add(shortURL); + } + } + + if (type != DomainObject.USELESS) {//grep domains from response and classify + if (urlString.endsWith(".gif") ||urlString.endsWith(".jpg") + || urlString.endsWith(".png") ||urlString.endsWith(".css")) { + + }else { + classifyDomains(messageinfo); + //classifyEmails(messageinfo); + } + } + } catch (Throwable error) {//java.lang.RuntimeException can't been catched, why? + } + } + } + + public void classifyDomains(IHttpRequestResponse messageinfo) { + byte[] response = messageinfo.getResponse(); + if (response != null) { + Set domains = DomainProducer.grepDomain(new String(response)); + for (String domain:domains) { + int type = GUI.domainResult.domainType(domain); + if (type == DomainObject.SUB_DOMAIN) + { + subDomainQueue.add(domain); + + }else if (type == DomainObject.SIMILAR_DOMAIN) { + similarDomainQueue.add(domain); + } + } + } + } + + public static Set grepDomain(String httpResponse) { + Set domains = new HashSet<>(); + //"^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$" + final String DOMAIN_NAME_PATTERN = "([A-Za-z0-9-]{1,63}(?