OWTF 0.12 Wicky release

.gitignore

@@ -0,0 +1,9 @@
+# Compiled source and other garbage #
+#####################################
+*.pyc
+*.swp
+
+# Directories with potential license restrictions that prevent re-distribution #
+################################################################################
+dictionaries/restricted/*
+tools/restricted/*

dictionaries/README

@@ -0,0 +1,4 @@
+Have a look at this for more background:
+
+http://pauldotcom.com/2011/08/dirbuster-to-burp-the-missing.html
+http://www.mavitunasecurity.com/blog/svn-digger-better-lists-for-forced-browsing/

dictionaries/update_convert_cms_explorer_dicts.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Description: This script grabs all the excellent CMS Explorer dictionaries, updates them and converts them into DirBuster format (much faster than CMS Explorer)
+#
+# owtf is an OWASP+PTES-focused try to unite great tools and facilitate pen testing
+# Copyright (c) 2011, Abraham Aranguren <name.surname@gmail.com> Twitter: @7a_ http://7-a.org
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright 
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of the copyright owner nor the
+# names of its contributors may be used to endorse or promote products
+# derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+CMS_EXPLORER_DIR="/pentest/enumeration/web/cms-explorer" # Backtrack 5 location, you may need to change this
+CMS_DICTIONARIES_DIR="/root/owtf/dictionaries/restricted/cms" # CMS dictionaries location, you may need to change this
+
+DICTIONARIES="$CMS_EXPLORER_DIR/drupal_plugins.txt
+$CMS_EXPLORER_DIR/joomla_themes.txt
+$CMS_EXPLORER_DIR/wp_plugins.txt
+$CMS_EXPLORER_DIR/drupal_themes.txt
+$CMS_EXPLORER_DIR/wp_themes.txt
+$CMS_EXPLORER_DIR/joomla_plugins.txt"
+
+echo "[*] Going into directory: $CMS_EXPLORER_DIR"
+cd $CMS_EXPLORER_DIR
+echo "[*] Updating cms-explorer.pl dictionaries.."
+./cms-explorer.pl -update
+
+echo "[*] Going into directory: $CMS_DICTIONARIES_DIR"
+cd $CMS_DICTIONARIES_DIR
+
+echo "[*] Copying updated dictionaries from $CMS_EXPLORER_DIR to $CMS_DICTIONARIES_DIR"
+for i in $(echo $DICTIONARIES); do
+	cp $i $CMS_DICTIONARIES_DIR # echo "[*] Copying $i .."
+done
+
+DIRBUSTER_PREFIX="dir_buster"
+for cms in $(echo "drupal joomla wp"); do
+	mkdir -p $cms # Create CMS specific directory
+	rm -f $cms/* # Remove previous dictionaries
+	mv $cms* $cms 2> /dev/null # Move relevant dictionaries to directory, getting rid of "cannot move to myself" error, which is ok
+	cd $cms # Enter directory
+	for dict in $(ls); do # Now process each CMS-specific dictionary and convert it
+		cat $dict | tr '/' "\n" | sort -u > $DIRBUSTER_PREFIX.$dict # Convert to DirBuster format (i.e. get rid of the "/" and duplicate parent directories)
+		rm -f $dict # Remove since this only works with CMS explorer
+	done
+	# Create all-in-one CMS-specific dictionaries:
+	cat $DIRBUSTER_PREFIX* > $DIRBUSTER_PREFIX.all.$cms.txt
+	cd ..
+done
+
+echo "[*] Creating all-in-one CMS dictionaries for DirBuster and CMS Explorer"
+for bruteforcer in $(echo "$DIRBUSTER_PREFIX"); do
+	ALLINONE_DICT="$bruteforcer.all_in_one.txt"
+	rm -f $ALLINONE_DICT # Remove previous, potentially outdated all-in-one dict
+	for all_dict in $(find . -name *all*.txt | grep $bruteforcer); do
+		cat $all_dict >> $ALLINONE_DICT
+	done
+	cat $ALLINONE_DICT | sort -u > $ALLINONE_DICT.tmp # Remove duplicates, just in case
+	mv $ALLINONE_DICT.tmp $ALLINONE_DICT
+done
+
+echo "[*] Done!]"

framework/config/config.py

@@ -305,39 +305,29 @@ def GetTXTTransacLog(self, Partial = False):
         def IsHostNameNOTIP(self):
                 return self.Get('HOST_NAME') != self.Get('HOST_IP') # Host
 
-        def GetIPFromHostname(self, hostname):
-                ip = ''
-                try:
-                    socket.inet_pton(socket.AF_INET, hostname)
-                    ip = hostname
-                except socket.error:
-                    # Check if it's an IPv6 address - not optimal
-                    # may be better to regex for : and decide
-                    # up front whether to check with AF_INET or AF_INET6
-                    try:
-                        socket.inet_pton(socket.AF_INET6, ip)
-                        ip = hostname
-                    except socket.error:
-                        # ideally we would not double up on this - wicky
-                        ip = socket.gethostbyname(hostname)
-                else:
-                    ip = socket.gethostbyname(hostname)
-
-                        #ip=self.Core.Shell.shell_exec('host '+hostname+'|grep "has address"|cut -f4 -d" "')
-		ipchunks = ip.strip().split("\n")
+        def GetIPFromHostname(self, Hostname):
+                IP = ''
+		for Socket in [ socket.AF_INET, socket.AF_INET6 ]: # IP validation based on @marcwickenden's pull request, thanks!
+			try:
+				socket.inet_pton(Socket, Hostname)
+				IP = Hostname
+				break
+			except socket.error: continue
+		if not IP:
+			try: IP = socket.gethostbyname(Hostname)
+			except socket.gaierror: self.Core.Error.FrameworkAbort("Cannot resolve Hostname: "+Hostname)
+
+		ipchunks = IP.strip().split("\n")
 		AlternativeIPs = []
 		if len(ipchunks) > 1:
-			ip = ipchunks[0]
-			cprint(hostname+" has several IP addresses: ("+", ".join(ipchunks)[0:-3]+"). Choosing first: "+ip+"")
+			IP = ipchunks[0]
+			cprint(Hostname+" has several IP addresses: ("+", ".join(ipchunks)[0:-3]+"). Choosing first: "+IP+"")
 			AlternativeIPs = ipchunks[1:]
 		self.Set('ALTERNATIVE_IPS', AlternativeIPs)
-                ip = ip.strip()
-		if len(ip.split('.')) != 4: # TODO: Add IPv6 support!
-			self.Core.Error.FrameworkAbort("Cannot resolve hostname: "+hostname)
-		# Good IPv4 IP
-		self.Set('INTERNAL_IP', self.Core.IsIPInternal(ip))
-		cprint("The IP address for "+hostname+" is: '"+ip+"'")
-		return ip
+                IP = IP.strip()
+		self.Set('INTERNAL_IP', self.Core.IsIPInternal(IP))
+		cprint("The IP address for "+Hostname+" is: '"+IP+"'")
+		return IP
 
 	def GetAll(self, Key): # Retrieves a config setting value on all target configurations
 		Matches = []

framework/config/framework_config.cfg

@@ -1,4 +1,4 @@
-VERSION: 0.11 "Vienna"
+VERSION: 0.12 "Wicky"
 
 INSTALL_SCRIPT: @@@FRAMEWORK_DIR@@@/install.sh
 WEB_TEST_GROUPS: @@@FRAMEWORK_DIR@@@/framework/config/web_testgroups.cfg
@@ -45,11 +45,15 @@ RESPONSE_REGEXP_FOR_SSI: Server Side Includes_____<!--#_____(<!--(#.*)?-->)
 
 # The following icons _must_ exist, but you can change the icon if you wish
 # WARNING!!!: The following icons are best kept as they are, if a new icon is wanted it should have the same name (at least for now):
+#FIXED_ICON_MATCHES: shopping_cart
+FIXED_ICON_MATCHES: target
 FIXED_ICON_INFO: info
+FIXED_ICON_PLUGIN_INFO: info24x24
 FIXED_ICON_NOFLAG: envelope
 FIXED_ICON_UNSTRIKETHROUGH: eraser
 FIXED_ICON_STRICKETHROUGH: pencil
 FIXED_ICON_NOTES: lamp_active
 FIXED_ICON_REMOVE: delete 
 FIXED_ICON_REFRESH: refresh
-COLLAPSED_REPORT_SIZE: 90
+FIXED_ICON_OPTIONS: options
+COLLAPSED_REPORT_SIZE: 64

framework/config/plugin.py

@@ -46,7 +46,7 @@ def GetTypesForGroup(self, PluginGroup):
                 PluginTypes = []
                 for PluginType, Plugins in self.AllPlugins[PluginGroup].items():
                         PluginTypes.append(PluginType)
-                return PluginTypes
+                return sorted(PluginTypes) # Return list in alphabetical order
 
         def GetAllTypes(self):
                 AllPluginTypes = []

framework/db/plugin_register.py

@@ -34,15 +34,16 @@
 # Start, End, Runtime, Command, LogStatus = self.Core.DB.DBCache['RUN_DB'][-1]#.split(" | ")
 CODE = 0
 TYPE = 1
-PATH = 2
-TARGET = 3 # The same plugin and type can be run against different targets, they should have different paths, but we need the target to get the right codes in the report
-ARGS = 4 # Auxiliary plugins have metasploit-like arguments
-REVIEW_OFFSET = 5
-START = 6
-END = 7
-RUNTIME = 8
+GROUP = 2 
+PATH = 3 
+TARGET = 4 # The same plugin and type can be run against different targets, they should have different paths, but we need the target to get the right codes in the report
+ARGS = 5 # Auxiliary plugins have metasploit-like arguments
+REVIEW_OFFSET = 6 
+START = 7 
+END = 8 
+RUNTIME = 9 
 
-NAME_TO_OFFSET = { 'Code' : CODE, 'Type' : TYPE, 'Path' : PATH, 'Target' : TARGET, 'Args' : ARGS, 'ReviewOffset' : REVIEW_OFFSET, 'Start' : START, 'End' : END, 'RunTime' : RUNTIME }
+NAME_TO_OFFSET = { 'Code' : CODE, 'Type' : TYPE, 'Group' : GROUP, 'Path' : PATH, 'Target' : TARGET, 'Args' : ARGS, 'ReviewOffset' : REVIEW_OFFSET, 'Start' : START, 'End' : END, 'RunTime' : RUNTIME }
 
 class PluginRegister:
 	def __init__(self, Core):
@@ -56,7 +57,10 @@ def AlreadyRegistered(self, Plugin, Path, Target):
 
 	def Add(self, Plugin, Path, Target): # Registers a Plugin/Path/Target combination only if not already registered
 		if not self.AlreadyRegistered(Plugin, Path, Target):
-			self.Core.DB.Add('PLUGIN_REPORT_REGISTER', [ Plugin['Code'], Plugin['Type'], Path, Target, Plugin['Args'], self.Core.Config.Get('REVIEW_OFFSET'), Plugin['Start'], Plugin['End'], Plugin['RunTime'] ] )
+			if 'RunTime' not in Plugin:
+				Plugin['RunTime'] = self.Core.Timer.GetElapsedTimeAsStr('Plugin')
+				Plugin['End'] = self.Core.Timer.GetEndDateTimeAsStr('Plugin')
+			self.Core.DB.Add('PLUGIN_REPORT_REGISTER', [ Plugin['Code'], Plugin['Type'], Plugin['Group'], Path, Target, Plugin['Args'], self.Core.Config.Get('REVIEW_OFFSET'), Plugin['Start'], Plugin['End'], Plugin['RunTime'] ] )
 
 	def Search(self, Criteria):
 		return self.Core.DB.Search('PLUGIN_REPORT_REGISTER', Criteria, NAME_TO_OFFSET)

framework/db/transaction_manager.py

@@ -287,4 +287,3 @@ def GrepMultiLineResponseRegexp(self, ResponseRegexp):
 			return [ Command, RegexpName, Matches ] # Return All matches and the file they were retrieved from
 		else: # wtf?
 			raise PluginAbortException("ERROR: Inforrect Configuration setting for Response Regexp: '"+str(ResponseRegexp)+"'")
-

framework/http/requester.py

@@ -128,7 +128,7 @@ def StringToDict(self, String):
 		Dict = defaultdict(list)
 		Count = 0
 		PrevItem = ''
-		for Item in String.split('='):
+		for Item in String.strip().split('='):
 			if Count % 2 == 1: # Key
 				Dict[PrevItem] = Item
 			else: # Value

framework/plugin/plugin_handler.py

@@ -210,8 +210,6 @@ def GetPluginFullPath(self, PluginDir, Plugin):
 		return PluginDir+"/"+Plugin['Type']+"/"+Plugin['File'] # Path to run the plugin 
 
 	def RunPlugin(self, PluginDir, Plugin):
-		self.Core.Timer.StartTimer('Plugin') # Time how long it takes the plugin to execute
-		Plugin['Start'] = self.Core.Timer.GetStartDateTimeAsStr('Plugin')
 		PluginPath = self.GetPluginFullPath(PluginDir, Plugin)
 		(Path, Name) = os.path.split(PluginPath)
 		#(Name, Ext) = os.path.splitext(Name)
@@ -220,6 +218,8 @@ def RunPlugin(self, PluginDir, Plugin):
 		self.SavePluginInfo(PluginOutput, Plugin) # Timer retrieved here
 
 	def ProcessPlugin(self, PluginDir, Plugin, Status):
+		self.Core.Timer.StartTimer('Plugin') # Time how long it takes the plugin to execute
+		Plugin['Start'] = self.Core.Timer.GetStartDateTimeAsStr('Plugin')
 		if not self.CanPluginRun(Plugin, True):
 			return None # Skip 
 		Status['AllSkipped'] = False # A plugin is going to be run

framework/plugin/plugin_helper.py

@@ -53,22 +53,49 @@ def DrawLinkList(self, LinkListName, Links): # Wrapper to allow rendering a bunc
 
         def DrawResourceLinkList(self, ResourceListName, ResourceList): # Draws an HTML Search box for defined Vuln Search resources
                 LinkList = []
-                List = [] 
+                HTMLLinkList = [] 
                 for Name, Resource in ResourceList:
                         URL = MultipleReplace(Resource.strip(), self.Core.Config.GetReplacementDict()).replace('"', '%22')
-                        #URL = Resource.replace('@@@PLACE_HOLDER@@@', self.Core.Config.Get('HOST_NAME')).strip()
                         LinkList.append(URL)
 			cprint("Generating link for "+Name+"..") # Otherwise there would be a lovely python exception and we would not be here :)
-                        #List += '<li>'+self.Core.Reporter.Render.DrawButtonLink(Name, URL)+"</li>\n"
-                        List.append(self.Core.Reporter.Render.DrawButtonLink(Name, URL))
-                #List += '</ul>'
+                        HTMLLinkList.append(self.Core.Reporter.Render.DrawButtonLink(Name, URL))
+		return self.DrawListPostProcessing(ResourceListName, LinkList, HTMLLinkList)
+
+	def DrawListPostProcessing(self, ResourceListName, LinkList, HTMLLinkList):
                 Content = '<hr />'+ResourceListName+': '
                 if len(LinkList) > 1: # Open All In Tabs only makes sense if num items > 1
                         Content += self.Core.Reporter.Render.DrawButton('Open All In Tabs', "OpenAllInTabs(new Array('"+"','".join(LinkList)+"'))")
-                #Content += '<br /><ul>\n'+List
-		Content += self.Core.Reporter.Render.DrawHTMLList(List)
+		Content += self.Core.Reporter.Render.DrawHTMLList(HTMLLinkList)
                 return Content
 
+	def RequestAndDrawLinkList(self, ResourceListName, ResourceList, PluginInfo):
+	        #for Name, Resource in Core.Config.GetResources('PassiveRobotsAnalysisHTTPRequests'):
+                LinkList = []
+                HTMLLinkList = [] 
+	        for Name, Resource in ResourceList:
+			Chunks = Resource.split('###POST###')
+			URL = Chunks[0]
+			POST = None
+			Method = 'GET'
+			if len(Chunks) > 1: # POST
+				Method = 'POST'
+				POST = Chunks[1]
+	                Transaction = self.Core.Requester.GetTransaction(True, URL, Method, POST)
+			if Transaction.Found:
+				Path, HTMLLink = self.SaveSandboxedTransactionHTML(Name, Transaction, PluginInfo)
+				HTMLLinkList.append(HTMLLink)
+				LinkList.append(Path)
+		return self.DrawListPostProcessing(ResourceListName, LinkList, HTMLLinkList)
+
+	def SaveSandboxedTransactionHTML(self, Name, Transaction, PluginInfo): # 1st filters HTML, 2nd builds sandboxed iframe, 3rd give link to sandboxed content
+		RawHTML = Transaction.GetRawResponseBody()
+		#print "RawHTML="+RawHTML
+		FilteredHTML = self.Core.Reporter.Sanitiser.CleanThirdPartyHTML(RawHTML)
+		#print "FilteredHTML="+FilteredHTML
+		NotSandboxedPath, Discarded = self.DumpFile("NOT_SANDBOXED_"+Name+".html", FilteredHTML, PluginInfo, Name)
+		SandboxedPath, HTMLLink = self.DumpFile("SANDBOXED_"+Name+".html", self.Core.Reporter.Render.DrawiFrame( { 'src' : NotSandboxedPath.split('/')[-1], 'sandbox' : '',  'security' : 'restricted', 'width' : '100%', 'height' : '100%', 'frameborder' : '0', 'style' : "overflow-y:auto; overflow-x:hidden;" } ), PluginInfo, Name)
+		return [ SandboxedPath, HTMLLink ]
+
         def DrawVulnerabilitySearchBox(self, SearchStr): # Draws an HTML Search box for defined Vuln Search resources
                 ProductId = 'prod'+self.Core.DB.GetNextHTMLID() # Keep product id unique among different search boxes (so that Javascript works)
                 Count = 0
@@ -222,15 +249,15 @@ def DrawCommandDump(self, CommandIntro, OutputIntro, ResourceList, PluginInfo, P
 				raise FrameworkAbortException(PreviousOutput+Content)
                 return Content
 
-	def DumpFile(self, Filename, Contents, PluginInfo):
+	def DumpFile(self, Filename, Contents, PluginInfo, LinkName = ''):
                 save_path = self.Core.PluginHandler.DumpPluginFile(Filename, Contents, PluginInfo)
+		if not LinkName:
+			LinkName = save_path
 		cprint("File: "+Filename+" saved to: "+save_path)
-                return [ save_path, '<br />Saved to: '+self.Core.Reporter.Render.DrawButtonLink(save_path, save_path, {}, True) ]
+                return [ save_path, self.Core.Reporter.Render.DrawButtonLink(LinkName, save_path, {}, True) ]
 
-	def DumpFileGetLink(self, Filename, Contents, PluginInfo, LinkName):
-                save_path = self.Core.PluginHandler.DumpPluginFile(Filename, Contents, PluginInfo)
-		cprint("File: "+Filename+" saved to: "+save_path)
-                return self.Core.Reporter.Render.DrawButtonLink(LinkName, save_path, {}, True)
+	def DumpFileGetLink(self, Filename, Contents, PluginInfo, LinkName = ''):
+		return self.DumpFile(Filename, Contents, PluginInfo, LinkName)[0]
 
 	def AnalyseRobotsEntries(self, Contents): # Find the entries of each kind and count them
                 num_lines = len(Contents.split("\n")) # Total number of robots.txt entries