Fixes sqlmapproject#3622

lib/controller/checks.py

@@ -29,7 +29,6 @@
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
 from lib.core.common import getSortedInjectionTests
-from lib.core.common import getUnicode
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
@@ -49,6 +48,7 @@
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.defaults import defaults
 from lib.core.data import conf
 from lib.core.data import kb

lib/core/agent.py

@@ -12,7 +12,6 @@
 from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
-from lib.core.common import getUnicode
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -26,6 +25,7 @@
 from lib.core.common import urlencode
 from lib.core.common import zeroDepthSearch
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import queries

lib/core/common.py

@@ -56,6 +56,7 @@
 from lib.core.convert import decodeHex
 from lib.core.convert import getBytes
 from lib.core.convert import getText
+from lib.core.convert import getUnicode
 from lib.core.convert import htmlunescape
 from lib.core.convert import stdoutencode
 from lib.core.data import conf
@@ -2418,50 +2419,6 @@ def getPartRun(alias=True):
     else:
         return retVal
 
-def getUnicode(value, encoding=None, noneToNull=False):
-    """
-    Return the unicode representation of the supplied value:
-
-    >>> getUnicode('test') == u'test'
-    True
-    >>> getUnicode(1) == u'1'
-    True
-    """
-
-    if noneToNull and value is None:
-        return NULL
-
-    if isinstance(value, six.text_type):
-        return value
-    elif isinstance(value, six.binary_type):
-        # Heuristics (if encoding not explicitly specified)
-        candidates = filterNone((encoding, kb.get("pageEncoding") if kb.get("originalPage") else None, conf.get("encoding"), UNICODE_ENCODING, sys.getfilesystemencoding()))
-        if all(_ in value for _ in (b'<', b'>')):
-            pass
-        elif any(_ in value for _ in (b":\\", b'/', b'.')) and b'\n' not in value:
-            candidates = filterNone((encoding, sys.getfilesystemencoding(), kb.get("pageEncoding") if kb.get("originalPage") else None, UNICODE_ENCODING, conf.get("encoding")))
-        elif conf.get("encoding") and b'\n' not in value:
-            candidates = filterNone((encoding, conf.get("encoding"), kb.get("pageEncoding") if kb.get("originalPage") else None, sys.getfilesystemencoding(), UNICODE_ENCODING))
-
-        for candidate in candidates:
-            try:
-                return six.text_type(value, candidate)
-            except UnicodeDecodeError:
-                pass
-
-        try:
-            return six.text_type(value, encoding or (kb.get("pageEncoding") if kb.get("originalPage") else None) or UNICODE_ENCODING)
-        except UnicodeDecodeError:
-            return six.text_type(value, UNICODE_ENCODING, errors="reversible")
-    elif isListLike(value):
-        value = list(getUnicode(_, encoding, noneToNull) for _ in value)
-        return value
-    else:
-        try:
-            return six.text_type(value)
-        except UnicodeDecodeError:
-            return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
-
 def longestCommonPrefix(*sequences):
     """
     Returns longest common prefix occuring in given sequences

lib/core/convert.py

@@ -16,8 +16,11 @@
 import re
 import sys
 
+from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
 from lib.core.settings import IS_WIN
+from lib.core.settings import NULL
 from lib.core.settings import PICKLE_PROTOCOL
 from lib.core.settings import SAFE_HEX_MARKER
 from lib.core.settings import UNICODE_ENCODING
@@ -89,6 +92,12 @@ def singleTimeWarnMessage(message):  # Cross-referenced function
     sys.stdout.write("\n")
     sys.stdout.flush()
 
+def filterNone(values):  # Cross-referenced function
+    raise NotImplementedError
+
+def isListLike(value):  # Cross-referenced function
+    raise NotImplementedError
+
 def stdoutencode(data):
     retVal = data
 
@@ -146,7 +155,7 @@ def decodeHex(value, binary=True):
     retVal = value
 
     if isinstance(value, six.binary_type):
-        value = value.decode(UNICODE_ENCODING)
+        value = getText(value)
 
     if value.lower().startswith("0x"):
         value = value[2:]
@@ -250,6 +259,50 @@ def getOrds(value):
 
     return [_ if isinstance(_, int) else ord(_) for _ in value]
 
+def getUnicode(value, encoding=None, noneToNull=False):
+    """
+    Return the unicode representation of the supplied value:
+
+    >>> getUnicode('test') == u'test'
+    True
+    >>> getUnicode(1) == u'1'
+    True
+    """
+
+    if noneToNull and value is None:
+        return NULL
+
+    if isinstance(value, six.text_type):
+        return value
+    elif isinstance(value, six.binary_type):
+        # Heuristics (if encoding not explicitly specified)
+        candidates = filterNone((encoding, kb.get("pageEncoding") if kb.get("originalPage") else None, conf.get("encoding"), UNICODE_ENCODING, sys.getfilesystemencoding()))
+        if all(_ in value for _ in (b'<', b'>')):
+            pass
+        elif any(_ in value for _ in (b":\\", b'/', b'.')) and b'\n' not in value:
+            candidates = filterNone((encoding, sys.getfilesystemencoding(), kb.get("pageEncoding") if kb.get("originalPage") else None, UNICODE_ENCODING, conf.get("encoding")))
+        elif conf.get("encoding") and b'\n' not in value:
+            candidates = filterNone((encoding, conf.get("encoding"), kb.get("pageEncoding") if kb.get("originalPage") else None, sys.getfilesystemencoding(), UNICODE_ENCODING))
+
+        for candidate in candidates:
+            try:
+                return six.text_type(value, candidate)
+            except UnicodeDecodeError:
+                pass
+
+        try:
+            return six.text_type(value, encoding or (kb.get("pageEncoding") if kb.get("originalPage") else None) or UNICODE_ENCODING)
+        except UnicodeDecodeError:
+            return six.text_type(value, UNICODE_ENCODING, errors="reversible")
+    elif isListLike(value):
+        value = list(getUnicode(_, encoding, noneToNull) for _ in value)
+        return value
+    else:
+        try:
+            return six.text_type(value)
+        except UnicodeDecodeError:
+            return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
+
 def getText(value):
     """
     Returns textual value of a given value (Note: not necessary Unicode on Python2)
@@ -263,7 +316,7 @@ def getText(value):
     retVal = value
 
     if isinstance(value, six.binary_type):
-        retVal = value.decode(UNICODE_ENCODING)
+        retVal = getUnicode(value)
 
     if six.PY2:
         try:

lib/core/dump.py

@@ -18,7 +18,6 @@
 from lib.core.common import dataToDumpFile
 from lib.core.common import dataToStdout
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import isListLike
 from lib.core.common import isMultiThreadMode
 from lib.core.common import normalizeUnicode
@@ -29,6 +28,7 @@
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

lib/core/option.py

@@ -17,13 +17,6 @@
 import threading
 import time
 
-import lib.controller.checks
-import lib.core.common
-import lib.core.threads
-import lib.core.convert
-import lib.request.connect
-import lib.utils.search
-
 from lib.controller.checks import checkConnection
 from lib.core.common import Backend
 from lib.core.common import boldifyMessage
@@ -32,7 +25,6 @@
 from lib.core.common import decodeStringEscape
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import filterNone
 from lib.core.common import findLocalPort
 from lib.core.common import findPageForms
@@ -61,6 +53,7 @@
 from lib.core.common import urldecode
 from lib.core.compat import round
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -2621,15 +2614,6 @@ def _basicOptionValidation():
             errMsg = "cookies file '%s' does not exist" % conf.loadCookies
             raise SqlmapFilePathException(errMsg)
 
-def _resolveCrossReferences():
-    lib.core.threads.readInput = readInput
-    lib.core.common.getPageTemplate = getPageTemplate
-    lib.core.convert.singleTimeWarnMessage = singleTimeWarnMessage
-    lib.request.connect.setHTTPHandlers = _setHTTPHandlers
-    lib.utils.search.setHTTPHandlers = _setHTTPHandlers
-    lib.controller.checks.setVerbosity = setVerbosity
-    lib.controller.checks.setWafFunctions = _setWafFunctions
-
 def initOptions(inputOptions=AttribDict(), overrideOptions=False):
     _setConfAttributes()
     _setKnowledgeBaseAttributes()
@@ -2663,7 +2647,6 @@ def init():
     _setWafFunctions()
     _setTrafficOutputFP()
     _setupHTTPCollector()
-    _resolveCrossReferences()
     _setHttpChunked()
     _checkWebSocket()
 

lib/core/patch.py

@@ -7,6 +7,24 @@
 
 import codecs
 
+import lib.controller.checks
+import lib.core.common
+import lib.core.threads
+import lib.core.convert
+import lib.request.connect
+import lib.utils.search
+import thirdparty.ansistrm.ansistrm
+
+from lib.request.templates import getPageTemplate
+
+from lib.core.common import filterNone
+from lib.core.common import isListLike
+from lib.core.common import singleTimeWarnMessage
+from lib.core.common import readInput
+from lib.core.convert import stdoutencode
+from lib.core.option import _setHTTPHandlers
+from lib.core.option import setVerbosity
+from lib.core.option import _setWafFunctions
 from lib.core.settings import IS_WIN
 from thirdparty.six.moves import http_client as _http_client
 
@@ -32,3 +50,19 @@ def _(self, *args):
 
         _http_client.LineAndFileWrapper._readline = _http_client.LineAndFileWrapper.readline
         _http_client.LineAndFileWrapper.readline = _
+
+def resolveCrossReferences():
+    """
+    Place for cross-reference resolution
+    """
+
+    lib.core.threads.readInput = readInput
+    lib.core.common.getPageTemplate = getPageTemplate
+    lib.core.convert.filterNone = filterNone
+    lib.core.convert.isListLike = isListLike
+    lib.core.convert.singleTimeWarnMessage = singleTimeWarnMessage
+    lib.request.connect.setHTTPHandlers = _setHTTPHandlers
+    lib.utils.search.setHTTPHandlers = _setHTTPHandlers
+    lib.controller.checks.setVerbosity = setVerbosity
+    lib.controller.checks.setWafFunctions = _setWafFunctions
+    thirdparty.ansistrm.ansistrm.stdoutencode = stdoutencode

lib/core/settings.py

@@ -18,7 +18,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.5.21"
+VERSION = "1.3.5.22"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/target.py

@@ -15,7 +15,6 @@
 
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import hashDBRetrieve
 from lib.core.common import intersect
 from lib.core.common import isNumPosStrValue
@@ -27,6 +26,7 @@
 from lib.core.common import resetCookieJar
 from lib.core.common import urldecode
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

lib/core/testing.py

@@ -22,12 +22,12 @@
 from lib.controller.controller import start
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
-from lib.core.common import getUnicode
 from lib.core.common import randomStr
 from lib.core.common import readXmlFile
 from lib.core.common import shellExec
 from lib.core.compat import round
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths

lib/parse/cmdline.py

@@ -22,8 +22,8 @@
 from lib.core.common import dataToStdout
 from lib.core.common import expandMnemonics
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
 from lib.core.data import logger

lib/parse/configfile.py

@@ -7,10 +7,10 @@
 
 from lib.core.common import checkFile
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import openFile
 from lib.core.common import unArrayizeValue
 from lib.core.common import UnicodeRawConfigParser
+from lib.core.convert import getUnicode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
 from lib.core.data import logger

lib/request/basic.py

@@ -19,7 +19,6 @@
 from lib.core.common import filterNone
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import isListLike
 from lib.core.common import randomStr
 from lib.core.common import readInput
@@ -29,6 +28,7 @@
 from lib.core.common import unArrayizeValue
 from lib.core.convert import decodeHex
 from lib.core.convert import getBytes
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

lib/request/connect.py

@@ -39,7 +39,6 @@ class WebSocketException(Exception):
 from lib.core.common import getHostHeader
 from lib.core.common import getRequestHeader
 from lib.core.common import getSafeExString
-from lib.core.common import getUnicode
 from lib.core.common import isMultiThreadMode
 from lib.core.common import logHTTPTraffic
 from lib.core.common import pushValue
@@ -60,6 +59,7 @@ class WebSocketException(Exception):
 from lib.core.compat import patchHeaders
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger