Add support to skip decode

README.md

@@ -186,6 +186,7 @@ CONFIGURATIONS:
    -sd, -skip-dedupe             disable dedupe input items (only used with stream mode)
    -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443
    -ztls                         use ztls library with autofallback to standard one for tls13
+   -no-decode                    avoid decoding body
 
 DEBUG:
    -health-check, -hc        run diagnostic check up

common/httpx/httpx.go

@@ -218,6 +218,10 @@ get_response:
 		return nil, closeErr
 	}
 
+	// Todo: replace with https://github.com/projectdiscovery/utils/issues/110
+	resp.RawData = make([]byte, len(respbody))
+	copy(resp.RawData, respbody)
+
 	respbody, err = DecodeData(respbody, httpresp.Header)
 	if err != nil && !shouldIgnoreBodyErrors {
 		return nil, closeErr

common/httpx/response.go

@@ -12,7 +12,8 @@ import (
 type Response struct {
 	StatusCode    int
 	Headers       map[string][]string
-	Data          []byte
+	RawData       []byte // undecoded data
+	Data          []byte // decoded data
 	ContentLength int
 	Raw           string
 	RawHeaders    string

runner/options.go

@@ -262,6 +262,7 @@ type Options struct {
 	OutputMatchCondition      string
 	OnResult                  OnResultCallback
 	DisableUpdateCheck        bool
+	NoDecode                  bool
 }
 
 // ParseOptions parses the command line options for application
@@ -388,6 +389,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"),
 		flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"),
 		flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
+		flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"),
 	)
 
 	flagSet.CreateGroup("debug", "Debug",

runner/runner.go

@@ -1301,17 +1301,25 @@ retry:
 		builder.WriteString(fmt.Sprintf(" [%s]", serverHeader))
 	}
 
-	var serverResponseRaw string
-	var request string
-	var rawResponseHeader string
-	var responseHeader map[string]interface{}
+	var (
+		serverResponseRaw string
+		request           string
+		rawResponseHeader string
+		responseHeader    map[string]interface{}
+	)
+
+	respData := string(resp.Data)
+	if r.options.NoDecode {
+		respData = string(resp.RawData)
+	}
+
 	if scanopts.ResponseInStdout {
-		serverResponseRaw = string(resp.Data)
+		serverResponseRaw = string(respData)
 		request = string(requestDump)
 		responseHeader = normalizeHeaders(resp.Headers)
 		rawResponseHeader = resp.RawHeaders
 	} else if scanopts.Base64ResponseInStdout {
-		serverResponseRaw = stringz.Base64(resp.Data)
+		serverResponseRaw = stringz.Base64([]byte(respData))
 		request = stringz.Base64(requestDump)
 		responseHeader = normalizeHeaders(resp.Headers)
 		rawResponseHeader = stringz.Base64([]byte(resp.RawHeaders))