Remove explicit test for file creation in travis.yml. Fix typos. Fix badges. Add Gowitness.

Aleksandar Straumann · Aleksandar Straumann · commit e92a0b784c71 · 2019-10-04T13:53:18.000-06:00
diff --git a/.travis.yml b/.travis.yml
@@ -18,73 +18,65 @@ jobs:
         - stage: start scans
           name: Sublist3r
           script:
-            - drrobot gather -sub github.com
-            - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/sublist3r)" ]; then exit 1; else echo "Success"; fi;
+            - drrobot gather -sub localhost
         - script:
-             - drrobot gather -knock github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/knock)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -knock localhost
           name: Knock
         - script:
-             - drrobot gather -amass github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/amass)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -amass localhost
           name: Amass 
         - script:
-             - drrobot gather -turbo github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/turbolist3r)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -turbo localhost
           name: Turbolist3r
         - script:
-             - drrobot gather -sfinder github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/subfinder)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -sfinder localhost
           name: Subfinder
         - script:
-             - drrobot gather -recon github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/reconng)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -recon localhost
           name: ReconNG
         - script:
-             - drrobot gather -altdns github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/altdns)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -altdns localhost
           name: AltDNS
         - script:
-             - drrobot gather -anubis github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/anubis)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -anubis localhost
           name: Anubis
         - script:
-             - drrobot gather -ctexpo github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/ctexpose)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -ctexpo localhost
           name: CTExpose
         - script:
-             - drrobot gather -ctfr github.com
+             - drrobot gather -ctfr localhost 
           name: CTFR
         - script:
-             - drrobot gather -pdlist github.com
-             - if [ -z "$(ls -A $HOME/.drrobot/output/github.com/pdlist)" ]; then exit 1; else echo "Success"; fi;
+             - drrobot gather -pdlist localhost
           name: PDList
         - stage: Test screenshots
           name: Webscreenshot
           script:
              - drrobot -h
              - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated
-             - echo "https://github.com" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
+             - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
              - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt  -webscreen singlewebsitetest
-             - if [ -z "$(ls -A $HOME/.drrobot/output/singlewebsitetest/webscreenshot)" ]; then exit 1; else echo "Success"; fi;
         - script:
              - drrobot -h
              - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated
-             - echo "https://github.com" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
+             - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
              - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt  -nmapscreen singlewebsitetest
-             - if [ -z "$(ls -A $HOME/.drrobot/output/singlewebsitetest/nmapscreen)" ]; then exit 1; else echo "Success"; fi;
           name: NMAP
         - script:
              - drrobot -h
              - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated
-             - echo "https://github.com" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
+             - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
              - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -eye singlewebsitetest
-             - if [ -z "$(ls -A $HOME/.drrobot/output/singlewebsitetest/Eyewitness)" ]; then exit 1; else echo "Success"; fi;
           name: Eyewitness
         - script:
              - drrobot -h
              - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated
-             - echo "https://github.com" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
+             - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
              - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt  -http singlewebsitetest
-             - if [ -z "$(ls -A $HOME/.drrobot/output/singlewebsitetest/httpscreenshot)" ]; then exit 1; else echo "Success"; fi;
           name: HTTPScreenshot 
+        - script:
+             - drrobot -h
+             - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated
+             - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt
+             - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt  -gowitness singlewebsitetest
+          name: GOWitness 
diff --git a/README.md b/README.md
@@ -10,15 +10,8 @@
  :                            :                  :             
 ```
 
-[![Dc27Badge](https://img.shields.io/badge/DEF%20CON-27-green)](https://defcon.org/html/defcon-27/dc-27-demolabs.html#Dr.%20ROBOT)
-[![License](http://img.shields.io/:license-mit-blue.svg)](https://github.com/sandialabs/dr_robot/blob/master/LICENSE)
-[![Build Status](https://travis-ci.org/sandialabs/dr_robot.svg?branch=master)](https://travis-ci.org/sandialabs/dr_robot)
-[Version](https://img.shields.io/github/v/release/SandiaLabs/dr_robot)
-[Last Commit](https://img.shields.io/github/last-commit/sandialabs/dr_robot)
-[Docker](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/docker)
-[Mattermost](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/mattermostdriver)
-[SlackClient](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/slackclient)
-[Size](https://img.shields.io/github/repo-size/sandialabs/dr_robot)
+[![Dc27Badge](https://img.shields.io/badge/DEF%20CON-27-green)](https://defcon.org/html/defcon-27/dc-27-demolabs.html#Dr.%20ROBOT)[![License](http://img.shields.io/:license-mit-blue.svg)](https://github.com/sandialabs/dr_robot/blob/master/LICENSE)[![Build Status](https://travis-ci.org/sandialabs/dr_robot.svg?branch=master)](https://travis-ci.org/sandialabs/dr_robot)[![GitHub release (latest by date)](https://img.shields.io/github/v/release/sandialabs/dr_robot)](https://github.com/sandialabs/dr_robot/blob/master/CHANGELOG.md)[![GitHub Pipenv locked dependency version](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/docker)]()[![GitHub Pipenv locked dependency version](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/mattermostdriver)]() ![GitHub Pipenv locked dependency version](https://img.shields.io/github/pipenv/locked/dependency-version/sandialabs/dr_robot/slackclient)
+
 Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software.
 
 ## Introduction
@@ -49,7 +42,7 @@ Dr.ROBOT is a tool for **Domain Reconnaissance and Enumeration**. By utilizing c
 
 ## Config Files
 
-DrROBOT adds config files, templates, logs, output files, and db files to your $HOME directory under ```.drrobot```
+Dr.ROBOT adds config files, templates, logs, output files, and db files to your ```$HOME``` directory under ```.drrobot```
 
 The directory structure will look like this:
 ```
@@ -61,7 +54,7 @@ drwxr-xr-x  21     672 Sep 16 13:51 docker_buildfiles
 drwxr-xr-x   4     128 Sep 16 15:38 logs
 drwxr-xr-x   3      96 Sep 16 12:46 output
 ```
-If you ever break your config beyond saving, you can delete the config.json file in your $HOME directory and rerun Dr.ROBOT, which will generate a new config file for you.
+If you ever break your config beyond saving, you can delete the config.json file in your ```$HOME``` directory and rerun Dr.ROBOT, which will generate a new config file for you.
 
 ## Installation (with pip)
 
@@ -150,7 +143,7 @@ optional arguments:
 
 ## Certs
 
-Running this behind a proxy was a pain. To make this less painful we create a certs directory under the **$HOME/.drrobot/*** where you can add your crt files. As part of the dockerfile build process we now generate tarfiles with the certificates so that applications, such as Amass, can run.
+Running this behind a proxy was a pain. To make this less painful we create a certs directory under the ```$HOME/.drrobot/*``` where you can add your crt files. As part of the dockerfile build process we now generate tarfiles with the certificates so that applications, such as Amass, can run.
 
 ## Minio
 
diff --git a/readmes/config.md b/readmes/config.md
@@ -85,7 +85,7 @@ Take special note of the `ansible_arguments`. The two required items are `config
 Note: 
 
 * `$infile` comes from the outermost **infile**, so that it is consistent for both docker and ansible. You can use a full path to a file for input if you desire.
-* `$outdir` comes from Dr.ROBOT. It will generate a path that points to `$HOME/.drrobot/output/<domain>/`. Again, you can specify a custom path if you like. 
+* `$outdir` comes from Dr.ROBOT. It will generate a path that points to ````$HOME/.drrobot/output/<domain>/`. ``` Again, you can specify a custom path if you like. 
 
 #### The Playbook
 This will simply be a standard playbook with a few changes so that Dr.ROBOT can use the parameters we fed it. To make sure a parameter that we specified in the "extra_flags" JSON blob is available,  use Ansible syntax for variables: ```"{{ variable_name|quote }}"``` (Note the *quote* helps prevent issues with variable names)
diff --git a/readmes/usage.md b/readmes/usage.md
@@ -20,7 +20,7 @@ drrobot gather <tool> <domain>
 
 ## Inspection
 
-Inspection is not a passive usage. It will run a tool that uses the aggregated information to grab screenshots for manual enumeration.
+The inspection process will generate noise due to the headless automated browsing which are easily detectable. If your list is large enough this could cause your IP to be blocked. 
 
 ```
 drrobot inspect --help
diff --git a/src/robot_api/configs/default_config.json b/src/robot_api/configs/default_config.json
@@ -304,6 +304,18 @@
             "infile" : "/tmp/output/aggregated/aggregated_hostnames.txt",
             "tool" : "chromium-browser"
         },
+        "Gowitness" : {
+            "name" : "Gowitness",
+            "short_name" : "gowitness",
+            "docker_name" : "gowitness",
+            "mode" : "DOCKER",
+            "network_mode": "host",
+            "default_conf" : "docker_buildfiles/Dockerfile.Gowitness.Screenshot.tmp",
+            "active_conf" : "docker_active/Dockerfile.Gowitness.Screenshot",
+            "description" : "gowitness is a website screenshot utility written in Golang",
+            "output" : "/tmp/output/",
+            "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt"
+        },
         "Webscreenshot" : {
             "name" : "WebScreenshot",
             "short_name" : "webscreen",
diff --git a/src/robot_api/docker_buildfiles/Dockerfile.Eyewitness.tmp b/src/robot_api/docker_buildfiles/Dockerfile.Eyewitness.tmp
@@ -41,4 +41,4 @@ RUN cd setup && \
     chown -R $$user:$$user /home/$$user/EyeWitness && \
     mkdir -p /tmp/EyeWitness
 
-ENTRYPOINT mkdir -p $output/Eyewitness && python EyeWitness.py -d $output -f $infile --no-prompt --web --threads 40 --max-retries 1
+ENTRYPOINT mkdir -p $output/Eyewitness && python EyeWitness.py -d $output/EyeWitness -f $infile --no-prompt --web --threads 40 --max-retries 1
diff --git a/src/robot_api/docker_buildfiles/Dockerfile.Gowitness.Screenshot.tmp b/src/robot_api/docker_buildfiles/Dockerfile.Gowitness.Screenshot.tmp
@@ -0,0 +1,32 @@
+FROM golang:1.13.1-buster as build
+
+RUN if [ -n $dns ]; \
+    then echo "nameserver $dns" >> /etc/resolv.conf;\
+    fi;\
+	apt update && \
+	apt install -y git ca-certificates chromium; 
+
+ADD certs/ /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+
+RUN if [ -n $dns ]; \
+    then echo "nameserver $dns" >> /etc/resolv.conf;\
+    fi;\
+	go get -u github.com/sensepost/gowitness
+
+ENV GO111MODULE on
+
+WORKDIR /go/src/github.com/sensepost/gowitness
+
+RUN if [ -n $dns ]; \
+    then echo "nameserver $dns" >> /etc/resolv.conf;\
+    fi;\
+	go build && \
+	cp gowitness /usr/bin/
+
+ENV http_proxy $proxy
+ENV https_proxy $proxy
+ENV HOME /
+
+
+ENTRYPOINT mkdir $output/gowitness && cd $output/gowitness && gowitness file -s $infile --threads 20
diff --git a/src/robot_api/docker_buildfiles/Dockerfile.HTTPScreenshot.tmp b/src/robot_api/docker_buildfiles/Dockerfile.HTTPScreenshot.tmp
@@ -23,7 +23,7 @@ RUN if [ -n $dns ]; \
 RUN if [ -n $dns ]; \
     then echo "nameserver $dns" >> /etc/resolv.conf;\
     fi;\
-    git clone https://github.com/Sandarmann/httpscreenshot.git
+    git clone https://github.com/CrimsonK1ng/httpscreenshot.git
 
 RUN cd httpscreenshot \
     && ./install-dependencies.sh \
@@ -33,7 +33,4 @@ RUN cd httpscreenshot \
     && unzip -o chromedriver_linux64.zip \
     && ln -s /httpscreenshot/chromedriver /usr/bin/chromedriver
 
-RUN mkdir -p $output/httpscreenshot
-WORKDIR $output
-
-ENTRYPOINT httpscreenshot -i $infile -b chrome -p -w 40  -a -vH
+ENTRYPOINT mkdir -p $output/httpscreenshot && cd $output/httpscreenshot && httpscreenshot -i $infile -b chrome -p -w 40  -a -vH
diff --git a/src/robot_api/docker_buildfiles/Dockerfile.Nmap.Screenshot.tmp b/src/robot_api/docker_buildfiles/Dockerfile.Nmap.Screenshot.tmp
@@ -14,7 +14,7 @@ RUN update-ca-certificates
 
 RUN if [ -n $dns ]; \
     then echo "nameserver $dns" >> /etc/resolv.conf; fi;\
-    git clone https://github.com/Sandarmann/nmap-screenshot.git \
+    git clone https://github.com/CrimsonK1ng/nmap-screenshot.git \
     && cp nmap-screenshot/http-screenshot.nse /usr/share/nmap/scripts/http-screenshot.nse \
     && nmap --script-updatedb 
 
@@ -23,7 +23,6 @@ RUN if [ -n $dns ]; \
     wget -O wkhtml.deb https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \
     && dpkg -i  wkhtml.deb
 
-RUN mkdir $output/nmapscreen
 WORKDIR $output
 
-ENTRYPOINT nmap --script http-screenshot --script-args tool=$tool -iL $infile -p "80,8080,443,8888"
+ENTRYPOINT mkdir $output/nmapscreen && cd $output/nmapscreen && nmap --script http-screenshot --script-args tool=$tool -iL $infile -p "80,8080,443,8888"
