Skip to content
This repository was archived by the owner on Aug 1, 2024. It is now read-only.

Commit af2a6ef

Browse files
author
Aleksandar Straumann
committed
Generate tarfiles for build. Create cert directory for individuals to use when behind proxy/firewall
1 parent 7cf0080 commit af2a6ef

22 files changed

+124
-44
lines changed

‎README.md

+5
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,12 @@ optional arguments:
143143
--dbfile DBFILE Specify what db file to use for saving data too
144144
```
145145

146+
## Certs
147+
148+
Running this behind a proxy was a pain. To make this less painful we create a certs directory under the **$HOME/.drrobot/*** where you can add your crt files. As part of the dockerfile build process we now generate tarfiles with the certificates so that applications, such as Amass, can run.
149+
146150
## Configuration
151+
147152
Dr.ROBOT is built in a modular fashion, making it easy to add new tools. You have three options for adding a new tool to Dr.ROBOT:
148153

149154
#### Important: To make sure no issues come from adding your own tool, make sure the key used to identify a json item, the name, and docker_name are all unique.

‎src/robot_api/api/dockerize.py

+18-6
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,19 @@
1212
status (str): If running or not
1313
name (str): name of docker image
1414
"""
15-
from os.path import isfile
15+
from os.path import isfile, basename
1616
from string import Template
1717
import logging
1818
import time
19+
import tarfile
1920
import json
2021
import multiprocessing
2122
from tqdm import tqdm
2223
import docker
2324
from docker.errors import APIError, BuildError, ContainerError, ImageNotFound, NotFound
2425

26+
from robot_api.parse import join_abs
27+
2528
LOG = logging.getLogger(__name__)
2629

2730

@@ -53,7 +56,6 @@ def __init__(self, **kwargs):
5356
self.status = None
5457
self.error = False
5558
self.done_building = False
56-
5759
self.OUTPUT_DIR = kwargs.get('output_dir', None)
5860

5961
def _print(self, msg):
@@ -74,7 +76,7 @@ def kill(self):
7476
"Error when trying to send kill signal to docker container.")
7577
LOG.exception("Killing container")
7678

77-
def _init_config(self):
79+
def gen_config(self):
7880
"""Creates active configuration from template
7981
8082
Raises:
@@ -96,6 +98,13 @@ def _init_config(self):
9698
for k, v
9799
in self._docker_options.items()
98100
}))
101+
def gen_tarfile(self):
102+
tarname = join_abs(self._docker_options['tarfiles'], basename(self._active_config_path) + ".tar.gz")
103+
with tarfile.open(name=tarname, mode="w:gz") as tar:
104+
tar.add(self._active_config_path, "Dockerfile")
105+
tar.add(self._docker_options['certs'], 'certs')
106+
return tarname
107+
99108

100109
def build(self):
101110
"""
@@ -107,16 +116,19 @@ def build(self):
107116
108117
"""
109118
try:
110-
self._init_config()
119+
self.gen_config()
120+
tarfile = self.gen_tarfile()
111121
self._print(f"""Built with options:
112122
-f {self._active_config_path}
113123
-t {self._docker_options['docker_name']}:{self._docker_options['docker_name']}
114124
--rm
115125
--network {self.network_mode}
116126
""")
117-
with open(self._active_config_path, 'rb') as _file:
118-
self.image = self.client.images.build(fileobj=_file,
127+
with open(tarfile, 'rb') as _file:
128+
self.image = self.client.images.build(fileobj=_file,
119129
tag=f"{self._docker_options['docker_name']}:{self._docker_options['docker_name']}",
130+
custom_context=True,
131+
encoding='gzip',
120132
rm=True,
121133
network_mode=self.network_mode,
122134
use_config_proxy=True)

‎src/robot_api/config.py

+12-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,18 @@ def generate_configs():
2626
if not path.exists(CONFIG_DIR):
2727
makedirs(CONFIG_DIR)
2828

29+
if not path.exists(path.join(CONFIG_DIR, "docker_buildfiles")):
30+
makedirs(path.join(CONFIG_DIR, "docker_buildfiles"))
31+
32+
if not path.exists(path.join(CONFIG_DIR, "docker_active")):
33+
makedirs(path.join(CONFIG_DIR, "docker_active"))
34+
35+
if not path.exists(path.join(CONFIG_DIR, "tarfiles")):
36+
makedirs(path.join(CONFIG_DIR, "tarfiles"))
37+
38+
if not path.exists(path.join(CONFIG_DIR, "certs")):
39+
makedirs(path.join(CONFIG_DIR, "certs"))
40+
2941
if not path.isfile(path.join(CONFIG_DIR, "config.json")):
3042
with open(path.join(CONFIG_DIR, "config.json"), 'wb') as _file:
3143
config = pkg_resources.resource_string(__name__,
@@ -38,8 +50,6 @@ def generate_configs():
3850
'configs/ansible_inventory')
3951
_file.write(config)
4052

41-
if not path.exists(path.join(CONFIG_DIR, "docker_buildfiles")):
42-
makedirs(path.join(CONFIG_DIR, "docker_buildfiles"))
4353

4454
for _file in pkg_resources.resource_listdir(__name__, "docker_buildfiles"):
4555
if not path.isfile(path.join(CONFIG_DIR, "docker_buildfiles", _file)):

‎src/robot_api/configs/default_config.json

+18-17
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@
7777
"network_mode": "host",
7878
"docker_name": "aqua",
7979
"default_conf": "docker_buildfiles/Dockerfile.Aquatone.tmp",
80-
"active_conf": "docker_buildfiles/Dockerfile.Aquatone",
80+
"active_conf": "docker_active/Dockerfile.Aquatone",
8181
"description": "AQUATONE is a set of tools for performing reconnaissance on domain names",
8282
"src": "https://github.com/michenriksen/aquatone",
8383
"output": "/aqua",
@@ -89,7 +89,7 @@
8989
"network_mode": "host",
9090
"docker_name": "sub",
9191
"default_conf": "docker_buildfiles/Dockerfile.Sublist3r.tmp",
92-
"active_conf": "docker_buildfiles/Dockerfile.Sublist3r",
92+
"active_conf": "docker_active/Dockerfile.Sublist3r",
9393
"description": "Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT",
9494
"src": "https://github.com/aboul3la/Sublist3r",
9595
"output": "/root/sublist3r",
@@ -101,7 +101,7 @@
101101
"network_mode": "host",
102102
"docker_name": "turbo",
103103
"default_conf": "docker_buildfiles/Dockerfile.Turbolist3r.tmp",
104-
"active_conf": "docker_buildfiles/Dockerfile.Turbolist3r",
104+
"active_conf": "docker_active/Dockerfile.Turbolist3r",
105105
"description": "Turbolist3r is a fork of the sublist3r subdomain discovery tool",
106106
"src": "https://github.com/fleetcaptain/Turbolist3r.git",
107107
"output": "/root/turbooutput",
@@ -113,7 +113,7 @@
113113
"network_mode": "host",
114114
"docker_name" : "brute",
115115
"default_conf": "docker_buildfiles/Dockerfile.Subbrute.tmp",
116-
"active_conf": "docker_buildfiles/Dockerfile.Subbrute",
116+
"active_conf": "docker_active/Dockerfile.Subbrute",
117117
"description": "SubBrute is a community driven project with the goal of creating the fastest, and most accurate subdomain enumeration tool.",
118118
"src": "https://github.com/TheRook/subbrute.git",
119119
"output": "/root/brute",
@@ -125,7 +125,7 @@
125125
"docker_name" : "sfinder",
126126
"network_mode": "host",
127127
"default_conf": "docker_buildfiles/Dockerfile.Subfinder.tmp",
128-
"active_conf": "docker_buildfiles/Dockerfile.Subfinder",
128+
"active_conf": "docker_active/Dockerfile.Subfinder",
129129
"description": "SubFinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources",
130130
"src": "https://github.com/subfinder/subfinder",
131131
"output": "/root/subfinder",
@@ -137,7 +137,7 @@
137137
"network_mode": "host",
138138
"docker_name" : "knock",
139139
"default_conf": "docker_buildfiles/Dockerfile.Knock.tmp",
140-
"active_conf": "docker_buildfiles/Dockerfile.Knock",
140+
"active_conf": "docker_active/Dockerfile.Knock",
141141
"description": "Knockpy is a python tool designed to enumerate subdomains on a target domain through a wordlist",
142142
"src": "https://github.com/guelfoweb/knock",
143143
"output": "/root/knock",
@@ -150,9 +150,10 @@
150150
"network_mode": "host",
151151
"docker_name" : "amass",
152152
"default_conf": "docker_buildfiles/Dockerfile.Amass.tmp",
153-
"active_conf": "docker_buildfiles/Dockerfile.Amass",
153+
"active_conf": "docker_active/Dockerfile.Amass",
154154
"description": "The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping.",
155155
"src": "https://github.com/OWASP/Amass",
156+
"resolvers": "",
156157
"output": "/root/amass",
157158
"output_folder": "amass"
158159
},
@@ -162,7 +163,7 @@
162163
"network_mode": "host",
163164
"docker_name" : "recon",
164165
"default_conf": "docker_buildfiles/Dockerfile.Reconng.tmp",
165-
"active_conf": "docker_buildfiles/Dockerfile.Reconng",
166+
"active_conf": "docker_active/Dockerfile.Reconng",
166167
"description": "Recon-ng is a full-featured Web Reconnaissance framework written in Python. DrRobot utilizes several of the recon/hosts-domain modules in this framework.",
167168
"src": "https://bitbucket.org/LaNMaSteR53/recon-ng",
168169
"output": "/tmp/output",
@@ -174,7 +175,7 @@
174175
"network_mode": "host",
175176
"docker_name" : "altdns",
176177
"default_conf": "docker_buildfiles/Dockerfile.Altdns.tmp",
177-
"active_conf": "docker_buildfiles/Dockerfile.Altdns",
178+
"active_conf": "docker_active/Dockerfile.Altdns",
178179
"description": "Generates permutations, alterations and mutations of subdomains and then resolves them",
179180
"src": "https://github.com/infosec-au/altdns",
180181
"infile" : "/root/altdns/aggregated/aggregated_hostnames.txt",
@@ -187,7 +188,7 @@
187188
"network_mode": "host",
188189
"docker_name" : "anubis",
189190
"default_conf": "docker_buildfiles/Dockerfile.Anubis.tmp",
190-
"active_conf": "docker_buildfiles/Dockerfile.Anubis",
191+
"active_conf": "docker_active/Dockerfile.Anubis",
191192
"description": "Anubis is a subdomain enumeration and information gathering tool.",
192193
"src": "https://github.com/jonluca/Anubis",
193194
"output": "/root/anubis",
@@ -199,7 +200,7 @@
199200
"network_mode": "host",
200201
"docker_name" : "ctexpo",
201202
"default_conf": "docker_buildfiles/Dockerfile.CT.tmp",
202-
"active_conf": "docker_buildfiles/Dockerfile.CT",
203+
"active_conf": "docker_active/Dockerfile.CT",
203204
"description": "An OSINT tool that discovers sub-domains by searching Certificate Transparency ",
204205
"src": "https://github.com/chris408/ct-exposer",
205206
"output": "/root/ct",
@@ -211,7 +212,7 @@
211212
"network_mode": "host",
212213
"docker_name" : "ctfr",
213214
"default_conf": "docker_buildfiles/Dockerfile.CTFR.tmp",
214-
"active_conf": "docker_buildfiles/Dockerfile.CTFR",
215+
"active_conf": "docker_active/Dockerfile.CTFR",
215216
"description": "Abusing Certificate Transparency logs for getting HTTPS websites subdomains.",
216217
"src": "https://github.com/UnaPibaGeek/ctfr",
217218
"output": "/root/ctfr",
@@ -222,7 +223,7 @@
222223
"network_mode": "host",
223224
"docker_name" : "pdlist",
224225
"default_conf": "docker_buildfiles/Dockerfile.PDList.tmp",
225-
"active_conf": "docker_buildfiles/Dockerfile.PDList",
226+
"active_conf": "docker_active/Dockerfile.PDList",
226227
"description": "pdlist is a passive subdomain finder written in python3",
227228
"src": "https://github.com/gnebbia/pdlist",
228229
"output": "/root/pdlistoutput",
@@ -238,7 +239,7 @@
238239
"network_mode": "host",
239240
"docker_name" : "httpscreen",
240241
"default_conf" : "docker_buildfiles/Dockerfile.HTTPScreenshot.tmp",
241-
"active_conf" : "docker_buildfiles/Dockerfile.HTTPScreenshot",
242+
"active_conf" : "docker_active/Dockerfile.HTTPScreenshot",
242243
"ansible_arguments" : {
243244
"config" : "$config/httpscreenshot_play.yml",
244245
"flags": "-e '$extra' -i configs/ansible_inventory",
@@ -261,7 +262,7 @@
261262
"mode" : "DOCKER",
262263
"network_mode": "host",
263264
"default_conf" : "docker_buildfiles/Dockerfile.Eyewitness.tmp",
264-
"active_conf" : "docker_buildfiles/Dockerfile.Eyewitness",
265+
"active_conf" : "docker_active/Dockerfile.Eyewitness",
265266
"ansible_arguments" : {
266267
"config" : "$config/eyewitness_play.yml",
267268
"flags": "-e '$extra' -i configs/ansible_inventory",
@@ -284,7 +285,7 @@
284285
"mode" : "DOCKER",
285286
"network_mode": "host",
286287
"default_conf" : "docker_buildfiles/Dockerfile.Nmap.Screenshot.tmp",
287-
"active_conf" : "docker_buildfiles/Dockerfile.Nmap.Screenshot",
288+
"active_conf" : "docker_active/Dockerfile.Nmap.Screenshot",
288289
"description" : "Post enumeration tool for screen grabbing websites. (Chrome is not installed in the dockerfile due. Options are chromium-browser/firefox/wkhtmltoimage)",
289290
"output" : "/tmp/output",
290291
"infile" : "/tmp/output/aggregated/aggregated_hostnames.txt",
@@ -297,7 +298,7 @@
297298
"mode" : "DOCKER",
298299
"network_mode": "host",
299300
"default_conf" : "docker_buildfiles/Dockerfile.Webscreenshot.Screenshot.tmp",
300-
"active_conf" : "docker_buildfiles/Dockerfile.Webscreenshot.Screenshot",
301+
"active_conf" : "docker_active/Dockerfile.Webscreenshot.Screenshot",
301302
"description" : "Post enumeration tool for screen grabbing websites. (Chrome is not installed in the dockerfile due. Options are chromium-browser/firefox/wkhtmltoimage)",
302303
"output" : "/tmp/output",
303304
"infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt",

‎src/robot_api/docker_buildfiles/Dockerfile.Altdns.tmp

+4-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ ENV DNS $dns
88
RUN if [ -n $dns ]; \
99
then echo "nameserver $dns" >> /etc/resolv.conf;\
1010
fi;\
11-
apt-get install git
11+
apt-get install git ca-certificates
12+
13+
ADD certs/ /usr/local/share/ca-certificates/
14+
RUN update-ca-certificates
1215

1316
RUN if [ -n $dns ]; \
1417
then echo "nameserver $dns" >> /etc/resolv.conf;\

‎src/robot_api/docker_buildfiles/Dockerfile.Amass.tmp

+6-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ FROM golang:1.13.0-alpine3.10 as build
33
RUN if [ -n "$dns" ]; \
44
then echo "nameserver $dns" >> /etc/resolv.conf;\
55
fi;\
6-
apk --no-cache add git;
6+
apk --no-cache add git ca-certificates;
7+
8+
ADD certs/ /usr/local/share/ca-certificates/
9+
RUN update-ca-certificates
710

811
RUN if [ -n "$dns" ]; \
912
then echo "nameserver $dns" >> /etc/resolv.conf;\
@@ -27,12 +30,12 @@ RUN if [ -n "$dns" ]; \
2730
apk --no-cache add ca-certificates
2831

2932
COPY --from=build /go/bin/amass /bin/amass
30-
COPY --from=build /go/src/github.com/OWASP/Amass/wordlists/ /wordlists/
33+
COPY --from=build /go/src/github.com/OWASP/Amass/examples/wordlists/ /wordlists/
3134

3235
ENV http_proxy $proxy
3336
ENV https_proxy $proxy
3437
ENV HOME /
3538

3639
RUN mkdir -p $output
3740

38-
ENTRYPOINT /bin/amass enum --passive -d "$target" -o $output/amass.txt
41+
ENTRYPOINT /bin/amass enum --passive -d "$target" -o $output/amass.txt

‎src/robot_api/docker_buildfiles/Dockerfile.Anubis.tmp

+3
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,12 @@ RUN if [ -n $dns ]; \
1010
libssl-dev \
1111
libffi-dev \
1212
python-dev \
13+
ca-certificates \
1314
git \
1415
&& rm -rf /var/lib/apt/lists/*
1516

17+
ADD certs/ /usr/local/share/ca-certificates/
18+
RUN update-ca-certificates
1619

1720
RUN if [ -n $dns ]; \
1821
then echo "nameserver $dns" >> /etc/resolv.conf; fi;\

‎src/robot_api/docker_buildfiles/Dockerfile.CT.tmp

+4-1
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@ RUN if [ -n $dns ]; \
77
then echo "nameserver $dns" >> /etc/resolv.conf; fi;\
88
apt-get update && apt-get install -y --no-install-recommends \
99
python-dev \
10-
git
10+
git \
11+
ca-certiticates
1112

13+
ADD certs/ /usr/local/share/ca-certificates/
14+
RUN update-ca-certificates
1215

1316
RUN if [ -n $dns ]; \
1417
then echo "nameserver $dns" >> /etc/resolv.conf; fi;\

‎src/robot_api/docker_buildfiles/Dockerfile.CTFR.tmp

+4-1
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@ RUN if [ -n $dns ]; \
77
then echo "nameserver $dns" >> /etc/resolv.conf; fi;\
88
apt-get update && apt-get install -y --no-install-recommends \
99
python-dev \
10-
git
10+
git \
11+
ca-certificates
1112

13+
ADD certs/ /usr/local/share/ca-certificates/
14+
RUN update-ca-certificates
1215

1316
RUN if [ -n $dns ]; \
1417
then echo "nameserver $dns" >> /etc/resolv.conf; fi;\

‎src/robot_api/docker_buildfiles/Dockerfile.Eyewitness.tmp

+5-1
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,13 @@ RUN if [ -n $dns ]; \
99
then echo "nameserver $dns" >> /etc/resolv.conf;\
1010
fi;\
1111
apt-get update && \
12-
apt-get install -y git wget && \
12+
apt-get install -y git wget ca-certificates && \
1313
rm -rf /var/lib/apt/lists/*
1414

15+
16+
ADD certs/ /usr/local/share/ca-certificates/
17+
RUN update-ca-certificates
18+
1519
RUN if [ -n $dns ]; \
1620
then echo "nameserver $dns" >> /etc/resolv.conf;\
1721
fi;\

‎src/robot_api/docker_buildfiles/Dockerfile.HTTPScreenshot.tmp

+5-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ RUN if [ -n $dns ]; \
77
then echo "nameserver $dns" >> /etc/resolv.conf;\
88
fi;\
99
apt-get update \
10-
&& apt-get install -y wget git python-dev python-pip libfontconfig unzip firefox
10+
&& apt-get install -y wget git python-dev python-pip libfontconfig unzip firefox ca-certificates
11+
12+
13+
ADD certs/ /usr/local/share/ca-certificates/
14+
RUN update-ca-certificates
1115

1216
RUN if [ -n $dns ]; \
1317
then echo "nameserver $dns" >> /etc/resolv.conf;\

‎src/robot_api/docker_buildfiles/Dockerfile.Knock.tmp

+4-1
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@ RUN if [ -n $dns ]; \
88
then echo "nameserver $dns" >> /etc/resolv.conf;\
99
fi;\
1010
apt-get update && \
11-
apt-get install -y git python-dnspython && \
11+
apt-get install -y git python-dnspython ca-certificates && \
1212
rm -rf /var/lib/apt/lists/*
1313

14+
ADD certs/ /usr/local/share/ca-certificates/
15+
RUN update-ca-certificates
16+
1417
RUN if [ -n $dns ]; \
1518
then echo "nameserver $dns" >> /etc/resolv.conf;\
1619
fi;\

0 commit comments

Comments
 (0)