Skip to content

Commit c750666

Browse files
joshsplunksjoshsplunks
committed
added ESCU content
1 parent 30deee3 commit c750666

11 files changed

+2883
-1
lines changed

‎dockerfiles/Dockerfile.5.1.0.debian.escu

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
ARG BASE_IMAGE
2+
FROM ${BASE_IMAGE}
3+
ARG TAG
4+
ARG REQUIREMENTS_PYTHON_BASE
5+
ARG REQUIREMENTS_PYTHON_SPECIFIC
6+
7+
# setup environment
8+
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
9+
ENV PATH /dltk/.local/bin:/dltk/.local/lib/python3.9/site-packages/:$PATH
10+
ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC
11+
12+
# curl not installed in some images :(
13+
RUN apt-get update
14+
RUN apt-get install curl -y
15+
16+
# install nodejs
17+
RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash -
18+
RUN apt-get update --fix-missing && apt-get install -y wget bzip2 git ca-certificates nodejs build-essential
19+
20+
# update everything
21+
RUN apt-get update && apt-get upgrade -y
22+
23+
# configure file system
24+
WORKDIR /srv
25+
RUN mkdir /dltk
26+
27+
# setup python+pip
28+
RUN apt-get install -y python3 python3-pip
29+
30+
# install base python requirements
31+
COPY ./requirements_files/${REQUIREMENTS_PYTHON_BASE} /dltk/${REQUIREMENTS_PYTHON_BASE}
32+
RUN pip3 install --no-cache-dir --upgrade -r /dltk/${REQUIREMENTS_PYTHON_BASE}
33+
34+
# install specific python requirements
35+
COPY ./requirements_files/${REQUIREMENTS_PYTHON_SPECIFIC} /dltk/${REQUIREMENTS_PYTHON_SPECIFIC}
36+
RUN pip3 install --no-cache-dir --upgrade -r /dltk/${REQUIREMENTS_PYTHON_SPECIFIC}
37+
38+
# configure spacy if it was installed
39+
RUN if pip3 freeze | grep -q spacy;\
40+
then echo 'Spacy is installed, downloading language file...';\
41+
python3 -m spacy download en_core_web_sm;\
42+
else echo 'Spacy is not installed'; fi
43+
44+
# creating new self signed certs
45+
RUN openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout dltk.key -out dltk.pem -subj="/CN=dsdl"
46+
RUN mkdir /dltk/.jupyter/; mv dltk.key /dltk/.jupyter/dltk.key; mv dltk.pem /dltk/.jupyter/dltk.pem
47+
48+
# Copy bootstrap entry point script
49+
COPY ./bootstrap_scripts/bootstrap_fast.sh /dltk/
50+
COPY app /dltk/app
51+
COPY notebooks /dltk/notebooks
52+
53+
# Install local DSDL supporting functions
54+
RUN mkdir /dltk/packages
55+
COPY package-dsdlsupport/dist/dsdlsupport-1.0.0.tar.gz /dltk/packages/dsdlsupport-1.0.0.tar.gz
56+
RUN pip3 install /dltk/packages/dsdlsupport-1.0.0.tar.gz
57+
58+
# Get ESCU DSDL Content
59+
RUN wget https://seal.splunkresearch.com/pretrained_dga_model_dsdl.tar.gz
60+
RUN wget https://seal.splunkresearch.com/detect_suspicious_dns_txt_records_using_pretrained_model_in_dsdl.tar.gz
61+
RUN wget https://seal.splunkresearch.com/detect_suspicious_processnames_using_pretrained_model_in_dsdl.tar.gz
62+
RUN wget https://seal.splunkresearch.com/detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.tar.gz
63+
64+
# Extract ESCU Content
65+
RUN tar -xf pretrained_dga_model_dsdl.tar.gz -C app/model/data
66+
RUN tar -xf detect_suspicious_dns_txt_records_using_pretrained_model_in_dsdl.tar.gz -C app/model/data
67+
RUN tar -xf detect_suspicious_processnames_using_pretrained_model_in_dsdl.tar.gz -C app/model/data
68+
RUN tar -xf detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.tar.gz -C app/model/data
69+
70+
# Copy jupyter config
71+
COPY config/jupyter_notebook_config.py /dltk/.jupyter/jupyter_notebook_config.py
72+
73+
# Since JupyterLab 3 jupyter server config needs to be set
74+
COPY config/jupyter_server_config.py /dltk/.jupyter/jupyter_server_config.py
75+
76+
# Copy jupyter notebook conversion template to export python module
77+
COPY config/jupyter_notebook_template.tpl /dltk/.jupyter/jupyter_notebook_conversion.tpl
78+
COPY config/null.tpl /dltk/.jupyter/null.tpl
79+
80+
# Handle user rights
81+
RUN chgrp -R 0 /dltk && \
82+
chmod -R g=u /dltk
83+
RUN chgrp -R 0 /srv && \
84+
chmod -R g=u /srv
85+
RUN chmod g+w /etc/passwd
86+
USER 1001
87+
88+
# Expose container port 5000 (MLTK Container Service) and 8888 (Notebook)
89+
EXPOSE 5000 8888
90+
# Define bootstrap as entry point to start container
91+
ENTRYPOINT ["/dltk/bootstrap_fast.sh"]

‎notebooks/data/detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"options": {"params": {"mode": "stage", "algo": "detect_dns_data_exfiltration_using_pretrained_model_in_dsdl"}, "args": ["is_exfiltration", "src", "query", "rank"], "target_variable": ["is_exfiltration"], "feature_variables": ["src", "query", "rank"], "model_name": "detect_dns_data_exfiltration_using_pretrained_model_in_dsdl", "algo_name": "MLTKContainer", "mlspl_limits": {"handle_new_cat": "default", "max_distinct_cat_values": "100", "max_distinct_cat_values_for_classifiers": "100", "max_distinct_cat_values_for_scoring": "100", "max_fit_time": "600", "max_inputs": "100000", "max_memory_usage_mb": "4000", "max_model_size_mb": "30", "max_score_time": "600", "use_sampling": "true"}, "kfold_cv": null}, "feature_variables": ["src", "query", "rank"], "target_variables": ["is_exfiltration"]}

‎notebooks/data/detect_suspicious_dns_txt_records_using_pretrained_model_in_dsdl.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"options": {"params": {"mode": "stage", "algo": "detect_suspicious_dns_txt_records_using_pretrained_model_in_dsdl"}, "args": ["is_unknown", "text"], "target_variable": ["is_unknown"], "feature_variables": ["text"], "model_name": "detect_suspicious_dns_txt_records_using_pretrained_model_in_dsdl", "algo_name": "MLTKContainer", "mlspl_limits": {"handle_new_cat": "default", "max_distinct_cat_values": "100", "max_distinct_cat_values_for_classifiers": "100", "max_distinct_cat_values_for_scoring": "100", "max_fit_time": "600", "max_inputs": "100000", "max_memory_usage_mb": "4000", "max_model_size_mb": "30", "max_score_time": "600", "use_sampling": "true"}, "kfold_cv": null}, "feature_variables": ["text"], "target_variables": ["is_unknown"]}

‎notebooks/data/detect_suspicious_processnames_using_pretrained_model_in_dsdl.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"options": {"params": {"mode": "stage", "algo": "detect_suspicious_processnames_using_pretrained_model_in_dsdl"}, "args": ["label", "text"], "target_variable": ["label"], "feature_variables": ["text"], "model_name": "detect_suspicious_processnames_using_pretrained_model_in_dsdl", "algo_name": "MLTKContainer", "mlspl_limits": {"handle_new_cat": "default", "max_distinct_cat_values": "100", "max_distinct_cat_values_for_classifiers": "100", "max_distinct_cat_values_for_scoring": "100", "max_fit_time": "600", "max_inputs": "100000", "max_memory_usage_mb": "4000", "max_model_size_mb": "30", "max_score_time": "600", "use_sampling": "true"}, "kfold_cv": null}, "feature_variables": ["text"], "target_variables": ["label"]}

‎notebooks/data/pretrained_dga_model_dsdl.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"options": {"params": {"mode": "stage", "algo": "pretrained_dga_model_dsdl"}, "args": ["is_dga", "domain"], "target_variable": ["is_dga"], "feature_variables": ["domain"], "model_name": "pretrained_dga_model_dsdl", "algo_name": "MLTKContainer", "mlspl_limits": {"handle_new_cat": "default", "max_distinct_cat_values": "100", "max_distinct_cat_values_for_classifiers": "100", "max_distinct_cat_values_for_scoring": "100", "max_fit_time": "600", "max_inputs": "100000", "max_memory_usage_mb": "4000", "max_model_size_mb": "30", "max_score_time": "600", "use_sampling": "true"}, "kfold_cv": null}, "feature_variables": ["domain"], "target_variables": ["is_dga"]}

0 commit comments

Comments
 (0)