From 0b67bc950313f5e2dea3d9f573838c5d7b7fcb15 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Wed, 27 Jul 2022 02:13:02 +0000 Subject: [PATCH 01/15] Documentation updates --- README.md | 4 ++-- docs/subpages/build_steps.rst | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8a83a19..b29583e 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Documentation on the project can be found project's [ReadTheDocs](https://marvel ## Support: -* Windows 10 + +* Windows 10+ * Windows Server 2016 + -* Ubuntu 18 + +* Ubuntu 22.04+ * MacOS Catalina # Authors: diff --git a/docs/subpages/build_steps.rst b/docs/subpages/build_steps.rst index c70cbeb..a92474d 100644 --- a/docs/subpages/build_steps.rst +++ b/docs/subpages/build_steps.rst @@ -65,26 +65,28 @@ Logging: Steps to get logging set up: **************************** -If you plan on using Splunk/Jupyter Notebooks/OSQuery/Kolide - install the required scripts on the Ubuntu box first before setting up logging on endpoints. +If you plan on using Splunk/Jupyter Notebooks/OSQuery/Kolide - install the required scripts on the Ubuntu box first before setting up logging on the endpoints. On Ubuntu box: -############### +************** -1. On the Ubuntu machine download the Marvel-Lab repository. -2. Go into ``Marvel-Lab\Logging\splunk`` and run ``splunk_logging.sh``. -3. Go to the hosts and AFTER Kolide has been set up from the ``On Windows Workstation and DC`` instructions, run ``fleet-pack.sh``. +1. Install Docker by following these steps, assuming you’re on Ubuntu: https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository. +2. Install Docker Compose by following the documentation here: https://docs.docker.com/compose/install/. +3. On the Ubuntu machine download the Marvel-Lab repository. +4. Go into ``Marvel-Lab\Logging\splunk`` and run ``setup_logging.sh``. + +**Note**: Only Ubuntu 22.04+ is supported for this script. -**Note**: Only Ubuntu 18+ is supported for this script. On Windows Workstation and DC: -############################## +****************************** -**In order to recieve logs in Splunk, the ``splunk_logging.sh`` script must have succeeded on the Logger box (Ubuntu).** +**Note**: In order to recieve logs in Splunk, the ``setup_logging.sh`` script must have succeeded on the Logger box (Ubuntu). 1. Download the Marvel-Lab repository in the ``C:\`` directory. (If you downloaded the .zip of the repo, move the child folder to the - C: directory and rename to ``Marvel-Lab``). + C:\ directory and rename to ``Marvel-Lab``). 2. Go to KolideIP:8443, set up Username/Password. 3. Set Organization Name to ``Marvel Lab``. You don't have to do the URL. When it shows you the fleet web address, press Submit, then Finish. 4. Go into ``Marvel-Lab\Set-Logging`` and run ``Set-Logging.ps1``. @@ -97,7 +99,7 @@ the ``C:\ArchivedFiles`` folder. On MacOS Workstation: -###################### +********************* 1. Run ``logging.sh`` 2. During installation there will be some prompt that will need to be filled in when accepting the Splunk License. Exact steps are below: - Press Enter From 9e8c34202ee9db0097bef47e85d8e03a119ce1d0 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Wed, 27 Jul 2022 04:27:09 +0000 Subject: [PATCH 02/15] Begin restructuring the setup script --- Logging/setup_logging.sh | 89 +++++++++++++++++++++++++++ Logging/splunk/splunk_logger.sh | 104 -------------------------------- 2 files changed, 89 insertions(+), 104 deletions(-) create mode 100755 Logging/setup_logging.sh delete mode 100755 Logging/splunk/splunk_logger.sh diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh new file mode 100755 index 0000000..ee7aa4a --- /dev/null +++ b/Logging/setup_logging.sh @@ -0,0 +1,89 @@ +#!/bin/bash +#Authors: Jonathan Johnson & Ben Shell +#References: https://stackoverflow.com/ && https://github.com/target/huntlib.git + +SETUP_SPLUNK="False" +SETUP_ELASTIC="False" +SETUP_ZEEK="False" + +# Checking to see if script is running as root +if [[ $EUID -ne 0 ]]; then + echo -e "\x1B[01;31m[X] Script Must Be Run As ROOT\x1B[0m" + exit 1 +fi + +echo -e "\x1B[01;34m[*] Setting timezone to UTC...\x1B[0m" +timedatectl set-timezone UTC + +# Checking Docker +echo -e "\x1B[01;34m[*] Checking to see if Docker is installed...\x1B[0m" + +if [[ $(which docker) && $(docker compose version) ]]; then + echo -e "\x1B[01;32m[*] Docker Compose is installed\x1B[0m" + else + echo -e "\x1B[01;31m[*] Docker was not found. See the Read the Docs installation documentation (https://marvel-lab.readthedocs.io/en/latest/subpages/build_steps.html#logging) \x1B[0m" + exit 0 +fi + +# Enabling docker service: +echo -e "\x1B[01;34m[*] Enabling Docker Service...\x1B[0m" +systemctl enable docker.service + +# Starting containers +echo -e "\x1B[01;34m[*] Starting containers\x1B[0m" +docker compose up -d + +# Zeek +if [ "$SETUP_ZEEK" = "True" ]; then + read -r -p "Zeek needs a network interface to monitor. Would you like to print out your interfaces to see which one to monitor? [y/N] " response + + if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]] + then + if hash ifconfig 2>/dev/null; then + ifconfig + else + ip address + fi + else + echo -e "\x1B[01;34m[*] Moving on...\x1B[0m" + fi + + read -p 'Input the network interface you would like Zeek to monitor and press [ENTER]: ' Interface + + echo -e "\x1B[01;34m[*] Creating Zeek:\x1B[0m" + docker compose up -d -f ./Config/zeek/zeek-compose.yml + +# Splunk +if [ "$SETUP_SPLUNK" = "True" ]; then + # Define healthcheck function for splunk + splunk_healthcheck(){ + echo -e "\x1B[01;32m[*] Waiting for splunk...\x1B[0m" + SPLUNK_STATUS="" + while [[ "$SPLUNK_STATUS" != "\"healthy"\" ]] + do + sleep 3 + SPLUNK_STATUS=$(docker inspect --format='{{json .State.Health.Status}}' splunk) + done + } + + # Wait for splunk to finish installing + splunk_healthcheck + + # The 'docker cp' commands are needed after Splunk install, otherwise our custom config would be overwritten + docker cp splunk/inputs.conf splunk:/opt/splunk/etc/system/local/inputs.conf + docker cp splunk/indexes.conf splunk:/opt/splunk/etc/system/local/indexes.conf + echo -e "\x1B[01;32m[*] Restarting splunk to apply inputs.conf and indexes.conf\x1B[0m" + docker restart splunk + splunk_healthcheck + + # Checking Jupyter Notebooks + echo -e "\x1B[01;34m[*] Checking Jupyter Notebooks...\x1B[0m" + sleep 10 + token="$(docker exec -it jupyter-notebooks sh -c 'jupyter notebook list' | grep token | sed 's/.*token=\([^ ]*\).*/\1/')" + + echo -e "\x1B[01;32m[*] Access Splunk at https://$Host_IP/splunk/ ; Credentials - admin:Changeme1! (unless you changed them in the DockerFile)\x1B[0m" + echo -e "\x1B[01;32m[*] Access Jupyter Notebook at: http://$Host_IP:8888\x1B[0m" + echo -e "\x1B[01;32m[*] Jupyter Notebook token is $token\x1B[0m" + +echo -e "\x1B[01;32m[*] Access Portainer at https://$Host_IP/portainer/ \x1B[0m" + diff --git a/Logging/splunk/splunk_logger.sh b/Logging/splunk/splunk_logger.sh deleted file mode 100755 index 145b03e..0000000 --- a/Logging/splunk/splunk_logger.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -#Authors: Jonathan Johnson & Ben Shell -#References: https://stackoverflow.com/ && https://github.com/target/huntlib.git - -# Checking to see if script is running as root -if [[ $EUID -ne 0 ]]; then - echo -e "\x1B[01;31m[X] Script Must Be Run As ROOT\x1B[0m" - exit 1 -fi - -echo -e "\x1B[01;34m[*] Setting timezone to UTC..\x1B[0m" - -timedatectl set-timezone UTC - -read -p 'Input your IP and press [ENTER]: ' Host_IP - -# Installing Docker Compose -echo -e "\x1B[01;34m[*] Checking to see if docker is installed\x1B[0m" - -if [[ $(which docker-compose) && $(docker-compose --version) ]]; then - echo -e "\x1B[01;32m[*] Docker Compose is installed\x1B[0m" - else - echo -e "\x1B[01;34m[*] Installing Docker Compose\x1B[0m" - apt-get install docker-compose -y -fi - -# Enabling docker service: -echo -e "\x1B[01;34m[*] Enabling Docker Service\x1B[0m" -systemctl enable docker.service - -# Pull quick-fleet -echo -e "\x1B[01;34m[*] Cloning/Pulling quick-fleet\x1B[0m" -if [ ! -d "quick-fleet" ]; then - git clone https://github.com/benjaminshell/quick-fleet.git -else - (cd quick-fleet; git pull) -fi - -# Zeek -read -r -p "Zeek needs a network interface to monitor, would you like to print out your interfaces to see which one to monitor? [y/N] " response - -if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]] -then - if hash ifconfig 2>/dev/null; then - ifconfig - else - ip address - fi -else - echo -e "\x1B[01;34mMoving on..\x1B[0m" -fi - -read -p 'Input the network interface you would like zeek to monitor and press [ENTER]: ' Interface -echo -e "\x1B[01;34m[*] Creating Zeek:\x1B[0m" -docker pull blacktop/zeek -if [ "$(docker ps -a -q -f name=zeek)" ]; then - docker stop zeek && docker rm zeek -fi -docker run -d --name zeek --restart always --cap-add=NET_RAW --net=host -v `pwd`/zeek/zeek-logs/:/pcap:rw -v `pwd`/zeek/__load__.zeek:/usr/local/zeek/share/zeek/base/bif/__load__.zeek blacktop/zeek -i $Interface -C - -# Starting containers -echo -e "\x1B[01;34m[*] Starting containers\x1B[0m" -docker-compose -f docker-compose.yml -f quick-fleet/docker-compose.yml up -d - -# Define healthcheck function for splunk -splunk_healthcheck(){ - echo -e "\x1B[01;32m[*] Waiting for splunk...\x1B[0m" - SPLUNK_STATUS="" - while [[ "$SPLUNK_STATUS" != "\"healthy"\" ]] - do - sleep 3 - SPLUNK_STATUS=$(docker inspect --format='{{json .State.Health.Status}}' splunk) - done -} - -# Wait for splunk to finish installing -splunk_healthcheck - -# The docker cp is needed after splunk install, otherwise our custom config would be overwritten -docker cp splunk/inputs.conf splunk:/opt/splunk/etc/system/local/inputs.conf -docker cp splunk/indexes.conf splunk:/opt/splunk/etc/system/local/indexes.conf -echo -e "\x1B[01;32m[*] Restarting splunk to apply inputs.conf and indexes.conf\x1B[0m" -docker restart splunk -splunk_healthcheck - -# To remove the containers -# docker-compose -f docker-compose.yml -f quick-fleet/docker-compose.yml down - -# Reset log files for quick-fleet -rm -rf quick-fleet/result.log/ -rm -rf quick-fleet/status.log/ -touch quick-fleet/result.log -touch quick-fleet/status.log - -# Checking Jupyter Notebooks -echo -e "\x1B[01;34m[*] Checking Jupyter Notebooks\x1B[0m" -sleep 10 -token="$(docker exec -it jupyter-notebooks sh -c 'jupyter notebook list' | grep token | sed 's/.*token=\([^ ]*\).*/\1/')" -echo -e "\x1B[01;32m[*] Access Portainer at https://$Host_IP/portainer/ \x1B[0m" -echo -e "\x1B[01;32m[*] Access Splunk at https://$Host_IP/splunk/ ; Credentials - admin:Changeme1! (unless you changed them in the DockerFile)\x1B[0m" -echo -e "\x1B[01;32m[*] Jupyter Notebook's IP is: http://$Host_IP:8888\x1B[0m" -echo -e "\x1B[01;32m[*] Jupyter Notebook token is $token\x1B[0m" -echo -e "\x1B[01;32m[*] Kolide Fleet's IP is: https://$Host_IP:8443\x1B[0m" - From 94717f7672a2e58d9955b69116d575692cc65ba0 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Wed, 27 Jul 2022 04:28:06 +0000 Subject: [PATCH 03/15] Move things around --- Logging/{splunk => }/.env | 8 +- Logging/{ => Config/Windows}/Set-Logging.ps1 | 0 .../{splunk => Config/Windows}/inputs.conf | 0 .../Windows}/research-sysmon-config.xml | 0 .../{splunk => Config}/splunk/indexes.conf | 0 Logging/{splunk => Config}/splunk/inputs.conf | 0 Logging/Config/splunk/splunk-compose.yml | 53 +++++++++ Logging/{splunk => Config}/splunk/web.conf | 0 Logging/{splunk => Config}/zeek/__load__.zeek | 0 Logging/Config/zeek/zeek-compose.yml | 13 +++ Logging/docker-compose.yml | 42 ++++++++ Logging/splunk/docker-compose.yml | 102 ------------------ Logging/splunk/fleet-pack.sh | 22 ---- Logging/splunk/thehive/application.conf | 49 --------- 14 files changed, 111 insertions(+), 178 deletions(-) rename Logging/{splunk => }/.env (70%) rename Logging/{ => Config/Windows}/Set-Logging.ps1 (100%) rename Logging/{splunk => Config/Windows}/inputs.conf (100%) rename Logging/{ => Config/Windows}/research-sysmon-config.xml (100%) rename Logging/{splunk => Config}/splunk/indexes.conf (100%) rename Logging/{splunk => Config}/splunk/inputs.conf (100%) create mode 100644 Logging/Config/splunk/splunk-compose.yml rename Logging/{splunk => Config}/splunk/web.conf (100%) rename Logging/{splunk => Config}/zeek/__load__.zeek (100%) create mode 100644 Logging/Config/zeek/zeek-compose.yml create mode 100644 Logging/docker-compose.yml delete mode 100644 Logging/splunk/docker-compose.yml delete mode 100755 Logging/splunk/fleet-pack.sh delete mode 100644 Logging/splunk/thehive/application.conf diff --git a/Logging/splunk/.env b/Logging/.env similarity index 70% rename from Logging/splunk/.env rename to Logging/.env index 243de46..a0751e0 100644 --- a/Logging/splunk/.env +++ b/Logging/.env @@ -2,9 +2,7 @@ # This allows us to update the docker-compose files without breaking containers like splunk PORTAINER_VER=latest -SPLUNK_VER=8.2.0 +SPLUNK_VER=8.2.7 JUPYTER_VER=latest -HIVE_VER=latest -CORTEX_VER=latest -TRAEFIK_VER=v2.4.8 -ELASTIC_VER=7.12.1 \ No newline at end of file +TRAEFIK_VER=v2.8.1 +INTERFACE=enp0s3 \ No newline at end of file diff --git a/Logging/Set-Logging.ps1 b/Logging/Config/Windows/Set-Logging.ps1 similarity index 100% rename from Logging/Set-Logging.ps1 rename to Logging/Config/Windows/Set-Logging.ps1 diff --git a/Logging/splunk/inputs.conf b/Logging/Config/Windows/inputs.conf similarity index 100% rename from Logging/splunk/inputs.conf rename to Logging/Config/Windows/inputs.conf diff --git a/Logging/research-sysmon-config.xml b/Logging/Config/Windows/research-sysmon-config.xml similarity index 100% rename from Logging/research-sysmon-config.xml rename to Logging/Config/Windows/research-sysmon-config.xml diff --git a/Logging/splunk/splunk/indexes.conf b/Logging/Config/splunk/indexes.conf similarity index 100% rename from Logging/splunk/splunk/indexes.conf rename to Logging/Config/splunk/indexes.conf diff --git a/Logging/splunk/splunk/inputs.conf b/Logging/Config/splunk/inputs.conf similarity index 100% rename from Logging/splunk/splunk/inputs.conf rename to Logging/Config/splunk/inputs.conf diff --git a/Logging/Config/splunk/splunk-compose.yml b/Logging/Config/splunk/splunk-compose.yml new file mode 100644 index 0000000..dd7d0ca --- /dev/null +++ b/Logging/Config/splunk/splunk-compose.yml @@ -0,0 +1,53 @@ +services: + splunk: + container_name: splunk + image: splunk/splunk:${SPLUNK_VER} + ports: + - 9997:9997 + - 8089:8089 + environment: + SPLUNK_PASSWORD: 'Changeme1!' + SPLUNK_START_ARGS: '--accept-license' + #SPLUNK_UPGRADE: 'true' # Only use when upgrading splunk. Modify SPLUNK_VER in .env to the version you want + volumes: + - splunk_etc:/opt/splunk/etc + - splunk_var:/opt/splunk/var + - ./Config/splunk/web.conf:/opt/splunk/etc/system/local/web.conf + - ./Config/zeek/zeek-logs/:/logs/zeek-logs/ + labels: + - "traefik.http.routers.splunk.rule=PathPrefix(`/splunk`)" + - "traefik.http.routers.splunk.middlewares=https-redirect" + - "traefik.http.routers.splunk-secure.tls=true" + - "traefik.http.routers.splunk-secure.rule=PathPrefix(`/splunk`)" + - "traefik.http.services.splunk.loadbalancer.server.port=8000" + restart: unless-stopped + + jupyter-notebooks: + container_name: jupyter-notebooks + image: jupyter/all-spark-notebook:${JUPYTER_VER} + ports: + - "8888:8888" + environment: + - NB_UID=1001 + - NB_GID=1001 + - JUPYTER_ENABLE_LAB=yes + - NB_USER=splunk + - CHOWN_EXTRA=/home/splunk + depends_on: + - "splunk" + volumes: + - splunk_etc:/home/splunk/etc + - splunk_var:/home/splunk/var + - jupyter-notebooks:/home/jovyan + labels: + - "traefik.http.routers.jupyter.rule=PathPrefix(`/jupyter`)" + - "traefik.http.routers.jupyter.middlewares=https-redirect" + - "traefik.http.routers.jupyter-secure.tls=true" + - "traefik.http.routers.jupyter-secure.rule=PathPrefix(`/jupyter`)" + - "traefik.http.services.jupyter.loadbalancer.server.port=8888" + restart: always + +volumes: + jupyter-notebooks: + splunk_etc: + splunk_var: \ No newline at end of file diff --git a/Logging/splunk/splunk/web.conf b/Logging/Config/splunk/web.conf similarity index 100% rename from Logging/splunk/splunk/web.conf rename to Logging/Config/splunk/web.conf diff --git a/Logging/splunk/zeek/__load__.zeek b/Logging/Config/zeek/__load__.zeek similarity index 100% rename from Logging/splunk/zeek/__load__.zeek rename to Logging/Config/zeek/__load__.zeek diff --git a/Logging/Config/zeek/zeek-compose.yml b/Logging/Config/zeek/zeek-compose.yml new file mode 100644 index 0000000..1f4d2d3 --- /dev/null +++ b/Logging/Config/zeek/zeek-compose.yml @@ -0,0 +1,13 @@ +services: + zeek: + container_name: zeek + image: blacktop/zeek + restart: always + network_mode: host + cap_add: + - NET_RAW + volumes: + - ./Config/zeek/zeek-logs/:/pcap:rw + - ./Config/zeek/__load__.zeek:/usr/local/zeek/share/zeek/base/bif/__load__.zeek + command: + - -i ${INTERFACE} -C \ No newline at end of file diff --git a/Logging/docker-compose.yml b/Logging/docker-compose.yml new file mode 100644 index 0000000..98278d5 --- /dev/null +++ b/Logging/docker-compose.yml @@ -0,0 +1,42 @@ +services: + portainer: + container_name: portainer + image: portainer/portainer-ce:${PORTAINER_VER} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer:/data + labels: + - "traefik.http.routers.portainer.rule=PathPrefix(`/portainer/`)" + - "traefik.http.routers.portainer.middlewares=https-redirect" + - "traefik.http.routers.portainer-secure.tls=true" + - "traefik.http.routers.portainer-secure.rule=PathPrefix(`/portainer/`)" + - "traefik.http.routers.portainer-secure.middlewares=portainer-stripprefix" + - "traefik.http.middlewares.portainer-stripprefix.stripprefix.prefixes=/portainer/" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" + restart: always + + traefik: + container_name: traefik + image: traefik:${TRAEFIK_VER} + command: + - --providers.docker=true + - --serversTransport.insecureSkipVerify=true + - --entrypoints.web.address=:80 + - --entrypoints.web-secure.address=:443 + ports: + - "80:80" + - "443:443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - .:/files + labels: + - "traefik.http.routers.api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`)" + - "traefik.http.routers.api.service=api@internal" + - "traefik.http.routers.api.middlewares=https-redirect" + - "traefik.http.routers.api-secure.tls=true" + - "traefik.http.routers.api-secure.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`)" + - "traefik.http.middlewares.https-redirect.redirectscheme.scheme=https" + restart: always + +volumes: + portainer: \ No newline at end of file diff --git a/Logging/splunk/docker-compose.yml b/Logging/splunk/docker-compose.yml deleted file mode 100644 index 638308b..0000000 --- a/Logging/splunk/docker-compose.yml +++ /dev/null @@ -1,102 +0,0 @@ -version: "3.5" -services: - portainer: - container_name: portainer - image: portainer/portainer-ce:${PORTAINER_VER} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - portainer:/data - labels: - - "traefik.http.routers.portainer.rule=PathPrefix(`/portainer/`)" - - "traefik.http.routers.portainer.middlewares=https-redirect" - - "traefik.http.routers.portainer-secure.tls=true" - - "traefik.http.routers.portainer-secure.rule=PathPrefix(`/portainer/`)" - - "traefik.http.routers.portainer-secure.middlewares=portainer-stripprefix" - - "traefik.http.middlewares.portainer-stripprefix.stripprefix.prefixes=/portainer/" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" - restart: always - - splunk: - container_name: splunk - image: splunk/splunk:${SPLUNK_VER} - ports: - - 9997:9997 - - 8089:8089 - environment: - SPLUNK_PASSWORD: 'Changeme1!' - SPLUNK_START_ARGS: '--accept-license' - TZ: 'America/Chicago' - #SPLUNK_UPGRADE: 'true' # Only use when upgrading splunk. Modify SPLUNK_VER in .env to the version you want - volumes: - - splunk_etc:/opt/splunk/etc - - splunk_var:/opt/splunk/var - - ./splunk/web.conf:/opt/splunk/etc/system/local/web.conf - - ./zeek/zeek-logs/:/logs/zeek-logs/ - - ./quick-fleet/result.log:/logs/osquery/result.log - - ./quick-fleet/status.log:/logs/osquery/status.log - labels: - - "traefik.http.routers.splunk.rule=PathPrefix(`/splunk`)" - - "traefik.http.routers.splunk.middlewares=https-redirect" - - "traefik.http.routers.splunk-secure.tls=true" - - "traefik.http.routers.splunk-secure.rule=PathPrefix(`/splunk`)" - - "traefik.http.services.splunk.loadbalancer.server.port=8000" - restart: unless-stopped - - jupyter-notebooks: - container_name: jupyter-notebooks - image: jupyter/all-spark-notebook:${JUPYTER_VER} - ports: - - "8888:8888" - environment: - - NB_UID=1001 - - NB_GID=1001 - - JUPYTER_ENABLE_LAB=yes - - NB_USER=splunk - - CHOWN_EXTRA=/home/splunk - volumes: - - splunk_etc:/home/splunk/etc - - splunk_var:/home/splunk/var - - jupyter-notebooks:/home/jovyan - labels: - - "traefik.http.routers.jupyter.rule=PathPrefix(`/jupyter`)" - - "traefik.http.routers.jupyter.middlewares=https-redirect" - - "traefik.http.routers.jupyter-secure.tls=true" - - "traefik.http.routers.jupyter-secure.rule=PathPrefix(`/jupyter`)" - - "traefik.http.services.jupyter.loadbalancer.server.port=8888" - restart: always - -# TheHive config on the way... - - traefik: - container_name: traefik - image: traefik:${TRAEFIK_VER} - command: - - --providers.docker=true - - --serversTransport.insecureSkipVerify=true - #- --api.dashboard=true - - --entrypoints.web.address=:80 - - --entrypoints.web-secure.address=:443 - #- --log.filePath=/files/traefik.log - #- --log.level=DEBUG - ports: - - "80:80" - - "443:443" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - .:/files - labels: - - "traefik.http.routers.api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`)" - - "traefik.http.routers.api.service=api@internal" - - "traefik.http.routers.api.middlewares=https-redirect" - - "traefik.http.routers.api-secure.tls=true" - - "traefik.http.routers.api-secure.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`)" - - "traefik.http.middlewares.https-redirect.redirectscheme.scheme=https" - restart: always - -volumes: - portainer: - splunk_etc: - splunk_var: - jupyter-notebooks: - elasticsearch: - diff --git a/Logging/splunk/fleet-pack.sh b/Logging/splunk/fleet-pack.sh deleted file mode 100755 index 146032f..0000000 --- a/Logging/splunk/fleet-pack.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Checking to see if script is running as root -if [[ $EUID -ne 0 ]]; then - echo -e "\x1B[01;31m[X] Script Must Be Run As ROOT\x1B[0m" - exit 1 -fi -read -p 'Input your IP and press [ENTER]: ' Host_IP - -# Pulling down OSQuery Configurations: -read -p 'Input the email you used when setting up Kolide, then press [ENTER]: ' EMAIL -read -p 'Input the password you used when setting up Kolide, then press [ENTER]: ' PASSWORD -sudo docker exec -ti fleet sh -c 'fleetctl config set --address https://'$Host_IP':8443 --tls-skip-verify' -docker exec -ti fleet sh -c 'fleetctl setup --email $EMAIL --password $PASSWORD' -git clone https://github.com/palantir/osquery-configuration.git -docker cp osquery-configuration/Fleet/Endpoints/packs/windows-application-security.yaml fleet:/windows-application-security.yaml -docker cp osquery-configuration/Fleet/Endpoints/packs/windows-registry-monitoring.yaml fleet:/windows-registry-monitoring.yaml -docker cp osquery-configuration/Fleet/Endpoints/packs/windows-compliance.yaml fleet:/windows-compliance.yaml -docker exec -ti fleet sh -c 'fleetctl apply -f windows-application-security.yaml' -docker exec -ti fleet sh -c 'fleetctl apply -f windows-registry-monitoring.yaml' -docker exec -ti fleet sh -c 'fleetctl apply -f windows-compliance.yaml' -rm -rf osquery-configuration/ - -echo -e "\x1B[01;34m[*] Configuration complete!\x1B[0m" diff --git a/Logging/splunk/thehive/application.conf b/Logging/splunk/thehive/application.conf deleted file mode 100644 index 8d234c4..0000000 --- a/Logging/splunk/thehive/application.conf +++ /dev/null @@ -1,49 +0,0 @@ -# Elasticsearch -search { - ## Basic configuration - # Index name. - index = the_hive - # ElasticSearch instance address. - uri = "http://elasticsearch:9200/" - - # Scroll keepalive - keepalive = 1m - # Size of the page for scroll - pagesize = 50 - # Number of shards - nbshards = 5 - # Number of replicas - nbreplicas = 1 - # Arbitrary settings - settings { - # Maximum number of nested fields - mapping.nested_fields.limit = 100 - } - - ## Authentication configuration - #search.username = "" - #search.password = "" -} - -## Enable the Cortex module -play.modules.enabled += connectors.cortex.CortexConnector - -cortex { - "CORTEX-SERVER-ID" { - # URL of the Cortex server - url = "http://cortex:9001" - # Key of the Cortex user, mandatory for Cortex 2 - key = "API key" - } - # HTTP client configuration, more details in section 8 - # ws { - # proxy {} - # ssl {} - # } - # Check job update time interval - refreshDelay = 1 minute - # Maximum number of successive errors before give up - maxRetryOnError = 3 - # Check remote Cortex status time interval - statusCheckInterval = 1 minute -} \ No newline at end of file From 9fad21dbee5c69c4b4ec653a7ab8ccbb8649168f Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Wed, 27 Jul 2022 04:32:48 +0000 Subject: [PATCH 04/15] Update gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9419575..5bd6279 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -Logging/splunk/quick-fleet -Logging/splunk/zeek/zeek-logs/* +Logging/Config/splunk/zeek/zeek-logs/* *.log +*.code-workspace From 3967ef4ac65a3946d98530c5fd69004ba0dd0e6b Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 18 Aug 2022 03:17:07 +0000 Subject: [PATCH 05/15] Fixes for setup_logging --- Logging/setup_logging.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index ee7aa4a..bdd1919 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -3,14 +3,16 @@ #References: https://stackoverflow.com/ && https://github.com/target/huntlib.git SETUP_SPLUNK="False" -SETUP_ELASTIC="False" -SETUP_ZEEK="False" +SETUP_ELASTIC="True" +SETUP_ZEEK="True" + +HOST_IP=localhost # Checking to see if script is running as root if [[ $EUID -ne 0 ]]; then - echo -e "\x1B[01;31m[X] Script Must Be Run As ROOT\x1B[0m" - exit 1 -fi + echo -e "\x1B[01;31m[X] Script Must Be Run As ROOT\x1B[0m" + exit 1 + fi echo -e "\x1B[01;34m[*] Setting timezone to UTC...\x1B[0m" timedatectl set-timezone UTC @@ -23,7 +25,7 @@ if [[ $(which docker) && $(docker compose version) ]]; then else echo -e "\x1B[01;31m[*] Docker was not found. See the Read the Docs installation documentation (https://marvel-lab.readthedocs.io/en/latest/subpages/build_steps.html#logging) \x1B[0m" exit 0 -fi + fi # Enabling docker service: echo -e "\x1B[01;34m[*] Enabling Docker Service...\x1B[0m" @@ -49,9 +51,16 @@ if [ "$SETUP_ZEEK" = "True" ]; then fi read -p 'Input the network interface you would like Zeek to monitor and press [ENTER]: ' Interface - + echo -e "\x1B[01;34m[*] Creating Zeek:\x1B[0m" - docker compose up -d -f ./Config/zeek/zeek-compose.yml + docker compose -f ./Config/zeek/zeek-compose.yml up -d + fi + +# Elastic +if [ "$SETUP_ELASTIC" = "True" ]; then + echo -e "\x1B[01;34m[*] Creating Elastic Stack:\x1B[0m" + docker compose -f ./Config/elasticstack/elasticstack-compose.yml up -d + fi # Splunk if [ "$SETUP_SPLUNK" = "True" ]; then @@ -81,9 +90,11 @@ if [ "$SETUP_SPLUNK" = "True" ]; then sleep 10 token="$(docker exec -it jupyter-notebooks sh -c 'jupyter notebook list' | grep token | sed 's/.*token=\([^ ]*\).*/\1/')" - echo -e "\x1B[01;32m[*] Access Splunk at https://$Host_IP/splunk/ ; Credentials - admin:Changeme1! (unless you changed them in the DockerFile)\x1B[0m" - echo -e "\x1B[01;32m[*] Access Jupyter Notebook at: http://$Host_IP:8888\x1B[0m" + echo -e "\x1B[01;32m[*] Access Splunk at https://$HOST_IP/splunk/ ; Credentials - admin:Changeme1! (unless you changed them in the DockerFile)\x1B[0m" + echo -e "\x1B[01;32m[*] Access Jupyter Notebook at: http://$HOST_IP:8888\x1B[0m" echo -e "\x1B[01;32m[*] Jupyter Notebook token is $token\x1B[0m" + fi -echo -e "\x1B[01;32m[*] Access Portainer at https://$Host_IP/portainer/ \x1B[0m" +# Print out info +echo -e "\x1B[01;32m[*] Access Portainer at https://$HOST_IP/portainer/ \x1B[0m" From a6edd71a65adc399bb241b890c4a8d301cbed953 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 18 Aug 2022 03:17:34 +0000 Subject: [PATCH 06/15] Add Elastic Stack --- Logging/Config/elasticstack/.env | 29 ++++ .../elasticstack/elasticstack-compose.yml | 131 ++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 Logging/Config/elasticstack/.env create mode 100644 Logging/Config/elasticstack/elasticstack-compose.yml diff --git a/Logging/Config/elasticstack/.env b/Logging/Config/elasticstack/.env new file mode 100644 index 0000000..f720c8f --- /dev/null +++ b/Logging/Config/elasticstack/.env @@ -0,0 +1,29 @@ +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD= + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD= + +# Version of Elastic products +STACK_VERSION=8.3.3 + +# Set the cluster name +CLUSTER_NAME=docker-cluster + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 +#ES_PORT=127.0.0.1:9200 + +# Port to expose Kibana to the host +KIBANA_PORT=5601 +#KIBANA_PORT=80 + +# Increase or decrease based on the available host memory (in bytes) +MEM_LIMIT=1073741824 + +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject \ No newline at end of file diff --git a/Logging/Config/elasticstack/elasticstack-compose.yml b/Logging/Config/elasticstack/elasticstack-compose.yml new file mode 100644 index 0000000..105e11c --- /dev/null +++ b/Logging/Config/elasticstack/elasticstack-compose.yml @@ -0,0 +1,131 @@ +# Taken from: +# https://elastic.co/guide/en/elasticsearch/reference/current/docker.html + + +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + es01: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + volumes: + - certs:/usr/share/kibana/config/certs + - kibanadata:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + mem_limit: ${MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + +volumes: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local \ No newline at end of file From c12f804d0fd61192d74ab8e56346f35e6fbf7644 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 02:15:04 +0000 Subject: [PATCH 07/15] Better handling of the Docker service --- Logging/setup_logging.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index bdd1919..270822d 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -28,9 +28,12 @@ if [[ $(which docker) && $(docker compose version) ]]; then fi # Enabling docker service: -echo -e "\x1B[01;34m[*] Enabling Docker Service...\x1B[0m" -systemctl enable docker.service - +echo -e "\x1B[01;34m[*] Checking for the Docker service...\x1B[0m" +if [[ $(systemctl list-unit-files --state=enabled | grep docker.service) ]]; then + echo -e "\x1B[01;32m[*] Docker service already enabled\x1B[0m" + else + systemctl enable docker.service + fi # Starting containers echo -e "\x1B[01;34m[*] Starting containers\x1B[0m" docker compose up -d From 7fd15a987a39bf334feeea7369c1e7889b29638b Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 03:54:43 +0000 Subject: [PATCH 08/15] Updated Zeek config --- Logging/Config/zeek/.env | 1 + Logging/Config/zeek/zeek-compose.yml | 6 +++--- Logging/setup_logging.sh | 31 ++++++++++++++++++---------- 3 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 Logging/Config/zeek/.env diff --git a/Logging/Config/zeek/.env b/Logging/Config/zeek/.env new file mode 100644 index 0000000..9d2ab60 --- /dev/null +++ b/Logging/Config/zeek/.env @@ -0,0 +1 @@ +INTERFACE= \ No newline at end of file diff --git a/Logging/Config/zeek/zeek-compose.yml b/Logging/Config/zeek/zeek-compose.yml index 1f4d2d3..63527c9 100644 --- a/Logging/Config/zeek/zeek-compose.yml +++ b/Logging/Config/zeek/zeek-compose.yml @@ -1,13 +1,13 @@ services: zeek: container_name: zeek - image: blacktop/zeek + image: blacktop/zeek:latest restart: always network_mode: host cap_add: - NET_RAW volumes: - - ./Config/zeek/zeek-logs/:/pcap:rw - - ./Config/zeek/__load__.zeek:/usr/local/zeek/share/zeek/base/bif/__load__.zeek + - ./zeek-logs/:/pcap:rw + - ./__load__.zeek:/usr/local/zeek/share/zeek/base/bif/__load__.zeek command: - -i ${INTERFACE} -C \ No newline at end of file diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index 270822d..93bd5fd 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -40,20 +40,29 @@ docker compose up -d # Zeek if [ "$SETUP_ZEEK" = "True" ]; then - read -r -p "Zeek needs a network interface to monitor. Would you like to print out your interfaces to see which one to monitor? [y/N] " response - - if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]] - then - if hash ifconfig 2>/dev/null; then - ifconfig + export $(grep -v '^#' Config/zeek/.env | xargs) + echo -e "Zeek needs a network interface to monitor. The currently configured interface is '${INTERFACE}'." + read -r -p "Would you like to change the selected interface? You'll need to set the interface if this is the first time running this script. Otherwise you can set it in Config/zeek/.env [y/N] " response1 + + if [[ "$response1" =~ ^([yY][eE][sS]|[yY])$ ]]; then + + read -r -p "Would you like to print out your interfaces to see which one to monitor? [y/N] " response2 + + if [[ "$response2" =~ ^([yY][eE][sS]|[yY])$ ]]; then + if hash ifconfig 2>/dev/null; then + ifconfig + else + ip address + fi else - ip address + echo -e "\x1B[01;34m[*] Moving on...\x1B[0m" fi - else - echo -e "\x1B[01;34m[*] Moving on...\x1B[0m" - fi - read -p 'Input the network interface you would like Zeek to monitor and press [ENTER]: ' Interface + read -p 'Input the network interface you would like Zeek to monitor and press [ENTER]: ' INTERFACE + + echo -e "\x1B[01;34m[*] Writing desired interface to Config/zeek/.env\x1B[0m" + echo "INTERFACE=${INTERFACE}" > ./Config/zeek/.env + fi echo -e "\x1B[01;34m[*] Creating Zeek:\x1B[0m" docker compose -f ./Config/zeek/zeek-compose.yml up -d From 8e04206db40ba7464ec3f2bef47fbbde676642e5 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 03:55:25 +0000 Subject: [PATCH 09/15] Update other .env files --- Logging/.env | 4 +--- Logging/Config/splunk/.env | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 Logging/Config/splunk/.env diff --git a/Logging/.env b/Logging/.env index a0751e0..7354d72 100644 --- a/Logging/.env +++ b/Logging/.env @@ -2,7 +2,5 @@ # This allows us to update the docker-compose files without breaking containers like splunk PORTAINER_VER=latest -SPLUNK_VER=8.2.7 JUPYTER_VER=latest -TRAEFIK_VER=v2.8.1 -INTERFACE=enp0s3 \ No newline at end of file +TRAEFIK_VER=v2.8.1 \ No newline at end of file diff --git a/Logging/Config/splunk/.env b/Logging/Config/splunk/.env new file mode 100644 index 0000000..4c5ab31 --- /dev/null +++ b/Logging/Config/splunk/.env @@ -0,0 +1 @@ +SPLUNK_VER=8.2.7 \ No newline at end of file From e73ef57a1f2e243408fbb7358d99ce3ace113010 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 04:47:59 +0000 Subject: [PATCH 10/15] Elastic Stack password config --- Logging/Config/elasticstack/.env | 2 +- Logging/setup_logging.sh | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Logging/Config/elasticstack/.env b/Logging/Config/elasticstack/.env index f720c8f..b76bc41 100644 --- a/Logging/Config/elasticstack/.env +++ b/Logging/Config/elasticstack/.env @@ -8,7 +8,7 @@ KIBANA_PASSWORD= STACK_VERSION=8.3.3 # Set the cluster name -CLUSTER_NAME=docker-cluster +CLUSTER_NAME=Marvel-Lab # Set to 'basic' or 'trial' to automatically start the 30-day trial LICENSE=basic diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index 93bd5fd..3572949 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -70,6 +70,15 @@ if [ "$SETUP_ZEEK" = "True" ]; then # Elastic if [ "$SETUP_ELASTIC" = "True" ]; then + export $(grep -v '^#' Config/elasticstack/.env | xargs) + + if [ "$ELASTIC_PASSWORD" = '' ]; then + echo -e "\x1B[01;34m[*] Elastic and Kibana need a password. You can enter it here, or close this script and configure in Config/elasticstack/.env\x1B[0m" + read -p 'Enter your desired password: ' NEW_ELASTIC_PASSWORD + sed -i "s/ELASTIC_PASSWORD=/ELASTIC_PASSWORD=$NEW_ELASTIC_PASSWORD/" Config/elasticstack/.env + sed -i "s/KIBANA_PASSWORD=/KIBANA_PASSWORD=$NEW_ELASTIC_PASSWORD/" Config/elasticstack/.env + fi + echo -e "\x1B[01;34m[*] Creating Elastic Stack:\x1B[0m" docker compose -f ./Config/elasticstack/elasticstack-compose.yml up -d fi @@ -108,5 +117,4 @@ if [ "$SETUP_SPLUNK" = "True" ]; then fi # Print out info -echo -e "\x1B[01;32m[*] Access Portainer at https://$HOST_IP/portainer/ \x1B[0m" - +echo -e "\x1B[01;32m[*] Access Portainer at https://$HOST_IP/portainer/ \x1B[0m" \ No newline at end of file From 3f50c64f812c815ea4d4245d6648b3718c00033a Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 04:51:14 +0000 Subject: [PATCH 11/15] General updates --- Logging/.env | 2 +- Logging/setup_logging.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Logging/.env b/Logging/.env index 7354d72..7936d1e 100644 --- a/Logging/.env +++ b/Logging/.env @@ -3,4 +3,4 @@ PORTAINER_VER=latest JUPYTER_VER=latest -TRAEFIK_VER=v2.8.1 \ No newline at end of file +TRAEFIK_VER=v2.8.3 \ No newline at end of file diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index 3572949..6cbe280 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -34,6 +34,7 @@ if [[ $(systemctl list-unit-files --state=enabled | grep docker.service) ]]; the else systemctl enable docker.service fi + # Starting containers echo -e "\x1B[01;34m[*] Starting containers\x1B[0m" docker compose up -d From 0c5bedaa12b5c3c6eb8183b9efd86c21fef9585c Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 25 Aug 2022 05:15:10 +0000 Subject: [PATCH 12/15] Check virtual memory for elastic --- Logging/setup_logging.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Logging/setup_logging.sh b/Logging/setup_logging.sh index 6cbe280..36325f0 100755 --- a/Logging/setup_logging.sh +++ b/Logging/setup_logging.sh @@ -80,6 +80,13 @@ if [ "$SETUP_ELASTIC" = "True" ]; then sed -i "s/KIBANA_PASSWORD=/KIBANA_PASSWORD=$NEW_ELASTIC_PASSWORD/" Config/elasticstack/.env fi + echo -e "\x1B[01;34m[*] Checking vm.max_map_count\x1B[0m" + if [[ "$(sysctl vm.max_map_count)" != "vm.max_map_count = 262144" ]]; then + echo -e "\x1B[01;34m[*] Updating vm.max_map_count in /etc/sysctl.conf\x1B[0m" + echo "vm.max_map_count=262144" >> /etc/sysctl.conf + sysctl -p + fi + echo -e "\x1B[01;34m[*] Creating Elastic Stack:\x1B[0m" docker compose -f ./Config/elasticstack/elasticstack-compose.yml up -d fi From c911782d88c8efeff96e8b37a9d0368f2e2be276 Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Wed, 31 Aug 2022 23:46:59 +0000 Subject: [PATCH 13/15] Set elastic to start on boot --- Logging/Config/elasticstack/elasticstack-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Logging/Config/elasticstack/elasticstack-compose.yml b/Logging/Config/elasticstack/elasticstack-compose.yml index 105e11c..c043250 100644 --- a/Logging/Config/elasticstack/elasticstack-compose.yml +++ b/Logging/Config/elasticstack/elasticstack-compose.yml @@ -94,6 +94,7 @@ services: interval: 10s timeout: 10s retries: 120 + restart: always kibana: depends_on: @@ -121,6 +122,7 @@ services: interval: 10s timeout: 10s retries: 120 + restart: always volumes: certs: From 9c8419e431238ab8dde6393e88901038e932dd8f Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 1 Sep 2022 00:19:41 +0000 Subject: [PATCH 14/15] Move certs to a local directory --- Logging/Config/elasticstack/elasticstack-compose.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Logging/Config/elasticstack/elasticstack-compose.yml b/Logging/Config/elasticstack/elasticstack-compose.yml index c043250..58f73f7 100644 --- a/Logging/Config/elasticstack/elasticstack-compose.yml +++ b/Logging/Config/elasticstack/elasticstack-compose.yml @@ -6,7 +6,7 @@ services: setup: image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} volumes: - - certs:/usr/share/elasticsearch/config/certs + - ./certs:/usr/share/elasticsearch/config/certs user: "0" command: > bash -c ' @@ -58,7 +58,7 @@ services: condition: service_healthy image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} volumes: - - certs:/usr/share/elasticsearch/config/certs + - ./certs:/usr/share/elasticsearch/config/certs - esdata01:/usr/share/elasticsearch/data ports: - ${ES_PORT}:9200 @@ -102,7 +102,7 @@ services: condition: service_healthy image: docker.elastic.co/kibana/kibana:${STACK_VERSION} volumes: - - certs:/usr/share/kibana/config/certs + - ./certs:/usr/share/kibana/config/certs - kibanadata:/usr/share/kibana/data ports: - ${KIBANA_PORT}:5601 @@ -125,8 +125,6 @@ services: restart: always volumes: - certs: - driver: local esdata01: driver: local kibanadata: From ca3da49f3f5b125d35f51dc1d9a447f171d542ec Mon Sep 17 00:00:00 2001 From: Ben Shell Date: Thu, 1 Sep 2022 00:33:20 +0000 Subject: [PATCH 15/15] fixes --- Logging/Config/Windows/Set-Logging.ps1 | 313 ------------------------- 1 file changed, 313 deletions(-) delete mode 100644 Logging/Config/Windows/Set-Logging.ps1 diff --git a/Logging/Config/Windows/Set-Logging.ps1 b/Logging/Config/Windows/Set-Logging.ps1 deleted file mode 100644 index bf031ee..0000000 --- a/Logging/Config/Windows/Set-Logging.ps1 +++ /dev/null @@ -1,313 +0,0 @@ -#Author Jonathan Johnson - -#References:https://powershellexplained.com/2016-10-21-powershell-installing-msi-files/ && https://docs.splunk.com/Documentation/Splunk/8.0.3/Installation/InstallonWindowsviathecommandline - -using namespace System.Management.Automation.Host - -function Show-Menu { - [CmdletBinding()] - param ( - [string]$Question = 'Which logging option would you like?' - ) - - Clear-Host - Write-Host " -███╗ ███╗ █████╗ ██████╗ ██╗ ██╗███████╗██╗ ██╗ ██████╗ ██████╗ ██████╗ ██╗███╗ ██╗ ██████╗ -████╗ ████║██╔══██╗██╔══██╗██║ ██║██╔════╝██║ ██║ ██╔═══██╗██╔════╝ ██╔════╝ ██║████╗ ██║██╔════╝ -██╔████╔██║███████║██████╔╝██║ ██║█████╗ ██║ ██║ ██║ ██║██║ ███╗██║ ███╗██║██╔██╗ ██║██║ ███╗ -██║╚██╔╝██║██╔══██║██╔══██╗╚██╗ ██╔╝██╔══╝ ██║ ██║ ██║ ██║██║ ██║██║ ██║██║██║╚██╗██║██║ ██║ -██║ ╚═╝ ██║██║ ██║██║ ██║ ╚████╔╝ ███████╗███████╗ ███████╗╚██████╔╝╚██████╔╝╚██████╔╝██║██║ ╚████║╚██████╔╝ -╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═══╝ ╚══════╝╚══════╝ ╚══════╝ ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝ - -" - Write-Host "================ $Question ==============" - Write-Host "1: Press '1' to install Sysmon with no forwarding." - Write-Host "2: Press '2' for Sysmon + Winlogbeat which will forward Sysmon and Windows Events to HELK/ELK Instance." - Write-Host "3: Press '3' for Sysmon + Splunk UF which will forward Sysmon/Windows Events to Splunk." - Write-Host "4: Press '4' for Sysmon + OSQuery + Splunk UF which will forward Sysmon/Windows Events/OSQuery to Splunk." - Write-Host "5: Press '5' to remove all logging components (sensors and forwarders)." - -} - -# Disable download progress bar to improve download speed -$ProgressPreference = 'SilentlyContinue' -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -Write-Host "[*] Checking for elevated permissions..." -ForegroundColor Green - -$id = [System.Security.Principal.WindowsIdentity]::GetCurrent() - -$principal = New-Object System.Security.Principal.WindowsPrincipal($id) - -if (!$principal.IsInRole([System.Security.Principal.WindowsBuiltInRole]::Administrator)) - { - Write-Warning "[*] You do not have the correct permissions to run this script. Please re-run as Administrator" -ErrorAction Stop - - } -else { - - function Install-Sysmon { - Write-Host "[*] Checking to see if Sysmon is installed on host..." - $sysmonservice = [bool](Get-Service Sysmon* -ErrorAction SilentlyContinue) - if ($sysmonservice -eq $true){ - Write-Warning "[*] Sysmon is installed already" - } - else { - #Sysmon Arguments: - $SysmonUrl = "https://download.sysinternals.com/files/Sysmon.zip" - $SysmonOutputFile = "sysmonconfig.xml" - - New-Item -Path "c:\" -Name "Sysmon" -ItemType "directory" - - Write-Host "[*] Downloading Sysmon" -ForegroundColor Green - Invoke-WebRequest $SysmonUrl -OutFile C:\Sysmon\Sysmon.zip - Expand-Archive -LiteralPath C:\Sysmon\Sysmon.zip -DestinationPath C:\Sysmon\ - - $configtype = Read-Host "Would you like to install a modular config for Sysmon (by Olaf) or a research config that is less restricted? Choose: modular OR research" - - switch ( $configtype ) - { - 'modular' - { - $SysmonConfig = "https://raw.githubusercontent.com/olafhartong/sysmon-modular/master/sysmonconfig.xml" - } - 'research' - { - $SysmonConfig = "https://raw.githubusercontent.com/jsecurity101/Marvel-Lab/master/Logging/research-sysmon-config.xml" - } - default - { - Write-Host "Invalid selection" - } - } - - Write-Host "[*] You chose $configtype. Installing now configuration now..." -ForegroundColor Green - Invoke-WebRequest $SysmonConfig -OutFile C:\Sysmon\$SysmonOutputFile - - Write-Host "Installing Sysmon.." -ForegroundColor Green - & cmd.exe /c 'C:\Sysmon\Sysmon64.exe -accepteula -i C:\Sysmon\sysmonconfig.xml -a ArchivedFiles 2>&1' - } - -} - - function Install-Winlogbeat { - - Write-Host "[*] Checking to see if winlogbeat is installed on host..." - $winlogbeatservice = [bool](Get-Service winlogbeat -ErrorAction SilentlyContinue) - if ($winlogbeatservice -eq $True) - { - Write-Warning "[*] Winlogbeat is installed already" - } - else { - #Winlogbeat Arguments: - $WinlogbeatVer = "7.13.0" - $WinlogbeatUrl = "https://artifacts.elastic.co/downloads/beats/winlogbeat/winlogbeat-" + $WinlogbeatVer + "-windows-x86_64.zip" - $WinlogbeatOutputFile = "winlogbeat.zip" - $WinlogbeatConfig = "https://gist.github.com/jsecurity101/ec4c829e6d32a984d7ccf4c1e9247590/archive/8d85c6c443704e821a7f53e536be61667c67febd.zip" - $WinlogZip = "winlogconfig.zip" - - $HELK_IP = Read-Host "Please input the IP of your HELK/ELK box" - - Write-Host "[*] Creating winlogbeat path - C:\Winlogbeat" -ForegroundColor Green - New-Item -Path "c:\" -Name "Winlogbeat" -ItemType "directory" - - Write-Host "[*] Installing Winlogbeat" -ForegroundColor Green - Invoke-WebRequest $WinlogbeatUrl -OutFile C:\Winlogbeat\$WinlogbeatOutputFile - Expand-Archive -LiteralPath C:\Winlogbeat\$WinlogbeatOutputFile -DestinationPath C:\Winlogbeat\ - - Invoke-WebRequest $WinlogbeatConfig -OutFile C:\Winlogbeat\$WinlogZip - Expand-Archive -LiteralPath C:\Winlogbeat\$WinlogZip -DestinationPath C:\Winlogbeat\ - - Remove-Item C:\Winlogbeat\winlogbeat-$WinlogbeatVer-windows-x86_64\winlogbeat.yml - Move-Item C:\Winlogbeat\ec4c829e6d32a984d7ccf4c1e9247590-8d85c6c443704e821a7f53e536be61667c67febd\winlogbeat.yml C:\Winlogbeat\winlogbeat-$WinlogbeatVer-windows-x86_64\ - Remove-Item C:\Winlogbeat\$WinlogZip, C:\Winlogbeat\ec4c829e6d32a984d7ccf4c1e9247590-8d85c6c443704e821a7f53e536be61667c67febd - - (Get-Content C:\Winlogbeat\winlogbeat-$WinlogbeatVer-windows-x86_64\winlogbeat.yml).replace('', $HELK_IP) | Set-Content C:\Winlogbeat\winlogbeat-$WinlogbeatVer-windows-x86_64\winlogbeat.yml - - Remove-Item C:\Winlogbeat\$WinlogbeatOutputFile - - & C:\Winlogbeat\winlogbeat-$WinlogbeatVer-windows-x86_64\install-service-winlogbeat.ps1 - - Start-Service winlogbeat - } - - } - - function Install-OSQuery { - Write-Host "[*] Checking to see if chocolatey is installed on host..." - $c = [bool](Get-Command -Name choco.exe -ErrorAction SilentlyContinue) - if ($c -eq $true) { - Write-Warning "[*] Chocolatey is already installed" - } - else { - Write-Host "Installing Chocolatey" -ForegroundColor Green - Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - } - - Write-Host "[*] Checking to see if Osquery is installed on host..." - $o = [bool](Get-Command osqueryd.exe -ErrorAction SilentlyContinue) - if ($o -eq $true) { - Write-Warning "[*] Osquery is already installed" - } - else { - choco install osquery -y - } - } - - function Install-Fleet { - Write-Host "[*] Checking to see if Fleet is installed on host..." - $f = [bool](Get-Service kolide_launcher -ErrorAction SilentlyContinue) - - if ($f -eq $true) { - Write-Warning "[*] Fleet is already installed" - } - else { - #OSQuery Arguments: - $KolideLauncher = "https://github.com/kolide/launcher/releases/download/v0.11.19/windows.amd64_v0.11.19.zip" - $FleetFolderName = "windows.amd64_v0.11.19" - New-Item -Path "c:\" -Name "OSQuery" -ItemType "directory" - $OSQuery_IP = Read-Host "Please Input the IP and port number of the Kolide Server" - $Enroll_Secret = Read-Host "Go to https://"$OSQuery_IP":8443, click on 'Add New Host', copy the Enroll Secret and paste here" - - Invoke-WebRequest $KolideLauncher -OutFile 'C:\OSQuery\kolidelauncher.zip' - Expand-Archive -LiteralPath C:\OSquery\kolidelauncher.zip -DestinationPath C:\OSquery\ - New-Service -Name "kolide_launcher" -BinaryPathName "C:\OSQuery\$FleetFolderName\launcher.exe --hostname=$OSQuery_IP`:8443 --enroll_secret=$Enroll_Secret --insecure" - sc.exe start kolide_launcher - Remove-Item C:\OSQuery\kolidelauncher.zip - Write-Host "OSQuery logs are now available in Kolide Fleet" -ForegroundColor Green - } - - } - - function Install-Splunk { - Write-Host "[*] Checking to see if Splunk is installed on host..." - $s = [bool](Get-Service SplunkForwarder -ErrorAction SilentlyContinue) - - if ($s -eq $true) { - Write-Warning "[*] Splunk is already installed" - } - else { - #Splunk Arugments: - $SplunkUF = "https://download.splunk.com/products/universalforwarder/releases/8.2.6/windows/splunkforwarder-8.2.6-a6fe1ee8894b-x64-release.msi" - $Splunk_IP = Read-Host "Please input the IP of your Splunk box" - - #Installing Splunk - Write-Host "Installing SplunkUF" -ForegroundColor Green - - Invoke-WebRequest $SplunkUF -OutFile $env:HOMEDRIVE:\splunk_forwarder.msi - - & cmd.exe /c msiexec.exe /i c:\splunk_forwarder.msi RECEIVING_INDEXER=$Splunk_IP":9997" LAUNCHSPLUNK=0 WINEVENTLOG_APP_ENABLE=1 WINEVENTLOG_SET_ENABLE=1 WINEVENTLOG_SEC_ENABLE=1 WINEVENTLOG_SYS_ENABLE=1 AGREETOLICENSE=Yes /quiet - - Remove-Item 'C:\Program Files\SplunkUniversalForwarder\etc\apps\SplunkUniversalForwarder\local\inputs.conf' - - Copy-Item C:\Marvel-Lab\Logging\splunk\inputs.conf 'C:\Program Files\SplunkUniversalForwarder\etc\apps\SplunkUniversalForwarder\local\inputs.conf' - - & cmd.exe /c 'C:\Program Files\SplunkUniversalForwarder\bin\splunk.exe' start - - } - - } - - function Remove-Logging { - Write-Host "[*] Checking to see if Splunk is installed on host..." - $s = [bool](Get-Service SplunkForwarder -ErrorAction SilentlyContinue) - - if ($s -eq $false) { - Write-Host "[*] Splunk is not installed" -ForegroundColor Green - } - else { - Write-Host "[*] Uninstalling Splunk and removing its folders.." - Stop-Service SplunkForwarder - $UniversalForwarder = Get-WmiObject -Class Win32_Product | Where-Object{$_.Name -eq "UniversalForwarder"} - $UniversalForwarder.Uninstall() - Remove-Item "C:\splunk_forwarder.msi" - } - - Write-Host "[*] Checking to see if Sysmon is installed on host..." - $sysmon = [bool](Get-Service Sysmon* -ErrorAction SilentlyContinue) - if ($sysmon -eq $false) { - Write-Host "[*] Sysmon is not installed" -ForegroundColor Green - } - else { - Write-Host "[*] Uninstalling Sysmon and removing its folders.." - C:\Sysmon\Sysmon64.exe -u - Remove-Item "C:\Sysmon\" -Recurse -Force - } - - Write-Host "[*] Checking to see if winlogbeat is installed..." - $winlog = [bool](Get-Service winlogbeat -ErrorAction SilentlyContinue) - if ($winlog -eq $false) { - Write-Host "[*] Winlogbeat is not installed" -ForegroundColor Green - } - else { - Write-Host "[*] Uninstalling Winlogbeat and removing its folders.." - Stop-Service winlogbeat - Remove-Item "C:\Winlogbeat\" -Recurse -Force - } - - Write-Host "[*] Checking to see if kolide is installed..." - $kolide = [bool](Get-Service kolide_launcher -ErrorAction SilentlyContinue) - if ($kolide -eq $false) { - Write-Host "[*] Kolide is not installed" -ForegroundColor Green - } - else { - Write-Host "[*] Uninstalling kolide and removing its folders.." - Stop-Service kolide_launcher - sc.exe delete kolide_launcher - $path = Test-Path C:\Osquery\ - if ($path -eq $false) { - Write-Host "Osuqery is note installed" - } - else{ - Remove-Item "C:\Osquery\" -Recurse -Force - } - } - - Write-Host "[*] Checking to see if Osquery is installed..." - $o = [bool](Get-Command osqueryd.exe -ErrorAction SilentlyContinue) - if ($o -eq $false) { - Write-Host "[*] Osquery is not installed" -ForegroundColor Green - } - else { - Write-Host "[*] Uninstalling kolide and removing its folders.." - choco uninstall osquery -y - } - } - - do { - Welcome_Banner - Show-Menu - $selection = Read-Host "Please make a selection" - - switch ($selection) { - '1' { - Write-Host "[*] You chose to only install Sysmon with no forwarding" -ForegroundColor Blue - Install-Sysmon - } - '2' { - Write-Host "[*] You chose to install Sysmon + Winlogbeat which will forward Sysmon and Windows Events to HELK/ELK Instance" -ForegroundColor Blue - Install-Sysmon - Install-Winlogbeat - } - '3' { - Write-Host "[*] You chose to install Sysmon + Splunk UF which will forward Sysmon/Windows Events to Splunk" -ForegroundColor Blue - Install-Sysmon - Install-Splunk - } - '4' { - Write-Host "[*] You chose to install Sysmon + OSQuery + Splunk UF which will forward Sysmon/Windows Events/OSQuery to Splunk" -ForegroundColor Blue - Install-Sysmon - Install-OSQuery - Install-Fleet - Install-Splunk - } - '5' { - Write-Host "[*] You chose to remove all logging components (sensors and forwarders)" -ForegroundColor Blue - Remove-Logging - } - } - } - until ($selection -eq '1' -or '2' -or '3' -or '4') - - } -