Initial commit

alanwill · alanwill · commit d1c947cbf594 · 2014-10-05T21:36:19.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,4 @@
+*.conf
+*.pem
+*.key
+
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Alan Williams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,54 @@
+Ansible Playbook for Splunk
+==============
+
+This Ansible playbook installs and configures a Splunk 6.1 cluster
+
+
+So far I've only used and tested this playbook on AWS instances. 
+
+* The [core](https://github.com/alanwill/cfn-core) CloudFormation template is used to create the VPC
+* The [cfn-splunk](https://github.com/alanwill/cfn-splunk) CloudFormation template is used to create the Splunk components
+
+You don't have to use the above templates and can surely use a pre-created VPC and instances, just be sure that your instances are tagged as Ansible expects or tweak the splunk-site.yml file to adapt to your tagging convention.
+
+This playbook will do the following:
+
+* Install the latest OS security updates on all instances
+* Update the hostsnames on all instances to be the EC2 instance ID
+* Download and install the Splunk Enterprise RPM on all instances, if it's not already installed
+* Delete the RPM after successful install
+* Run config_splunk_inputs.sh which updates the inputs.conf on each component to include the instance hostname
+* Start Splunk and set to auto-start on boot
+* Update ACLs to allow Splunk to read /var/log files
+* Reset the default Splunk password
+* Copy custom configuration files (authentication.conf, web.conf, authorize.conf, ui-prefs.conf, alert_actions.conf)
+* Copy custom certs for Splunk Web
+* Add nodes to the License Master
+* Restart Splunk
+* Install packages to enable Cloudwatch metrics
+* Configure the Cluster Master with a replication factor of 3 and search factor of 2 then restart Splunk on the instance
+* Add the Search Heads to the cluster, then restart Splunk on them
+* Add the Peer Nodes to the cluster
+* Partition and mount the Peer node volumes
+* Disable Splunk web on the Peer Nodes
+
+By the time this playbook completes you'll have a working Splunk cluster.
+
+##Future
+
+There's a few things I'm looking to do to make this playbook more re-usable, namely:
+
+* Increase the idempotency
+* Make the peer node role more dynamic to various instance sizes. As it, it works best with i2.2xlarge instances
+* Consolidate all variables to a single master file
+
+##Contributing
+
+I can't say this enough, Pull Requests are very much welcomed. Hope this playbook helps others as much as it helps me. If you have any feedback on ways to improve it, I'm all ears. Submit an Issue if something doesn't work as advertised.
+
+alan
+
+
+
+
+
diff --git a/roles/cluster-master/tasks/main.yml b/roles/cluster-master/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+# This role contains plays to install and configure the Cluster Master
+
+- name: Check if clustering is enabled
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk list cluster-peers -auth admin:{{ new_pass }}"
+  register: cluster_master_clustering_enabled
+  ignore_errors: True
+
+- name: Enable Cluster Master
+  command: runuser -l splunk -c "splunk edit cluster-config -mode master -replication_factor 3 -search_factor 2 -secret {{ replication_key }}"
+  when: cluster_master_clustering_enabled|failed
+  register: cluster_master_configure
+
+- name: Restart Cluster Master
+  command: runuser -l splunk -c "splunk restart"
+  when: cluster_master_clustering_enabled|failed
diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml
@@ -0,0 +1,5 @@
+---
+# This role contains common plays that will run on all nodes.
+
+- name: upgrade all packages
+  yum: name=* state=latest
diff --git a/roles/deployment-server/tasks/main.yml b/roles/deployment-server/tasks/main.yml
@@ -0,0 +1,3 @@
+---
+# This role contains plays to configure the Deployment Server 
+
diff --git a/roles/license-master/tasks/main.yml b/roles/license-master/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+# This role contains plays to configure the License Master
+
+- name: Create license folder
+  file: path=/opt/splunk/licenses state=directory owner=splunk group=splunk mode=744
+
+- name: Download license key from S3
+  s3: bucket={{ splunk-config-bucket }} object=/licenses/splunk.license dest=/opt/splunk/licenses/splunk.license mode=get
+
+- name: Install license key
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk add licenses /opt/splunk/licenses/splunk.license"
+  ignore_errors: True
+
+- name: Restart Splunk
+  command: runuser -l splunk -c "splunk restart"
+
diff --git a/roles/license-master/vars/main.yml b/roles/license-master/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# S3 bucket name containing Splunk licenses and configs
+splunk-config-bucket:
diff --git a/roles/peer-nodes/tasks/main.yml b/roles/peer-nodes/tasks/main.yml
@@ -0,0 +1,67 @@
+---
+# This role contains plays to install and configure the Peer Nodes
+
+- name: Check if clustering is enabled
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk list cluster-peers -auth admin:{{ new_pass }}"
+  register: peer_nodes_clustering_enabled
+  ignore_errors: True
+
+- name: Enable Peer Nodes
+  command: runuser -l splunk -c "splunk edit cluster-config -mode slave -master_uri https://{{ splunk_cluster_master_ip }}:8089 -replication_port 9887 -secret {{ replication_key }}"
+  when: peer_nodes_clustering_enabled|failed
+  register: peer_nodes_cluster_configure
+
+- name: Check if Splunk data volume exists
+  mount: name=/opt/splunk/data src=/dev/md127 fstype=ext4 state=mounted
+  register: splunk_volume_exists
+  ignore_errors: True
+
+- name: Gather EC2 facts
+  action: ec2_facts
+
+#- name: Prewarm EBS volume1
+#  command: dd if=/dev/zero of=/dev/sdf bs=1M
+#  when: splunk_volume_exists|failed
+#  ignore_errors: True
+
+#- name: Prewarm EBS volume2
+#  command: dd if=/dev/zero of=/dev/sdg bs=1M
+#  when: splunk_volume_exists|failed
+#  ignore_errors: True
+
+- name: Partition 90% of disk0 for use
+  shell: (echo n; echo p; echo 1; echo 2048; echo +720G; echo w) | fdisk /dev/xvdb 
+  when: splunk_volume_exists|failed
+
+- name: Partition 90% of disk1 for use
+  shell: (echo n; echo p; echo 1; echo 2048; echo +720G; echo w) | fdisk /dev/xvdc
+  when: splunk_volume_exists|failed
+
+- name: Create RAID 0 device
+  command: mdadm --create --verbose /dev/md127 --level=stripe --raid-devices=2 /dev/xvdb1 /dev/xvdc1
+  when: splunk_volume_exists|failed
+
+- name: Create filesystem
+  filesystem: fstype=ext4 dev=/dev/md127
+  when: splunk_volume_exists|failed
+
+- name: Create data directory
+  command: runuser -l splunk -c "mkdir -p /opt/splunk/data"
+  when: splunk_volume_exists|failed
+
+- name: Mount volume
+  mount: name=/opt/splunk/data src=/dev/md127 fstype=ext4 state=mounted
+  when: splunk_volume_exists|failed
+
+- name: Set default data store
+  command: runuser -l splunk -c "splunk set datastore-dir /opt/splunk/data -auth admin:{{ new_pass }}"
+
+- name: Change permissions of data mount point
+  command: chown -R splunk.splunk /opt/splunk/data
+
+- name: Disable splunkweb
+  command: runuser -l splunk -c "splunk disable webserver -auth admin:{{ new_pass }}"
+
+- name: Restart Peer Nodes
+  command: runuser -l splunk -c "splunk restart"
+  when: peer_nodes_cluster_configure|success
diff --git a/roles/peer-nodes/vars/main.yml b/roles/peer-nodes/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# IP address of the cluster master
+splunk_cluster_master_ip:
diff --git a/roles/search-heads/tasks/main.yml b/roles/search-heads/tasks/main.yml
@@ -0,0 +1,20 @@
+---
+# This role contains plays to install and configure the Search Heads
+
+- name: Check if clustering is enabled
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk list cluster-peers -auth admin:{{ new_pass }}"
+  register: search_head_clustering_enabled
+  ignore_errors: True
+
+- name: Enable Search Heads
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk edit cluster-config -mode searchhead -master_uri https://{{ splunk_cluster_master_ip }}:8089 -secret {{ replication_key }}"
+  when: search_head_clustering_enabled|failed
+  register: search_head_cluster_configure
+
+- name: Restart Search Heads
+  command: runuser -l splunk -c "splunk restart"
+
+- name: Install nfs-utils 
+  yum: name={{ item }} state=present
+  with_items:
+   - nfs-utils
diff --git a/roles/search-heads/vars/main.yml b/roles/search-heads/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# IP address of the cluster master
+splunk_cluster_master_ip:
diff --git a/roles/splunk-base/files/config_splunk_inputs.sh b/roles/splunk-base/files/config_splunk_inputs.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -v
+
+# Set environment variables
+export SPLUNK_HOST_NAME=`curl http://169.254.169.254/latest/meta-data/instance-id`
+export SPLUNK_HOME=/opt/splunk
+
+# Configure Splunk inputs.conf
+echo [default] > $SPLUNK_HOME/etc/system/local/inputs.conf
+echo host=$SPLUNK_HOST_NAME >> $SPLUNK_HOME/etc/system/local/inputs.conf
+chown splunk.splunk $SPLUNK_HOME/etc/system/local/inputs.conf
diff --git a/roles/splunk-base/handlers/main.yml b/roles/splunk-base/handlers/main.yml
diff --git a/roles/splunk-base/tasks/main.yml b/roles/splunk-base/tasks/main.yml
@@ -0,0 +1,150 @@
+---
+# This role contains plays to install and configure the base splunk instance
+
+- set_fact: new_password={{ new_pass }}
+- set_fact: current_password={{ current_pass }}
+- set_fact: ldap_password={{ ldap_pass }}
+
+- name: Update hostname in /etc/sysconfig/network
+  shell: sed -i "/HOSTNAME=localhost.localdomain/ c\\HOSTNAME=`curl http://169.254.169.254/latest/meta-data/instance-id`.{{ splunk_host_domain }}" /etc/sysconfig/network
+
+- name: Dynamically change hostname
+  shell: "hostname `curl http://169.254.169.254/latest/meta-data/instance-id`.{{ splunk_host_domain }}"
+
+- name: Install the GPG key for Splunk
+  rpm_key: state=present key={{ splunk_pgp_public_key }}
+
+- name: Check if Splunk is already installed
+  yum: name=splunk state=present
+  register: splunk_installed_result
+  ignore_errors: True
+
+- name: Check if Splunk is running
+  shell: /opt/splunk/bin/splunk status
+  register: splunk_running
+  ignore_errors: True
+
+- name: Download Splunk server binary
+  get_url: dest=/home/ec2-user  url={{ splunk_binary_url }} sha256sum={{ splunk_binary_sha256sum }}
+  when: splunk_installed_result|failed
+
+- name: Check that Splunk installer binary exists
+  stat: path=/home/ec2-user/{{ splunk_binary_file }}
+  register: splunk_installer_present
+
+- name: Install Splunk server binary
+  yum: pkg=/home/ec2-user/{{ splunk_binary_file }} state=installed
+  when: splunk_installer_present.stat.exists == true
+
+- name: Clean up RPM file
+  shell: rm -rf /home/ec2-user/{{ splunk_binary_file }}
+  when: splunk_installer_present.stat.exists == true
+
+- name: Copy scripts to run
+  copy: src="{{ item }}" dest="/home/ec2-user/{{ item }}" mode=755
+  with_items:
+   - config_splunk_inputs.sh
+  when: splunk_running|failed
+
+- name: Execute config_splunk_inputs.sh script
+  shell: /home/ec2-user/config_splunk_inputs.sh
+  when: splunk_running|failed
+
+- name: Clean up shell file
+  command: rm -rf /home/ec2-user/config_splunk_inputs.sh
+
+- name: Start Splunk for the first time
+  command: /bin/su --shell=/bin/bash --session-command="/opt/splunk/bin/splunk start --accept-license" splunk
+  when: splunk_running|failed
+
+- name: Enable Splunk to auto-start on boot
+  command: /bin/su --shell=/bin/bash --session-command="/opt/splunk/bin/splunk enable boot-start -user splunk"
+  when: splunk_running|failed
+
+- name: Set appropriate file ACLs for /var/log
+  command: /usr/bin/setfacl -m "u:splunk:r-x" /var/log
+  command: /usr/bin/setfacl -m "u:splunk:r--" /var/log/*
+  command: /usr/bin/setfacl -m d:user:splunk:r /var/log
+
+- name: Reset Splunk password
+  command: runuser -l splunk -c "/opt/splunk/bin/splunk edit user admin -password '{{ new_password }}' -role admin -auth admin:{{ current_password }}"
+  ignore_errors: True
+
+- name: Copy Splunk conf files
+  copy: src="{{ item }}" dest="/opt/splunk/etc/system/local/{{ item }}" mode=600 owner=splunk group=splunk
+  with_items:
+   - authentication.conf
+   - web.conf
+   - authorize.conf
+   - ui-prefs.conf
+   - alert_actions.conf
+
+- name: Creates cert directory
+  file: path=/opt/splunk/etc/auth/{{ company }} state=directory owner=splunk group=splunk mode=744 
+
+- name: Copy splunkweb cert and key
+  copy: src="{{ item }}" dest="/opt/splunk/etc/auth/{{ company }}/{{ item }}" mode=400 owner=splunk group=splunk
+  with_items:
+   - SplunkWebCert.pem
+   - SplunkWebPrivateKey.key
+
+- name: Decrypt splunkweb cert 
+  command: openssl aes-256-cbc -salt -a -d -in /opt/splunk/etc/auth/{{ company }}/{{ item }}
+           -out /opt/splunk/etc/auth/{{ company }}/decrypted.pem -k {{ cert_decryption_password }}
+           creates=/opt/splunk/etc/auth/{{ company }}/decrypted.pem
+  with_items:
+   - SplunkWebCert.pem
+
+- name: Decrypt splunkweb key
+  command: openssl aes-256-cbc -salt -a -d -in /opt/splunk/etc/auth/{{ company }}/{{ item }}
+           -out /opt/splunk/etc/auth/{{ company }}/decrypted.key -k {{ cert_decryption_password }}
+           creates=/opt/splunk/etc/auth/{{ company }}/decrypted.key
+  with_items:
+   - SplunkWebPrivateKey.key
+
+- name: Rename the decrypted cert 
+  command: mv /opt/splunk/etc/auth/{{ company }}/decrypted.pem /opt/splunk/etc/auth/{{ company }}/{{ item }} 
+           removes=/opt/splunk/etc/auth/{{ company }}/decrypted.pem
+  with_items:
+   - SplunkWebCert.pem
+
+- name: Rename the decrypted key
+  command: mv /opt/splunk/etc/auth/{{ company }}/decrypted.key /opt/splunk/etc/auth/{{ company }}/{{ item }}
+           removes=/opt/splunk/etc/auth/{{ company }}/decrypted.key
+  with_items:
+   - SplunkWebPrivateKey.key
+
+- name: Set configuration file permissions
+  file: path=/opt/splunk/etc/system/local/authentication.conf state=file owner=splunk group=splunk mode=0600
+  file: path=/opt/splunk/etc/system/local/web.conf state=file owner=splunk group=splunk mode=0600
+  file: path=/opt/splunk/etc/system/local/authorize.conf state=file owner=splunk group=splunk mode=0600
+  file: path=/opt/splunk/etc/system/local/ui-prefs.conf state=file owner=splunk group=splunk mode=0600
+  file: path=/opt/splunk/etc/system/local/alert_actions.conf state=file owner=splunk group=splunk mode=0600
+
+- name: Set certificate file permissions
+  file: path=/opt/splunk/etc/auth/{{ company }}/SplunkWebCert.pem state=file owner=splunk group=splunk mode=0400
+
+- name: Set key file permissions
+  file: path=/opt/splunk/etc/auth/{{ company }}/SplunkWebPrivateKey.key state=file owner=splunk group=splunk mode=0400
+
+- name: Update LDAP password
+  shell: sed -i 's/^bindDNpassword.*/bindDNpassword = {{ ldap_password }}/' /opt/splunk/etc/system/local/authentication.conf
+
+- name: Add nodes to license master
+  command: runuser -l splunk -c "splunk edit licenser-localslave -master_uri 'https://{{ splunk_license_master }}:8089' -auth admin:{{ new_pass }}"
+  ignore_errors: True
+
+- name: Restart Splunk
+  command: runuser -l splunk -c "splunk restart"
+  when: splunk_running|failed
+
+- name: Install packages for git, Splunk CPU usage reporting and Cloudwatch monitoring 
+  yum: name={{ item }} state=present
+  with_items:
+   - git
+   - sysstat
+   - yum-cron
+   - perl-Switch
+   - perl-Sys-Syslog
+   - perl-LWP-Protocol-https
+
diff --git a/roles/splunk-base/vars/main.yml b/roles/splunk-base/vars/main.yml
diff --git a/splunk-base.yml b/splunk-base.yml
diff --git a/splunk-site.yml b/splunk-site.yml

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +*.conf
 +*.pem
 +*.key
++
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+---`
	`2`	`+# This role contains plays to configure the Deployment Server`
	`3`	`+`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+---`
	`2`	`+# S3 bucket name containing Splunk licenses and configs`
	`3`	`+splunk-config-bucket:`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+---`
	`2`	`+# IP address of the cluster master`
	`3`	`+splunk_cluster_master_ip:`