Idle culler

.gitmodules

@@ -3,7 +3,19 @@
 	url = https://github.com/jtyr/ansible-udev_rename_netiface.git
 [submodule "ansible/roles/network_interface"]
 	path = ansible/roles/network_interface
-	url = https://github.com/MartinVerges/ansible.network_interface.git
+	url = https://github.com/mit-scripts/ansible.network_interface.git
 [submodule "ansible/roles/pacemaker-corosync"]
 	path = ansible/roles/pacemaker-corosync
 	url = https://github.com/mit-scripts/ansible-pacemaker-corosync.git
+[submodule "ansible/roles/proxy-munin-node/files/monitoring-munin-haproxy"]
+	path = ansible/roles/proxy-munin-node/files/monitoring-munin-haproxy
+	url = https://github.com/mit-scripts/monitoring-munin-haproxy.git
+[submodule "ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy"]
+	path = ansible/roles/ldirectord-status/files/ldirectord-status/gnlpy
+	url = https://github.com/facebook/gnlpy.git
+[submodule "ansible/roles/jupyter-jupyter/files/webathena"]
+	path = ansible/roles/jupyter-jupyter/files/webathena
+	url = https://github.com/davidben/webathena
+[submodule "ansible/collections/community.general"]
+	path = ansible/collections/community.general
+	url = https://github.com/ansible-collections/community.general.git

ansible/.gitignore

@@ -1 +1,2 @@
 *.retry
+*.pyc

ansible/ansible.cfg

@@ -1,6 +1,12 @@
 [defaults]
-inventory = inventory.yml
+inventory = inventory
 remote_user = root
+force_handlers = True
+callback_whitelist = log_plays
+stdout_callback = yaml
+conditional_bare_variables = False
+interpreter_python = auto
+collections_paths = ./collections
 
 [ssh_connection]
 pipelining = True

ansible/files/conntrack

@@ -0,0 +1,4 @@
+[fw_conntrack]
+command /bin/false
+[fw_forwarded_local]
+command /bin/false

ansible/files/dotfiles/.bashrc

@@ -0,0 +1,23 @@
+# .bashrc
+
+# User specific aliases and functions
+
+alias rm='rm -i'
+alias cp='cp -i'
+alias mv='mv -i'
+
+DEFAULTVISUAL=emacs
+if [ "$SSH_GSSAPI_NAME" = "adehnert/root@ATHENA.MIT.EDU" ]; then
+        DEFAULTVISUAL=vim
+fi
+export VISUAL=${VISUAL:-$DEFAULTVISUAL}
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+        . /etc/bashrc
+fi
+
+alias vi=vim
+alias view='vim -R'
+
+logger -p authpriv.warning -t bash -- "Root bash shell for ${SSH_GSSAPI_NAME:-unknown} from ${SSH_CLIENT:-local}"

ansible/files/dotfiles/.emacs

@@ -0,0 +1,24 @@
+;; .emacs
+
+(custom-set-variables
+ ;; uncomment to always end a file with a newline
+ ;'(require-final-newline t)
+ ;; uncomment to disable loading of "default.el" at startup
+ ;'(inhibit-default-init t)
+ ;; default to unified diffs
+ '(diff-switches "-u"))
+
+;;; uncomment for CJK utf-8 support for non-Asian users
+;; (require 'un-define)
+
+; show column numbers
+(setq column-number-mode t)
+
+; use spaces, not tabs
+(setq-default indent-tabs-mode nil)
+(setq-default tab-width 4)
+(setq indent-line-function 'insert-tab)
+
+; recognize python executables
+(add-to-list 'interpreter-mode-alist
+             '("python2" . python-mode))

ansible/files/dotfiles/.ldapvirc

@@ -0,0 +1,11 @@
+profile default
+host ldap://scripts-ldap.mit.edu/
+base dc=scripts,dc=mit,dc=edu
+# kinit -k -t /etc/signup.keytab daemon/scripts-signup.mit.edu
+bind sasl
+sasl-mech GSSAPI
+
+profile local
+host ldapi://%2fvar%2frun%2fslapd-scripts.socket/
+bind sasl
+sasl-mech EXTERNAL

ansible/files/dotfiles/.screenrc

@@ -0,0 +1,6 @@
+startup_message off
+msgwait 1
+hardstatus string "[screen %n*%f %t] %h"
+caption always "%{= bW}%H  %{+ c}%-Lw%50>%?%F%{+b W}%:%{+ w}%?%n*%f %t%{-}%+Lw%<%-010=%{+ W}"
+altscreen on
+defbce on

ansible/files/dotfiles/.vimrc

@@ -0,0 +1,3 @@
+set background=dark
+set nocompatible
+syntax on

ansible/files/scripts-syslog.conf

@@ -1,4 +1,24 @@
-if \
+ruleset(name="zpublic") {
+  |/run/zephyr-syslog-public;RSYSLOG_SyslogProtocol23Format
+  stop
+}
+ruleset(name="zprivate") {
+  |/run/zephyr-syslog-private;RSYSLOG_SyslogProtocol23Format
+  stop
+}
+
+# Putting zroot in a queue means we can use the "stop" operator
+# without affecting file output.
+# See https://www.rsyslog.com/doc/v8-stable/rainerscript/rainerscript_call.html
+ruleset(name="zroot" queue.type="Direct") {
+  # https://www.rsyslog.com/doc/v8-stable/rainerscript/control_structures.html
+  # https://rainer.gerhards.net/2012/10/how-to-use-rsyslogs-ruleset-and-call-statements.html
+  # https://www.rsyslog.com/doc/v8-stable/configuration/filters.html
+
+  # $msg always has a leading space: https://www.rsyslog.com/log-normalization-and-the-leading-space/
+
+  # First, audit-related messages go to scripts-auto
+  if \
     ($programname == 'sshd' and ( \
         $msg startswith ' Authorized to root, ' \
         or \
@@ -8,21 +28,93 @@ if \
         or \
         $msg == ' pam_unix(sshd:session): session closed for user root' \
     )) \
-then |/run/zephyr-syslog-public;RSYSLOG_SyslogProtocol23Format
+  then {
+    call zpublic
+  }
+  # TODO: Look up ssh keys and annotate with whose key it is
+  # Publicly log all root sessions, except cron or sudo
+  if (re_match($msg, '^ pam_unix\\([^:]+:session\\): session \\S+ for user root')) then {
+    # Ignore all PAM session messages from cron
+    if ($programname == 'CRON') then stop
+    # sudo logs invocations itself with more information; ignore the
+    # PAM messages it generates.
+    if ($programname == 'sudo') then stop
+    # systemd --user can arbitrarily start PAM sessions; the
+    # underlying login session will trigger its own PAM logs so no
+    # need to report it twice.
+    if ($programname == 'systemd') then stop
+    call zpublic
+  }
+  if (re_match($msg, 'Root (\\S+) shell')) then call zpublic
+  if ($msg startswith ' Out of memory:') then call zpublic
+  if ($programname == 'admof') then call zpublic
+  # TODO: Spew when root runs su or sudo?
+
+  # Next, ignore known-safe chatty messages (list taken from the old
+  # d_zroot.pl, with some F30 rewordings added)
+  if (re_match($msg, '^ pam_unix\\([^:]+:session\\): session')) then stop
+  if ($programname == 'sshd') then {
+    if ($msg startswith ' Authorized to ') then stop
+    if ($msg startswith ' Accepted ') then stop
+    if ($msg startswith ' Connection closed') then stop
+    if ($msg startswith ' Closing connection to') then stop
+    if ($msg startswith ' Starting session: ') then stop
+    if ($msg startswith ' Close session: ') then stop
+    if (re_match($msg, '^ Connection from \\S+ port \\S+')) then stop
+    if ($msg startswith ' Invalid user') then stop
+    if ($msg startswith ' Disconnecting invalid user') then stop
+    if ($msg startswith ' input_userauth_request: invalid user') then stop
+    if ($msg startswith ' userauth_hostbased mismatch: ') then stop
+    if ($msg startswith ' Received disconnect from ') then stop
+    if ($msg startswith ' Disconnected from ') then stop
+    if ($msg startswith ' Postponed keyboard-interactive') then stop
+    if ($msg startswith ' Postponed gssapi-with-mic for ') then stop
+    if ($msg startswith ' Failed keyboard-interactive/pam') then stop
+    if ($msg startswith ' fatal: Read from socket failed: Connection reset by peer') then stop
+    if ($msg startswith ' error: kex_exchange_identification: read: Connection reset by peer') then stop
+    if ($msg startswith ' error: kex_exchange_identification: read: Connection closed by remote host') then stop
+    if ($msg startswith ' error: kex_exchange_identification: Connection closed by remote host') then stop
+    if ($msg startswith ' Connection reset by ') then stop
+    if ($msg startswith ' reverse mapping checking getaddrinfo') then stop
+    if ($msg startswith ' pam_succeed_if(sshd:auth):') then stop
+    if ($msg startswith ' error: PAM: Authentication failure') then stop
+    if ($msg startswith ' pam_unix(sshd:auth): authentication failure') then stop
+    if ($msg startswith ' pam_unix(sshd:auth): check pass; user unknown') then stop
+    if (re_match($msg, '^ Address \\S+ maps to \\S+, but this does not map back to the address')) then stop
+    if (re_match($msg, '^ Nasty PTR record .* is set up for .*, ignoring')) then stop
+    if ($msg startswith ' User child is on pid ') then stop
+    if (re_match($msg, '^ Accepted \\S+ public key \\S+ from \\S+$')) then stop
+    if ($msg startswith ' error: maximum authentication attempts exceeded for ') then stop
+  }
+  if (re_match($msg, '^ Transferred: sent \\d+, received \\d+ bytes$')) then stop
+  if ($msg == ' Setting tty modes failed: Invalid argument') then stop
+  if ($programname == 'sudo') then {
+    if (re_match($msg, '^ *nrpe .* COMMAND=/etc/nagios/check_ldap_mmr.real$')) then stop
+    if (re_match($msg, '^ *scripts : .*; USER=root ; COMMAND=/etc/httpd/export-scripts-certs$')) then stop
+    if (re_match($msg, '^ *pony : .*; USER=root ; COMMAND=/etc/pki/tls/gencsr-pony ')) then stop
+    if (re_match($msg, '^ *root : TTY=')) then stop
+  }
+  if ($msg startswith ' Set /proc/self/oom_adj to ') then stop
+  if ($msg startswith ' Set /proc/self/oom_score_adj to ') then stop
+  if ($msg == ' selinux sandbox not useful [preauth]') then stop
+  if ($programname == 'postfix') then {
+    if (re_match($msg, '^ warning: hostname .* does not resolve to address .*:')) then stop
+  }
 
+  # Everything else goes to scripts-spew
+  call zprivate
+}
+
+# Send errors, authpriv, and OOM events to the zroot queue
 if \
-    $syslogseverity <= '4' \
-    and \
-    not ($programname == 'sshd' and ( \
-        $msg == ' pam_unix(sshd:auth): check pass; user unknown' \
-        or \
-        $msg startswith ' PAM service(sshd) ignoring max retries; ' \
-        or \
-        $msg startswith ' error: maximum authentication attempts exceeded for ' \
-        or \
-        $msg startswith ' error: Received disconnect from ' \
-    )) \
-then |/run/zephyr-syslog-private;RSYSLOG_SyslogProtocol23Format
+  $syslogseverity <= '4' \
+  or \
+  $syslogfacility-text == 'authpriv' \
+  or \
+  ($syslogfacility-text == 'kern' and ($msg contains 'Out of memory:' or $msg contains 'Killed process')) \
+then {
+  call zroot
+}
 
 $ModLoad imrelp
 $InputRELPServerRun 2514

ansible/files/zephyr-syslog

@@ -88,14 +88,14 @@ facilities = [
 ]
 
 severity_symbols = [
-    '@b(@color(magenta)☠)',
-    '@b(@color(magenta)☣)',
-    '@b(@color(magenta)☢)',
-    '@b(@color(red)⊗)',
-    '@b(@color(yellow)⚠)',
-    '@b(@color(blue)☞)',
-    '@b(@color(cyan)ⓘ)',
-    '@b(@color(green)☻)'
+    '@b(@color(magenta)EMERG)',
+    '@b(@color(magenta)ALERT)',
+    '@b(@color(magenta)CRIT)',
+    '@b(@color(red)ERR)',
+    '@b(@color(yellow)WARN)',
+    '@b(@color(blue)NOTICE)',
+    '@b(@color(cyan)INFO)',
+    '@b(@color(green)DEBUG)'
 ]
 
 syslog_re = re.compile(r'''^<(?P<pri>\d+)>(?P<version>1) (?P<timestamp>\S*) (?P<hostname>\S*) (?P<app_name>\S*) (?P<procid>\S*) (?P<msgid>\S*) (?P<sd>(?:\[[^]= "]+(?: [^]= "]+="(?:[^]"\\]|\\.)*")*\])*|-) (?P<msg>.*)$''')

ansible/filter_plugins/permute.py

@@ -0,0 +1,13 @@
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+def permute(value, shift):
+    shift %= len(value)
+    return value[shift:] + value[:shift]
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'permute': permute,
+        }

ansible/filter_plugins/subnetmath.py

@@ -0,0 +1,88 @@
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import netaddr
+from itertools import groupby
+
+def _round_prefixes(value):
+    """Takes a list of subnets, and produces a new list of subnets that are /8, /16, or /24."""
+    _ret = []
+    for net in netaddr.cidr_merge(netaddr.IPNetwork(v) for v in value):
+        newprefix = ((net.prefixlen+7)//8)*8
+        _ret.extend(net.subnet(newprefix))
+    return _ret
+
+def inaddr_zones(value):
+    """inaddr_zones converts a list of IP subnets into a list of in-addr.arpa zone names that cover the subents."""
+    nets = _round_prefixes(value)
+    _ret = []
+    for net in nets:
+        val = "in-addr.arpa"
+        addr = int(net.network)
+        for i in range(0, net.prefixlen, 8):
+            val = str((addr >> (24-i)) & 0xff) + '.' + val
+        _ret.append(val)
+    return _ret
+
+def ipsubnets_regex(value):
+    """ipsubnets_regex converts a list of IP subnets into a regex that matches IP addresses on those subnets."""
+    nets = _round_prefixes(value)
+    prefixes = [net.network.ipv4().format().split('.')[:net.prefixlen//8] for net in nets]
+    return '^' + _prefixes_to_regex(prefixes) + r'\.'
+
+def _prefixes_to_regex(prefixes):
+    """
+    Convert a list of tuples containing IP prefixes into a regex that matches them.
+
+    Args:
+      prefixes: list of prefixes like [(10,), (18, 1), (18, 2)]
+
+    Returns:
+      regex like "(10|18\.[1-2])"
+    """
+    out = []
+    if max(len(x) for x in prefixes) == 1:
+        # Last component, try to use character classes
+        return _numbers_regex(x[0] for x in prefixes)
+    for octet, g in groupby(prefixes, lambda x: x[0]):
+        sub = [x[1:] for x in g if len(x) > 1]
+        match = str(octet)
+        if sub:
+            match += r'\.' + _prefixes_to_regex(sub)
+        out.append(match)
+    return '(' + '|'.join(out) + ')'
+
+def _numbers_regex(numbers):
+    """Find a simplified regex for matching a list of numbers"""
+    def key(x): return (x[0], len(x[1]), x[1][:-1])
+    numbers = sorted((('', str(x)) for x in numbers), key=key)
+    simplified = False
+    while not simplified:
+        simplified = True
+        out = []
+        for (suffix, _, prefix), g in groupby(numbers, key):
+            g = list(g)
+            if len(g) == 1 and not g[0][1]:
+                out.append(g[0])
+                continue
+            simplified = False
+            digits = sorted(x[1][-1] for x in g)
+            if len(digits) == 1:
+                match = digits[0]
+            elif len(digits) == 10:
+                match = r'\d'
+            else:
+                match = '['+''.join(digits)+']'
+            out.append((match+suffix, prefix))
+        numbers = out
+    if len(numbers) == 1:
+        return numbers[0][0]
+    return '('+ '|'.join(x[0] for x in numbers) + ')'
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'inaddr_zones': inaddr_zones,
+            'ipsubnets_regex': ipsubnets_regex,
+        }