First preparation to stretch

aelsharawi · Apr 29, 2017 · 8857e9d · 8857e9d
1 parent 44f3642
commit 8857e9d
Show file tree

Hide file tree

Showing 29 changed files with 171 additions and 71 deletions.
diff --git a/clone-server-snapshot.sh b/clone-server-snapshot.sh
@@ -31,7 +31,7 @@ ifdown eth0; ifup eth0
 mv -f /etc/resolv.conf.clone /etc/resolv.conf
 
 # Hostname
-# See: ${D}/debian-setup.sh
+# See: /debian-setup.sh
 
 # DNS A
 host -t A "$H"

diff --git a/debian-image-check.sh b/debian-image-check.sh
@@ -2,7 +2,7 @@
 #
 # Check Debian OS: jessie 8.x netinst (essential, required, important) and standard packages.
 #
-# VERSION       :1.0.7
+# VERSION       :1.0.8
 # DEPENDS       :apt-get install aptitude
 
 # Generated lists
@@ -26,10 +26,10 @@ export APT_LISTCHANGES_FRONTEND="none"
 APTI_SEARCH="aptitude --disable-columns --display-format %p search"
 
 {
-    ${APTI_SEARCH} '?and(?essential, ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(required), ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(important), ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(standard), ?not(?installed))' | grep -Evx "$STANDARD_BLACKLIST"
+    ${APTI_SEARCH} '?and(?essential, ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(required), ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(important), ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(standard), ?not(?installed), ?architecture(native))' | grep -Evx "$STANDARD_BLACKLIST"
 } 2>&1 | tee missing.pkgs | grep "." && echo "Missing packages" 1>&2
 
 {
@@ -42,6 +42,13 @@ APTI_SEARCH="aptitude --disable-columns --display-format %p search"
     dpkg -l | grep "\~[a-z]\+" | grep -Ev "^ii  (${TILDE_VERSION})\s" | cut -c 1-55 | sed -e 's/$/ # tilde version/'
     # "-dev" versioned packages
     ${APTI_SEARCH} '?and(?installed, ?name(-dev))' | sed -e 's/$/ # development/'
+    # Overridden priorities
+    aptitude --disable-columns --display-format "%p %P" search \
+        '?and(?installed, ?not(?automatic), ?not(?essential), ?not(?priority(required)), ?not(?priority(important)), ?not(?priority(standard)))' \
+        | while read -r PKG_PRIO; do
+            DPKG_PRIO="$(dpkg-query -s "${PKG_PRIO% *}" | grep "^Priority:" | cut -d " " -f 2)"
+            [ "$DPKG_PRIO" != "${PKG_PRIO#* }" ] && echo "${PKG_PRIO% *} # ${PKG_PRIO#* } <- ${DPKG_PRIO} override"
+        done
 } 2>&1 | tee extra.pkgs | grep "." && echo "Extra packages" 1>&2
 
 # List packages by size

diff --git a/debian-image-normalize.sh b/debian-image-normalize.sh
@@ -2,7 +2,7 @@
 #
 # Normalize Debian OS: jessie 8.x netinst (essential, required, important) and standard packages.
 #
-# VERSION       :1.0.7
+# VERSION       :1.0.8
 # DEPENDS       :apt-get install aptitude
 
 # Generated lists
@@ -23,10 +23,11 @@ STANDARD_BLACKLIST="exim.*|procmail|bsd-mailx|mutt\
 |ftp|mlocate|texinfo|info|install-info|debian-faq|doc-debian"
 
 # Don't ever remove these
-BOOT_PACKAGES="grub-pc|grub-efi-amd64|extlinux|syslinux-common|linux-image-amd64|initramfs-tools\
+BOOT_PACKAGES="debian-archive-keyring|lsb-release|ca-certificates|wget|apt|apt-utils|aptitude|aptitude-common|net-tools\
+|grub-pc|grub-efi-amd64|extlinux|syslinux-common|linux-image-amd64|initramfs-tools\
 |firmware-.*|usbutils|mdadm|lvm2|xfsprogs\
 |task-ssh-server|task-english|ssh|openssh-server|isc-dhcp-client|pppoeconf|ifenslave|ethtool|vlan\
-|sysvinit|sysvinit-core|sysvinit-utils|insserv|discover\
+|sysvinit|sysvinit-core|sysvinit-utils|initscripts|insserv|discover\
 |systemd|libpam-systemd|systemd-sysv|dbus\
 |open-vm-tools|open-vm-tools-dkms|dkms|sudo|cloud-init|cloud-initramfs-growroot\
 |elasticstack-container|waagent|scx|omi"
@@ -81,7 +82,7 @@ echo "tasksel tasksel/desktop multiselect" | debconf-set-selections -v
 echo "tasksel tasksel/first multiselect ssh-server, standard" | debconf-set-selections -v
 echo "tasksel tasksel/tasks multiselect ssh-server" | debconf-set-selections -v
 apt-get install -qq tasksel
-# May take a while
+# This may take a while
 tasksel --new-install
 
 Info "Mark dependencies of standard packages as automatic"
@@ -99,25 +100,26 @@ set -x
 Info "Install standard packages"
 
 STANDARD_PACKAGES="$(${APTI_SEARCH} \
- '?and(?not(?obsolete), ?or(?essential, ?priority(required), ?priority(important), ?priority(standard)))' \
+ '?and(?not(?obsolete), ?or(?essential, ?priority(required), ?priority(important), ?priority(standard)), ?architecture(native))' \
  | grep -Evx "$STANDARD_BLACKLIST")"
 # Native arch.
 #STANDARD_PACKAGES="$(${APTI_SEARCH} \
-# '?and(?architecture(native), ?or(?essential, ?priority(required), ?priority(important), ?priority(standard)))' \
+# '?and(?architecture(native), ?or(?essential, ?priority(required), ?priority(important), ?priority(standard)), ?architecture(native))' \
 # | grep -Evx "$STANDARD_BLACKLIST")"
 # shellcheck disable=SC2086
 apt-get -qq install ${STANDARD_PACKAGES}
 
 Info "Install missing recommended packages"
 
 MISSING_RECOMMENDS="$(${APTI_SEARCH} \
- '?and(?reverse-recommends(?installed), ?version(TARGET), ?not(?installed))' | grep -Evx "$STANDARD_BLACKLIST" || true)"
+ '?and(?reverse-recommends(?installed), ?version(TARGET), ?not(?installed), ?architecture(native))' | grep -Evx "$STANDARD_BLACKLIST" || true)"
 # shellcheck disable=SC2086
 apt-get -qq install ${MISSING_RECOMMENDS}
 echo "$MISSING_RECOMMENDS" | xargs -r -L 1 apt-mark auto
 
 Info "Remove non-standard packages"
 
+# See override.${CODENAME}.main.gz at http://ftp.debian.org/debian/indices/
 MANUALLY_INSTALLED="$(${APTI_SEARCH} \
  '?and(?installed, ?not(?automatic), ?not(?essential), ?not(?priority(required)), ?not(?priority(important)), ?not(?priority(standard)))' \
  | grep -Evx "$BOOT_PACKAGES" | tee removed.pkgs || true)"
@@ -148,10 +150,10 @@ Info "Check for missing and extra packages"
 set +e +x
 
 {
-    ${APTI_SEARCH} '?and(?essential, ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(required), ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(important), ?not(?installed))'
-    ${APTI_SEARCH} '?and(?priority(standard), ?not(?installed))' | grep -Evx "$STANDARD_BLACKLIST"
+    ${APTI_SEARCH} '?and(?essential, ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(required), ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(important), ?not(?installed), ?architecture(native))'
+    ${APTI_SEARCH} '?and(?priority(standard), ?not(?installed), ?architecture(native))' | grep -Evx "$STANDARD_BLACKLIST"
 } 2>&1 | tee missing.pkgs | grep "." && echo "Missing packages" 1>&2
 
 {
@@ -164,6 +166,13 @@ set +e +x
     dpkg -l | grep "\~[a-z]\+" | grep -Ev "^ii  (${TILDE_VERSION})\s" | cut -c 1-55 | sed -e 's/$/ # tilde version/'
     # "-dev" versioned packages
     ${APTI_SEARCH} '?and(?installed, ?name(-dev))' | sed -e 's/$/ # development/'
+    # Overridden priorities
+    aptitude --disable-columns --display-format "%p %P" search \
+        '?and(?installed, ?not(?automatic), ?not(?essential), ?not(?priority(required)), ?not(?priority(important)), ?not(?priority(standard)))' \
+        | while read -r PKG_PRIO; do
+            DPKG_PRIO="$(dpkg-query -s "${PKG_PRIO% *}" | grep "^Priority:" | cut -d " " -f 2)"
+            [ "$DPKG_PRIO" != "${PKG_PRIO#* }" ] && echo "${PKG_PRIO% *} # ${PKG_PRIO#* } <- ${DPKG_PRIO} override"
+        done
 } 2>&1 | tee extra.pkgs | grep "." && echo "Extra packages" 1>&2
 
 # List packages by size

diff --git a/debian-setup.sh b/debian-setup.sh
@@ -2,7 +2,7 @@
 #
 # Debian jessie setup on a virtual server.
 #
-# VERSION       :1.0.1
+# VERSION       :1.0.2
 # URL           :https://github.com/szepeviktor/debian-server-tools
 # AUTHOR        :Viktor Szépe <viktor@szepe.net>
 # LICENSE       :The MIT License (MIT)
@@ -14,14 +14,15 @@
 #
 # 1. wget -O- https://github.com/szepeviktor/debian-server-tools/archive/master.tar.gz|tar xz
 #    cd debian-server-tools-master/
-# 2. Aquire settings: hostname, networking, resolvers, NTP servers, custom kernel, user names, SSH keys
+# 2. Aquire settings: hostname, networking, DNS resolvers, NTP servers, custom kernel, user names, SSH keys
 # 3. Compile /root/server.yml from /server.yml and from /debian-setup/providers/*.yml
 # 4. Set up DNS resource records: PTR, A, AAAA, MX (domain registrar, DNS provider)
-# 5. Start!
+# 5. Set volume labels:  lsblk -f;tune2fs -L "instanceID-root" /dev/sda1
+# 6. Start!
 #    script --timing=debian-setup.time debian-setup.script
 #    ./debian-setup.sh
-# 6. Consider creating a disk or vm template with isc-dhcp-client installed
-# 7. Continue!
+# 7. Consider creating a disk or vm template with isc-dhcp-client installed
+# 8. Continue!
 #    @FIXME  export MONIT_EXCLUDED_PACKAGES=apache2:php5-fpm:php7.0-fpm
 #    script --timing=debian-setup2.time debian-setup2.script
 #    ./debian-setup2.sh
@@ -91,14 +92,14 @@ export SETUP_SHYAML_URL="https://github.com/szepeviktor/debian-server-tools/raw/
 set -e -x
 
 # Am I root?
-[ "$(id -u)" == 0 ]
+test "$(id -u)" == 0
 
 # Common functions
 source debian-setup-functions
 
 # Necessary packages
 IS_FUNCTIONAL="yes"
-[ -n "$(which dpkg-query)" ]
+test -n "$(which dpkg-query)"
 for PKG in ${SETUP_PACKAGES}; do
     if ! Is_installed "$PKG"; then
         IS_FUNCTIONAL="no"
@@ -109,9 +110,9 @@ if [ "$IS_FUNCTIONAL" != "yes" ]; then
     apt-get update -qq || true
     # shellcheck disable=SC2086
     apt-get install -y --force-yes ${SETUP_PACKAGES} || true
-    # These packages should be auto-installed
-    apt-mark auto lsb-release ca-certificates || true
 fi
+# These packages should be auto-installed
+apt-mark auto lsb-release ca-certificates
 
 # Package sources
 debian-setup/apt
@@ -129,7 +130,7 @@ if Is_installed "libgnutls26"; then
         libprocps0 libtasn1-3 libudev0 python2.6 python2.6-minimal
 fi
 # Remove ClamAV data
-rm -rf /var/lib/clamav /var/log/clamav || true
+rm -rf /var/lib/clamav /var/log/clamav
 
 # Packages used on top of SETUP_PACKAGES
 apt-get install -qq ssh sudo apt-transport-https virt-what python-yaml
@@ -146,7 +147,7 @@ eval "$(grep -h -A 5 "^deb " /etc/apt/sources.list.d/*.list | grep "^#K: " | cut
 # Get package lists
 apt-get update -qq
 
-IP="$(ifconfig | sed -n -e '0,/^\s*inet addr:\([0-9\.]\+\)\b.*$/s//\1/p')"
+IP="$(ifconfig|sed -n -e '0,/^\s*inet \(addr:\)\?\([0-9\.]\+\)\b.*$/s//\2/p')"
 export IP
 
 # Virtualization environment

diff --git a/debian-setup/_check-system b/debian-setup/_check-system
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Manual system checks
+# Manual system checks.
 #
 
 Titled_pager() {

diff --git a/debian-setup/adduser b/debian-setup/adduser
@@ -69,7 +69,7 @@ cat > /root/.bashrc <<"EOF"
 
 #export IP="$(ip addr show dev eth0|sed -n -e 's/^\s*inet \([0-9.]\+\)\b.*$/\1/p')"
 #export IP="$(ifconfig|sed -n -e '/^eth/{n;s/^\s*inet addr:\([0-9.]\+\)\b.*$/\1/p;q}')"
-export IP="$(ifconfig|sed -n -e '0,/^\s*inet addr:\([0-9\.]\+\)\b.*$/s//\1/p')"
+export IP="$(ifconfig|sed -n -e '0,/^\s*inet \(addr:\)\?\([0-9\.]\+\)\b.*$/s//\2/p')"
 export LS_OPTIONS='--color=auto'
 eval "$(dircolors)"
 alias ls='ls $LS_OPTIONS'

diff --git a/debian-setup/login b/debian-setup/login
@@ -3,6 +3,6 @@
 set -e -x
 
 # Create missing lastlog
-if ! [ -f /var/log/lastlog ]; then
+if [ ! -f /var/log/lastlog ]; then
     install -o root -g utmp -m 0664 /dev/null /var/log/lastlog
 fi
diff --git a/debian-setup/providers/_provider-datacenter.yml b/debian-setup/providers/_provider-datacenter.yml
@@ -12,7 +12,7 @@ networking:
     method: static
     name: eth0
     ipv4: 1.2.3.4
-    netmask: 255.255.252.0
+    netmask: 255.255.255.0
     gateway: 1.2.3.255
     resolver:
         # DNS Advantage by Neustar
@@ -21,7 +21,7 @@ networking:
         - $local.NS1
         # rdns2.ultradns.net.
         #- 156.154.71.1
-        # DNS.WATCH / Ideal-Hosting
+        # DNS.WATCH by Ideal-Hosting
         # resolver1.dns.watch
         #- 84.200.69.80
         #- 2001:1608:10:25::1c04:b12f
@@ -47,8 +47,8 @@ networking:
     ntp:
         - ntp.ezit.hu.
 software:
-    ovhrc: Instance data in /etc/ovhrc
-    serclient-deb: Aruba ExtraControl
+    ovhrc: "Instance data in /etc/ovhrc"
+    serclient-deb: "Aruba ExtraControl"
 documentation:
     - http://help.ovh.com/InstallOvhKey
     - http://help.ovh.com/RealTimeMonitoring

diff --git a/debian-setup/providers/atw-alfa.yaml b/debian-setup/providers/atw-alfa.yaml
@@ -1,4 +1,4 @@
-# in Alfa Victor Hugo Office
+# ATW in Alfa Victor Hugo Office
 hosting-company: ATW
 virtualization: kvm
 networking:

diff --git a/debian-setup/providers/contabo-mcn.yml b/debian-setup/providers/contabo-mcn.yml
@@ -0,0 +1,29 @@
+# Contabo in München
+hosting-company: Contabo
+virtualization: kvm
+networking:
+    method: static
+    name: eth0
+    resolver:
+        # München
+        - 79.143.183.251
+        # MAXNS=3
+        #- 79.143.183.252
+        # Google Public DNS
+        - 8.8.8.8
+    inet6:
+        method: static
+        name: eth0
+        resolver:
+            # Contabo München
+            - 2a02:c205::1:53
+            # MAXNS=3
+            #- 2a02:c205::2:53
+provider-package:
+    # Use busybox in initramfs
+    - busybox
+#auto-check-system: true
+dist-mod:
+    - "sync" cron job
+    - sysctl "kernel.panic = 10"
+    - volume label file "/vmi000000"
diff --git a/debian-setup/providers/contabo-nug.yml b/debian-setup/providers/contabo-nug.yml
@@ -0,0 +1,29 @@
+# Contabo in Nürnberg
+hosting-company: Contabo
+virtualization: kvm
+networking:
+    method: static
+    name: eth0
+    resolver:
+        # Contabo Nürnberg
+        - 213.136.95.10
+        # MAXNS=3
+        #- 213.136.95.11
+        # Google Public DNS
+        - 8.8.8.8
+    inet6:
+        method: static
+        name: eth0
+        resolver:
+            # Contabo Nürnberg
+            - 2a02:c207::1:53
+            # MAXNS=3
+            #- 2a02:c207::2:53
+provider-package:
+    # Use busybox in initramfs
+    - busybox
+#auto-check-system: true
+dist-mod:
+    - "sync" cron job
+    - sysctl "kernel.panic = 10"
+    - volume label file "/vmi000000"
diff --git a/debian-setup/providers/upcloud-defra1.yml b/debian-setup/providers/upcloud-defra1.yml
@@ -1,5 +1,4 @@
 # UpCloud defra1
-# UpCloud datacenters: defra1, fihel1, uklon1, uschi1
 hosting-company: UpCloud
 virtualization: kvm
 networking:

diff --git a/debian-setup/providers/upcloud-fihel1.yml b/debian-setup/providers/upcloud-fihel1.yml
@@ -1,5 +1,4 @@
 # UpCloud fihel1
-# UpCloud datacenters: defra1, fihel1, uklon1, uschi1
 hosting-company: UpCloud
 virtualization: kvm
 networking:

diff --git a/debian-setup/providers/vultr-fra.yml b/debian-setup/providers/vultr-fra.yml
@@ -4,6 +4,8 @@ virtualization: kvm
 networking:
     method: dhcp
     name: eth0
+    resolver:
+        - 108.61.10.10
     # IPv6 interface
     inet6:
         method: auto
@@ -14,8 +16,6 @@ networking:
 #    private:
 #        method: dhcp
 #        name: eth1
-    resolver:
-        - 108.61.10.10
 provider-package:
     # Provides VirtIO RNG
     - rng-tools

diff --git a/debian-setup/tzdata b/debian-setup/tzdata
@@ -5,11 +5,13 @@ set -e -x
 # UTC timezone
 # http://yellerapp.com/posts/2015-01-12-the-worst-server-setup-you-can-make.html
 cat /etc/timezone
+
 echo "tzdata tzdata/Areas select Etc" | debconf-set-selections -v
 echo "tzdata tzdata/Zones/Etc select UTC" | debconf-set-selections -v
+# https://serverfault.com/a/332490/213862
 
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=704089
-rm -f /etc/timezone
+rm /etc/timezone
 
 dpkg-reconfigure -f noninteractive tzdata
 

diff --git a/debian-setup2.sh b/debian-setup2.sh
@@ -24,7 +24,7 @@ set -e -x
 VIRT="$(Data get-value virtualization)"
 export VIRT
 
-IP="$(ifconfig | sed -n -e '0,/^\s*inet addr:\([0-9\.]\+\)\b.*$/s//\1/p')"
+IP="$(ifconfig|sed -n -e '0,/^\s*inet \(addr:\)\?\([0-9\.]\+\)\b.*$/s//\2/p')"
 export IP
 
 # _check-system needs most
@@ -39,7 +39,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -q -y \
     localepurge unattended-upgrades apt-listchanges debsums \
     ncurses-term mc most less time moreutils unzip \
     logtail apg dos2unix ccze colordiff \
-    whois ntpdate ipset netcat-openbsd lftp heirloom-mailx \
+    net-tools whois ntpdate ipset netcat-openbsd lftp heirloom-mailx \
     gcc libc6-dev make strace \
 
 # From backports