diff --git a/Maintenance.md b/Maintenance.md index d145d36c..3dd7907c 100644 --- a/Maintenance.md +++ b/Maintenance.md @@ -1,28 +1,31 @@ ## Honlap karbantartás a részleges és teljes leállások megelőzésére - betörés (malware fertőzés) esetén takarítás -- * biztonsági mentés fájlokról, adatbázisról naponta +- * biztonsági mentés fájlokról, adatbázisról naponta (5 GB-ig) - magas fokú honlap biztonság, fájl változás jelentés 30 percenként - domain név lejárat figyelés, új regisztráció, módosítás - DNS (domain név kiszolgáló) beállítás, óránkénti monitorozás - főoldali tartalom óránkénti monitorozás és hiba észlelés - honlap elérhetőség monitorozás 1 percenként -- * PHP és MySQL futás monitorozás +- * PHP és MySQL futás monitorozás óránként - WMT (Google Webmestereszközök) teljes rendberakása hetente - * nem létező tartalomra mutató linkek (404) javítása naponta ### Honlap gyorsítás SEO-hoz -- PageSpeed 80+ ponton tartása -- * tartalom kiszolgáló hálózat (CDN) használat [Amazon CloudFront](http://aws.amazon.com/cloudfront/pricing/) (max. 20 GB forgalom havonta) +- PageSpeed 80+ ponton tartása (ha van mobil nézete a honlapnak, akkor ott is) +- * tartalom kiszolgáló hálózat (CDN) használat [Amazon CloudFront](http://aws.amazon.com/cloudfront/pricing/) (20 GB havi forgalomig) - * valós oldal betöltési idő alacsonyan tartása - böngésző gyorsítótárazás beállítása -- napi SEO kimutatás - grafikon a Google keresőben való helyezésről +- SEO kimutatás a Google keresőben való helyezésről grafikonon és emailben naponta - alapvető SEO hiányosságokról javaslat emailben ### Kód karbantartás - WordPress (honlap motor) frissítés -- * HTML hibák javítása -- * Google Analytics és más követő kódok, beágyazott mérőkódok beállítása, ellenőrzése -- * újonnan felvitt tartalom ellenőrzése (tipográfia és HTML használat) +- WordPress bővítmények frissítése +- * HTML hibák javítása (amelyekhez nem kell újraírni a honlapot) +- * Google Analytics és más követő kódok, beágyazott mérőkódok beállítása és ellenőrzése +- * újonnan felvitt tartalom ellenőrzése hetente (tipográfia és HTML használat) + +A csillagozottak ( * ) csak saját szerveren (VPS) valósíthatók meg. diff --git a/backup/hubic b/backup/hubic index 500ce7d5..e48e7f2d 160000 --- a/backup/hubic +++ b/backup/hubic @@ -1 +1 @@ -Subproject commit 500ce7d587f0f7364e7e13306bd5785f384dbda0 +Subproject commit e48e7f2dd4578b7b5244b83771addce04847e528 diff --git a/backup/obnam.sh b/backup/obnam.sh old mode 100644 new mode 100755 diff --git a/debian-setup.sh b/debian-setup.sh index 29bfd3bc..828a4d05 100644 --- a/debian-setup.sh +++ b/debian-setup.sh @@ -3,7 +3,9 @@ # Debian server setup - wheezy amd64 # Not a script but a manual. # -# OVH VPS data: /etc/ovhrc +# OVH /etc/ovhrc +# cdns.ovh.net. +# ntp.ovh.net # How to choose VPS provider? # disk access time @@ -16,6 +18,11 @@ exit 0 +# Whitelist outgoing SMTP server sending notifications. +# /etc/courier/smtpaccess/default +1.2.3.4 allow,RELAYCLIENT + + # download this repo git clone https://github.com/szepeviktor/debian-server-tools.git cd debian-server-tools/ && git submodule init && git submodule update @@ -30,7 +37,6 @@ apt-get autoremove --purge -y # packages sources nano /etc/apt/sources.list - # Linode: http://mirrors.linode.com/debian # OVH: http://debian.mirrors.ovh.net/debian # server4you: http://debian.intergenia.de/debian @@ -48,12 +54,11 @@ deb wheezy-backports main # disable apt languages echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/00languages -# thrrottle package downloads (1000 kB/s) +# throttle package downloads (1000 kB/s) echo 'Acquire::Queue-mode "access"; Acquire::http::Dl-Limit "1000";' > /etc/apt/apt.conf.d/76download # upgrade -apt-get update -apt-get dist-upgrade -y +apt-get update && apt-get dist-upgrade -y apt-get install -y ssh sudo ca-certificates most lftp bash-completion htop bind9-host mc lynx ncurses-term ln -sv /usr/bin/host /usr/local/bin/mx @@ -61,7 +66,6 @@ ln -sv /usr/bin/host /usr/local/bin/mx echo "alias e='mcedit'" > /etc/profile.d/editor.sh || echo "ERROR: alias 'e'" sed -i 's/^# \(".*: history-search-.*ward\)$/\1/' /etc/inputrc || echo "ERROR: history-search-backward" sed -e 's/\(#.*enable bash completion\)/#\1/' -e '/#.*enable bash completion/,+8 { s/^#// }' -i /etc/bash.bashrc || echo "ERROR: bash completion" -echo -e "\ncontent_disposition = on" >> /etc/wgetrc update-alternatives --set pager /usr/bin/most update-alternatives --set editor /usr/bin/mcedit # ### Markdown for mc ### @@ -70,40 +74,42 @@ update-alternatives --set editor /usr/bin/mcedit # regex/\.md(own)?$ # View=pandoc -s -f markdown -t man %p | man -l - -# bash as default shell +# bash echo "dash dash/sh boolean false"|debconf-set-selections -v dpkg-reconfigure -f noninteractive dash - -# bashrc nano /root/.bashrc export PS1="[\[$(tput setaf 3)\]\u\[\033[1;31m\]@\h\[$(tput sgr0)\]:\[$(tput setaf 8)\]\[$(tput setab 4)\]\ \w\[$(tput sgr0)\]:\t:\[$(tput setaf 0)\]\!\[$(tput sgr0)\]]\n" -# ls -1 /usr/share/mc/skins/ export GREP_OPTIONS="--color" alias grep='grep $GREP_OPTIONS' alias iotop='iotop -d 0.1 -qqq -o' alias iftop='NCURSES_NO_UTF8_ACS=1 iftop -nP' # putty / Connection / Data / Terminal-type string: putty-256color +# ls -1 /usr/share/mc/skins/ export MC_SKIN="modarin256root-defbg" alias transit='xz -9|base64 -w $((COLUMNS-1))' alias transit-receive='base64 -d|xz -d' +# wget defaults. +echo -e "\ncontent_disposition = on" >> /etc/wgetrc -# user +# Username. U="viktor" adduser $U -# enter password... +# Enter password twice. K="" S="/home/$U/.ssh"; mkdir --mode 700 "$S"; echo "$K" >> "${S}/authorized_keys2"; chown -R $U:$U "$S" adduser $U sudo -# remove root password +# Remove root etc. passwords. nano /etc/shadow -# sshd on another port +# sshd on another port. sed 's/^Port 22$/#Port 22\nPort 3022/' -i /etc/ssh/sshd_config -# disable password login for sudoers +# Disable root login. +sed 's/^PermitRootLogin yes$/#PermitRootLogin yes/' -i /etc/ssh/sshd_config +# Disable password login for sudoers. echo -e 'Match Group sudo\n PasswordAuthentication no' >> /etc/ssh/sshd_config -# add IP blocking +# Add IP blocking. # see: security/README.md nano /etc/hosts.deny service ssh restart @@ -124,6 +130,10 @@ cat /proc/mdstat cat /proc/partitions cat /proc/mounts cat /proc/swaps +# dd if=/dev/zero of=/swap0 bs=1M count=768 +# chmod 0600 /swap0 +# echo "/swap0 none swap sw 0 0" >> /etc/fstab + grep "relatime" /proc/mounts || echo "ERROR: no relAtime" # kernel @@ -149,17 +159,17 @@ editor /etc/motd editor /etc/network/interfaces # iface eth0 inet static # address -# gateway +# gateway ifconfig -a route -n -4 route -n -6 netstat -antup editor /etc/resolv.conf -#nameserver 8.8.8.8 -#nameserver 8.8.4.4 -#nameserver -#options timeout:2 -##options rotate +# nameserver 8.8.8.8 +# nameserver 8.8.4.4 +# nameserver +# options timeout:2 +# #options rotate ping6 -c 4 ipv6.google.com # should be A 93.184.216.119 host -v -t A example.com @@ -225,7 +235,7 @@ apt-get install -t wheezy-backports -y rsyslog whois git goaccess # editor /etc/rsyslog.conf # $ModLoad immark # $MarkMessagePeriod 1800 -cd /root/; git clone https://github.com/szepeviktor/debian-server-tools.git +cd /root/src/ && git clone https://github.com/szepeviktor/debian-server-tools.git cd debian-server-tools/ && git submodule init && git submodule update # IRQ balance @@ -238,10 +248,11 @@ cp -v monitoring/ntpdated /usr/local/sbin/ # set nearest time server: http://www.pool.ntp.org/en/ # NTPSERVERS="0.uk.pool.ntp.org 1.uk.pool.ntp.org 2.uk.pool.ntp.org 3.uk.pool.ntp.org" # NTPSERVERS="0.de.pool.ntp.org 1.de.pool.ntp.org 2.de.pool.ntp.org 3.de.pool.ntp.org" +# NTPSERVERS="0.fr.pool.ntp.org 1.fr.pool.ntp.org 2.fr.pool.ntp.org 3.fr.pool.ntp.org" # NTPSERVERS="0.hu.pool.ntp.org 1.hu.pool.ntp.org 2.hu.pool.ntp.org 3.hu.pool.ntp.org" -editor /etc/default/ntpdate # OVH # NTPSERVERS="ntp.ovh.net" +editor /etc/default/ntpdate #TODO measure CPU speed bz2 25MB, disk access time and throughput hdd-, network speed multiple connections # https://github.com/mgutz/vpsbench/blob/master/vpsbench @@ -253,6 +264,7 @@ editor /etc/nscd.conf # enable-cache hosts yes # positive-time-to-live hosts 60 # negative-time-to-live hosts 20 +chown unscd:unscd socket /var/run/nscd/socket service unscd restart # sanitize files diff --git a/mail/Courier-alias-skeleton b/mail/Courier-alias-skeleton index cee4266a..04a4fe02 100644 --- a/mail/Courier-alias-skeleton +++ b/mail/Courier-alias-skeleton @@ -6,4 +6,4 @@ webmaster@@@DOMAIN@@: admin@@@DOMAIN@@ postmaster@@@DOMAIN@@: admin@@@DOMAIN@@ hostmaster@@@DOMAIN@@: admin@@@DOMAIN@@ -@@APACHEUSER@@@szepe.net: webmaster@@@DOMAIN@@ +@@SITE_USER@@@szepe.net: webmaster@@@DOMAIN@@ diff --git a/mail/README.md b/mail/README.md index ce3cc390..a34c315b 100644 --- a/mail/README.md +++ b/mail/README.md @@ -13,6 +13,10 @@ https://toolbox.googleapps.com/apps/checkmx/ see: mail/imapsync +### Send messages in an mbox file to an email address + +see: mail/mbox_send2.py + ### Email sending and receiving - SSL? @@ -21,6 +25,11 @@ see: mail/imapsync ### Courier catchall address http://www.courier-mta.org/makehosteddomains.html +http://www.courier-mta.org/dot-courier.html + +```bash +echo "|pipe/command" > /var/mail/domain.net/user/.courier-foo-default +``` ### Online email tests @@ -29,11 +38,22 @@ http://www.courier-mta.org/makehosteddomains.html - checkmyauth@auth.returnpath.net http://www.returnpath.com/solution-content/dmarc-support/what-is-dmarc/ - https://winning.email/checkup/ -### Spamassassin test +### Email forwarding (srs) + +https://couriersrs.com/ https://github.com/szepeviktor/couriersrs +see: http://szepeviktor.github.io/ +Create users SRS0 and SRS1. + +```bash +echo "|/usr/bin/couriersrs --reverse" > /etc/courier/aliasdir/.courier-SRS0-default +echo "|/usr/bin/couriersrs --reverse" > /etc/courier/aliasdir/.courier-SRS1-default +``` + +### Spamassassin test and DKIM test ```bash spamassassin --test-mode -D < msg.eml -# specific test, see: man spamassassin-run +# For specific tests see: man spamassassin-run spamassassin --test-mode -D dkim < msg-signed.eml opendkim -vvv -t msg-signed.eml ``` @@ -53,7 +73,7 @@ https://support.google.com/mail/answer/81126?hl=en - setup - check -- monitor +- monitor `host -t TXT ` #### DKIM @@ -64,6 +84,7 @@ https://support.google.com/mail/answer/81126?hl=en #### ADSP +An optional extension to the DKIM E-mail authentication scheme. #### DMARC @@ -80,11 +101,11 @@ Specs: https://datatracker.ietf.org/doc/draft-kucherawy-dmarc-base/?include_text - Return-Path:, Reply-to:, From:, To:, Subject: - SMTP "MAIL FORM: " -#### Content +#### Bulk mail musts -- online version +- link to online version - who (email address) is subscribed -- contact +- sender's contact details - unsubscribe link - HTML and plain payload @@ -93,11 +114,17 @@ Specs: https://datatracker.ietf.org/doc/draft-kucherawy-dmarc-base/?include_text - https://www.dnswl.org/?page_id=87 - . -### Courier kitchen sink +### Kitchen sink - `echo > /etc/courier/aliasdir/.courier-kitchensink` - alias: `any.address@any-domain.net: kitchensink@localhost` ### Scan Class C network -`for I in $(seq 1 255); do host -t A 1.2.3.${I}; done` +```bash +for I in $(seq 1 255); do host -t A 1.2.3.${I}; done +``` + +### Email templates + +https://www.klaviyo.com/ diff --git a/mail/mbox_send2.py b/mail/mbox_send2.py new file mode 100755 index 00000000..2c6eb587 --- /dev/null +++ b/mail/mbox_send2.py @@ -0,0 +1,161 @@ +#!/usr/bin/python +"""\ +A command-line utility that can (re)send all messages in an mbox file +to a specific email address, with options for controlling the rate at +which they are sent, etc. +""" + +# Upstream: https://gist.github.com/wojdyr/1176398#comment-1300024 + +# I got this script from Robin Dunn a few years ago, see +# https://github.com/wojdyr/fityk/wiki/MigrationToGoogleGroups + +import sys +import os +import time +import mailbox +import email +import smtplib + +from optparse import OptionParser, make_option + + +#--------------------------------------------------------------------------- +# Set some defaults + +defTo = [] +defFrom = None +defChunkSize = 100 +defChunkDelay = 30 +defSmtpHost = 'localhost' +defSmtpPort = 25 +defCount = -1 +defStart = -1 + +# define the command line options +option_list = [ + make_option('--to', action='append', dest='toAddresses', default=defTo, + help="The address to send the messages to. May be repeated."), + + make_option('--from', dest='fromAddress', default=defFrom, + help="The address to send the messages from."), + + make_option('--chunk', type='int', dest='chunkSize', default=defChunkSize, + help='How many messages to send in each batch before pausing, default: %d' % defChunkSize), + + make_option('--pause', type='int', dest='chunkDelay', default=defChunkDelay, + help='How many seconds to delay between chunks. default: %d' % defChunkDelay), + + make_option('--count', type='int', dest='count', default=defCount, + help='How many messages to send before exiting the tool, default is all messages in the mbox.'), + + make_option('--start', type='int', dest='start', default=defStart, + help='Which message number to start with. Defaults to where the tool left off the last time, or zero.'), + + make_option('--smtpHost', dest='smtpHost', default=defSmtpHost, + help='Hostname where SMTP server is running'), + + make_option('--smtpPort', type='int', dest='smtpPort', default=defSmtpPort, + help='Port number to use for connecting to SMTP server'), + ] + +smtpPassword = None # implies using TLS +#--------------------------------------------------------------------------- + +def get_hwm(hwmfile): + if not os.path.exists(hwmfile): + return -1 + hwm = int(file(hwmfile).read()) + return hwm + +def set_hwm(hwmfile, count): + f = file(hwmfile, 'w') + f.write(str(count)) + f.close() + + + +def main(args): + if sys.version_info < (2,5): + print "Python 2.5 or better is required." + sys.exit(1) + + # Parse the command line args + parser = OptionParser(usage="%prog [options] mbox_file(s)", + description=__doc__, + version="%prog 0.9.1", + option_list=option_list) + + options, arguments = parser.parse_args(args) + + # ensure we have the required options + if not options.toAddresses: + parser.error('At least one To address is required (use --to)') + + if not options.fromAddress: + parser.error('From address is required (use --from)') + + if not arguments: + parser.error('At least one mbox file is required') + + # process the mbox file(s) + for mboxfile in arguments: + print "Opening %s..." % mboxfile + mbox = mailbox.mbox(mboxfile) + totalInMbox = len(mbox) + print "Total messages in mbox: %d" % totalInMbox + + hwmfile = mboxfile + '.hwm' + print 'Storing last message processed in %s' % hwmfile + start = get_hwm(hwmfile) + if options.start != -1: + start = options.start + start += 1 + print 'Starting with message #%d' % start + + totalSent = 0 + current = start + + # Outer loop continues until either the whole mbox or options.count + # messages have been sent, + while (current < totalInMbox and + (totalSent < options.count or options.count == -1)): + + # Inner loop works one chunkSize number of messages at a time, + # pausing and reconnecting to the SMTP server for each chunk. + print 'Connecting to SMTP(%s, %d)' % (options.smtpHost, options.smtpPort) + smtp = smtplib.SMTP(options.smtpHost, options.smtpPort) + if smtpPassword: # use TLS + smtp.ehlo() + smtp.starttls() + smtp.ehlo() + smtp.login(options.fromAddress, smtpPassword) + + chunkSent = 0 + while chunkSent < options.chunkSize: + msg = mbox[current] + print 'Processing message %d: %s' % (current, msg['Subject']) + + # Here is where we actually send the message + smtp.sendmail(options.fromAddress, options.toAddresses, msg.as_string()) + + set_hwm(hwmfile, current) # set new 'high water mark' + current += 1 + totalSent += 1 + chunkSent += 1 + if (current >= totalInMbox or + (totalSent >= options.count and options.count != -1)): + break + else: + smtp.close() + del smtp + print "Pausing for %d seconds..." % options.chunkDelay, + time.sleep(options.chunkDelay) + print + + print 'Goodbye' + +#--------------------------------------------------------------------------- + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/mail/smtp-auth.sh b/mail/smtp-auth.sh index 5a827b8d..884094a0 100755 --- a/mail/smtp-auth.sh +++ b/mail/smtp-auth.sh @@ -142,7 +142,7 @@ which openssl &> /dev/null || exit 99 while getopts ":aplch:u:P:" opt; do case $opt in - a) # AUTH support + a) # Test AUTH support MODE="auth" ;; p) # AUTH PLAIN @@ -154,13 +154,13 @@ while getopts ":aplch:u:P:" opt; do c) # AUTH CRAM-MD5 MODE="md5" ;; - h) # host + h) # Host SMTP_HOST="$OPTARG" ;; - u) # user name + u) # User name SMTP_USER="$OPTARG" ;; - P) # password + P) # Password SMTP_PASS="$OPTARG" ;; \?) diff --git a/monitoring/DNS.md b/monitoring/DNS.md index 55aaf350..2d9ff1ff 100644 --- a/monitoring/DNS.md +++ b/monitoring/DNS.md @@ -1,7 +1,10 @@ # DNS test tools + http://dnscheck.pingdom.com/?domain= http://www.dnsinspect.com/ http://intodns.com/ http://www.solvedns.com/ +https://www.ultratools.com/domainHealthReport +https://www.ultratools.com/tools/dnsHostingSpeed diff --git a/monitoring/cpu-speed/README.md b/monitoring/cpu-speed/README.md index 71303153..8da892a8 100644 --- a/monitoring/cpu-speed/README.md +++ b/monitoring/cpu-speed/README.md @@ -1,7 +1,10 @@ ### CPU benchmarks ```bash -sysbench --test=cpu --cpu-max-prime=100000 run --num-threads=8 +# One thread, all threads, 4 times overload +time sysbench --test=cpu --cpu-max-prime=100000 run --num-threads=1 +time sysbench --test=cpu --cpu-max-prime=100000 run --num-threads=$(grep -c "^processor" /proc/cpuinfo) +time sysbench --test=cpu --cpu-max-prime=100000 run --num-threads=$((4 * $(grep -c "^processor" /proc/cpuinfo))) ``` ``` @@ -11,7 +14,7 @@ power8 ×176 | 9 power8 small×8 | 57 FORPSI Smart | 325 vps5 X3440×2 | 89 - +RunAbove labs | 215 ``` diff --git a/monitoring/hdd-seeker/hdd-bench.sh b/monitoring/hdd-seeker/hdd-bench.sh index 6b4abf0a..dd2dc50a 100755 --- a/monitoring/hdd-seeker/hdd-bench.sh +++ b/monitoring/hdd-seeker/hdd-bench.sh @@ -2,8 +2,8 @@ # # Measure disk access time # -# VERSION :0.3 -# DATE :2014-11-28 +# VERSION :0.5 +# DATE :2015-04-14 # AUTHOR :Viktor Szépe # LICENSE :The MIT License (MIT) # URL :https://github.com/szepeviktor/debian-server-tools @@ -102,7 +102,7 @@ hdparm -T "$DEVICE" echo ------------------------------------ # ioping -if ! which hdparm &> /dev/null; then +if ! which ioping &> /dev/null; then echo "to install ioping on a Debian-based system:" echo "apt-get install -y ioping" echo "ioping -q -i 0 -w 5 -S 64m ${DEVICE}" diff --git a/webserver/README.md b/webserver/README.md new file mode 100644 index 00000000..815b48f8 --- /dev/null +++ b/webserver/README.md @@ -0,0 +1,7 @@ +### File revving on nginx + +```nginx +location ~* ^(.+)\.\d\d+\.(js|css|png|jpg|jpeg|gif|ico)$ { + try_files $1.$2 /index.php?$args; + } +``` diff --git a/wheezy-repo b/wheezy-repo index d4590ecf..54fcab3e 160000 --- a/wheezy-repo +++ b/wheezy-repo @@ -1 +1 @@ -Subproject commit d4590ecf493ef63796de236a9834b62e98eff659 +Subproject commit 54fcab3e6707630e8ecd9dc7db92270e3f08f21a