RUN-ALL-STEPS

#!/bin/sh

MODE="$1"  # "", force, antizapret, rublacklist

T=$(date +%Y-%m-%d-%H%M)

TOP="$(cd "$(dirname "$0")/.." && pwd)" #"
BIN="$TOP/bin"
DAT="$TOP/download"
LOG="$TOP/logs/$T.log"
OLD="$TOP/old/$T.zip"
KEY="$TOP/keys/request.xml"
SIG="$KEY.p7s"

######  Functions  #############################################

Say() { date "+%Y-%m-%d %H:%M:%S -- $@"; }

Fail() { echo "FATAL ERROR: $@"; exit 1; }

Check_lines_count() {
    local count="$(wc -l < "$1" 2>/dev/null)"
    test -n "$count" -a "$count" -ge "$2" -a "$count" -le "$3" && return
    Fail "Wrong lines count in $1, must be between $2 and $3: $count"
}

Fetch() {
    local FILE="$1" MIN="$2" MAX="$3" URL="$4" SED="$5"

    wget -qO- "$URL" | sed "$SED" > "$FILE.new"

    if test -s "$FILE" && diff -q "$FILE" "$FILE.new" >/dev/null 2>&1; then
        echo "FRESH_OK"
        /bin/rm -f "$FILE.new"

        Say "SKIPPED."
        exit 0
    fi

    mv "$FILE.new" "$FILE"
    Check_lines_count  "$FILE"  "$MIN"  "$MAX"
    zip "$OLD"         "$FILE"
}

Fetch_default() {
    $BIN/rkn-fetch "$KEY" "$SIG" update result.zip

    if ! test -s result.zip; then
        /bin/rm -f result.zip
        Fail "Empty ZIP."
    fi

    if test "$MODE" != "force" && grep -q "^FRESH_OK" "$LOG"; then
        Say "SKIPPED."
        exit 0
    fi

    unzip -t result.zip || Fail "ZIP check failed."

    Say "Extract..."

    cp -p result.zip "$OLD"

    #funzip < result.zip | sed 's,><,>\n<,g' > dump.xml
    unzip -p result.zip dump.xml | sed 's,><,>\n<,g' > dump.xml

    Check_lines_count dump.xml 100000 10000000
}

######  Main  ##################################################

exec 1>"$LOG"
exec 2>&1

cd "$DAT"

Say "Check another instances..."
   procname="$(basename "$0")"
   procnum="$(pgrep -fc "$procname")"
   test "$procnum" -gt 1 && Fail "$0 seems already running."

Say "Fetch..."
   case "$MODE" in
   saved       ) ;;
   antizapret  ) Fetch antizapret.xml   900000  10000000  'http://api.antizapret.info/all.php?type=xml'         's,><,>\n<,g' ;;
   rublacklist ) Fetch rublacklist.json  50000    300000  'https://reestr.rublacklist.net/api/v2/current/json'  's,{"postanovlenie":,\n{"postanovlenie":,g' ;;
   * )           Fetch_default ;;
   esac

Say "Parse..."
   find . -type f -name "*.lst" -delete

   case "$MODE" in
   antizapret  ) $BIN/rkn-parse-antizapret   antizapret.xml    u.lst d.lst m.lst a.lst n.lst ;;
   rublacklist ) $BIN/rkn-parse-rublacklist  rublacklist.json  u.lst d.lst m.lst a.lst n.lst ;;
   * )           $BIN/rkn-parse              dump.xml          u.lst d.lst m.lst a.lst n.lst ;;
   esac

Say "Filter..."
   $BIN/rkn-filter  d.lst m.lst u.lst   dd.lst uu.lst pp.lst
   $BIN/rkn-filter2 a.lst n.lst  > an.lst

Say "Check lines count..."
   Check_lines_count   a.lst  5000  50000  # ipaddrs
   Check_lines_count   n.lst    20    200  # ipnets
   Check_lines_count  dd.lst 30000 150000  # domains
   Check_lines_count  uu.lst  3000  30000  # urls
   Check_lines_count  pp.lst     2     10  # tcp ports
   Check_lines_count  an.lst  5000  50000  # non-intersected CIDRs

Say "Apply ipset..."
   $BIN/ipset-sync  a.lst  n.lst

Say "Apply Squid..."
   cp -p  dd.lst  /etc/squid/blacklist_domains.txt
   cp -p  uu.lst  /etc/squid/blacklist_urls.txt
   cp -p  an.lst  /etc/squid/blacklist_ipaddrs.txt

   /usr/sbin/squid -k parse 2>&1 | grep "ERROR:" && Fail "Invalid Squid configs"
   $BIN/squid-reload

Say "ALL DONE."

## END ##