diff --git a/RELEASE b/RELEASE index 1c09c74..60a2d3e 100644 --- a/RELEASE +++ b/RELEASE @@ -1 +1 @@ -0.3.3 +0.4.0 \ No newline at end of file diff --git a/deployment/config b/deployment/config index 4fd8c74..73eda57 100644 --- a/deployment/config +++ b/deployment/config @@ -1,6 +1,7 @@ #!/bin/bash -GIT_BASE=https://github.com/ExaWorks/psij-testing-service/archive/refs/tags +PACKAGE_NAME=psij-testing-service +GIT_BASE=https://github.com/ExaWorks/$PACKAGE_NAME/archive/refs/tags VERSION=`cat ../../RELEASE` # The service version to build the docker container with or use for @@ -8,7 +9,7 @@ VERSION=`cat ../../RELEASE` SERVICE_VERSION=$VERSION # Docker image to use -IMAGE=hategan/psij-testing-service +IMAGE=hategan/$PACKAGE_NAME # If DEV is not 0, the deployment script will install the service from # the parent directory diff --git a/deployment/deploy/deploy.sh b/deployment/deploy/deploy.sh index d77a462..23d9057 100755 --- a/deployment/deploy/deploy.sh +++ b/deployment/deploy/deploy.sh @@ -38,9 +38,10 @@ run() { echo -n "Running $@..." echo "> $@" >>deploy.log OUT=`"$@" 2>&1` + EC=$? # ` - if [ "$?" != "0" ]; then - echo "FAILED" + if [ "$EC" != "0" ]; then + echo "FAILED ($EC)" echo $OUT exit 2 fi @@ -53,9 +54,22 @@ getId() { run docker ps -f "name=service-$TYPE" --format "{{.ID}}" } +waitForContainer() { + ID=$1 + echo "Waiting for container $ID..." + while true; do + STATUS=`docker inspect -f '{{.State.Status}}' $ID` + if [ "$STATUS" == "running" ]; then + break + fi + sleep 1 + done +} + deployContainer() { TYPE=$1 PORT=$2 + HOST_NAME=$3 getId $TYPE ID=$OUT echo "ID: $OUT" @@ -71,34 +85,51 @@ deployContainer() { mkdir -p $DATA_DIR/$TYPE/mongodb mkdir -p $DATA_DIR/$TYPE/web cp $ROOT/web/$TYPE/* $DATA_DIR/$TYPE/web + run docker run \ - -d -p $PORT:9909 --name "service-$TYPE" \ + -d -p $PORT:9909 --name "service-$TYPE" -h $HOST_NAME \ --restart=on-failure:3 \ --volume=$DATA_DIR/$TYPE/mongodb:/var/lib/mongodb \ --volume=$DATA_DIR/$TYPE/web:/var/www/html \ + --volume=/etc/letsencrypt:/etc/letsencrypt \ $EXTRA_VOL \ - $IMAGE:latest + $IMAGE:$SERVICE_VERSION UPDATE_CONTAINER=1 - fi - if [ "$UPDATE_CONTAINER" != "0" ]; then getId $TYPE ID=$OUT + sleep 5 + waitForContainer $ID + run docker exec -it $ID bash -c "echo $HOST_NAME > /etc/hostname" + + if [ ! -f ../docker/fs/etc/psij-testing-service/secrets.json ]; then + error "No secrets.json found. Please edit/create secrets.json in ../docker/fs/etc/psij-testing-service" + fi + run docker cp ../docker/fs/. $ID:/ + run docker exec -it $ID sed -i "s/\$myhostname/$HOST_NAME/g" /etc/postfix/main.cf + run docker exec -it $ID sed -i "s/\$mydomain/$DOMAIN_NAME/g" /etc/postfix/main.cf + run docker exec -it $ID postmap -v hash:/etc/postfix/sasl_passwd + run docker exec -it $ID bash -c "pip show $PACKAGE_NAME | grep 'Location: ' | sed 's/Location: //' | tr -d '\n'" + PACKAGE_LOC=$OUT + run docker cp ../web/. "$ID:$PACKAGE_LOC/psij/web/" + fi + if [ "$UPDATE_CONTAINER" != "0" ]; then if [ "$DEV" == "1" ]; then - run docker exec -it $ID update-psi-j-testing-service -y $TYPE /psi-j-testing-service-dev + ./upgrade.sh -y --force --component $TYPE /psi-j-testing-service-dev else - run docker exec -it $ID update-psi-j-testing-service -y $TYPE $SERVICE_VERSION + ./upgrade.sh -y --force --component $TYPE $SERVICE_VERSION fi fi } -if [ "$USER" != "root" ]; then - error "You need root permissions to run this script." -fi if service nginx status >/dev/null 2>&1; then echo "Nginx is already running. Skipping deployment." UPDATE_NGINX=$FORCED_UPDATE else + if [ "$USER" != "root" ]; then + error "You need root permissions to run this script." + fi + run apt-get update run apt-get install -y nginx UPDATE_NGINX=1 @@ -111,12 +142,13 @@ filterConf() { PROXY_REDIRECT="$4" INTERNAL_PORT="$5" - if [ ! -f "$DST" ]; then - sed -e "s/\${DOMAIN_NAME}/${DOMAIN_NAME}/g" "$SRC" | \ - sed -e "s/\${SERVER_NAME}/${SERVER_NAME}/g" | \ - sed -e "s/\${PROXY_REDIRECT}/${PROXY_REDIRECT}/g" | \ - sed -e "s/\${INTERNAL_PORT}/${INTERNAL_PORT}/g" >"$DST" + if [ -f "$DST" ]; then + cp "$DST" "$DST.bk" fi + sed -e "s/\${DOMAIN_NAME}/${DOMAIN_NAME}/g" "$SRC" | \ + sed -e "s/\${SERVER_NAME}/${SERVER_NAME}/g" | \ + sed -e "s/\${PROXY_REDIRECT}/${PROXY_REDIRECT}/g" | \ + sed -e "s/\${INTERNAL_PORT}/${INTERNAL_PORT}/g" >"$DST" } deploySite() { @@ -130,8 +162,9 @@ deploySite() { ln -f -s "/etc/nginx/sites-available/$NAME" "/etc/nginx/sites-enabled/$NAME" } +DOMAIN_NAME=`cat DOMAIN_NAME | tr -d '\n'` + if [ "$UPDATE_NGINX" != "0" ]; then - DOMAIN_NAME=`cat DOMAIN_NAME | tr -d '\n'` filterConf nginx/headers.conf /etc/nginx/snippets/headers.conf filterConf nginx/nginx.conf /etc/nginx/nginx.conf filterConf nginx/ssl.conf /etc/nginx/ssl.conf @@ -147,5 +180,5 @@ if [ "$UPDATE_NGINX" != "0" ]; then run service nginx restart fi -deployContainer psij 9901 -deployContainer sdk 9902 +deployContainer psij 9901 "psij.$DOMAIN_NAME" +deployContainer sdk 9902 "sdk.$DOMAIN_NAME" diff --git a/deployment/deploy/nginx/headers.conf b/deployment/deploy/nginx/headers.conf index c4a59f2..b752480 100644 --- a/deployment/deploy/nginx/headers.conf +++ b/deployment/deploy/nginx/headers.conf @@ -20,5 +20,5 @@ add_header Referrer-Policy same-origin; # to switch to fullscreen add_header Permissions-Policy fullscreen=(self); -add_header Content-Security-Policy "img-src *; frame-src 'none'; media-src 'none'; object-src 'none'; base-uri 'self'; form-action 'self'"; +add_header Content-Security-Policy "img-src *; frame-src https://www.google.com; media-src 'none'; object-src 'none'; base-uri 'self'; form-action 'self'"; diff --git a/deployment/deploy/upgrade.sh b/deployment/deploy/upgrade.sh index fb3d5db..11a2b4c 100755 --- a/deployment/deploy/upgrade.sh +++ b/deployment/deploy/upgrade.sh @@ -1,18 +1,55 @@ #!/bin/bash +set -e + error() { echo "$@" exit 1 } +usage() { + echo "Usage:" + echo " upgrade.sh [-h|--help] [-f|--force] [-c | --component] [-y | --assume-yes]" + exit 1 +} + + +FORCE="" +DONTASK="" +COMPONENTS="psij sdk" +DOMAIN_NAME=`cat DOMAIN_NAME | tr -d '\n'` + +while [ "$1" != "" ]; do + case "$1" in + -h | --help) + usage + ;; + -f | --force) + FORCE="--force" + shift + ;; + -y | --assume-yes) + DONTASK="-y" + shift + ;; + -c | --component) + COMPONENTS="$2" + shift + shift + ;; + *) + TARGET_VERSION=$1 + shift + ;; + esac +done + if [ ! -f ../config ]; then error "This script must be run from the deploy directory" fi source ../config -TARGET_VERSION=$1 - getId() { TYPE=$1 @@ -23,11 +60,28 @@ update() { TYPE=$1 ID=`docker ps -f "name=service-$TYPE" --format "{{.ID}}"` if [ "$ID" != "" ]; then - docker exec -it $ID update-psi-j-testing-service $TYPE $TARGET_VERSION + # Make sure everything is up to date + docker exec -it $ID apt-get update + docker exec -it $ID apt-get upgrade -y + # Make sure that all files are there if needed + docker cp ../docker/fs/. $ID:/tmp/fs/ + docker exec -it $ID bash -c "echo $TYPE.$DOMAIN_NAME > /etc/hostname" + if [ "$DEV" == "1" ]; then + pushd ../.. + python setup.py sdist + popd + docker cp ../../dist/$PACKAGE_NAME-$SERVICE_VERSION.tar.gz $ID:/tmp/$PACKAGE_NAME-$SERVICE_VERSION.tar.gz + docker exec -it $ID update-psi-j-testing-service $DONTASK $FORCE --src /tmp/$PACKAGE_NAME-$SERVICE_VERSION.tar.gz $TYPE $TARGET_VERSION + else + # Actual update + docker exec -it $ID update-psi-j-testing-service $DONTASK $FORCE $TYPE $TARGET_VERSION + fi else echo "Service $TYPE not running." fi } -update psij -update sdk \ No newline at end of file + +for COMPONENT in $COMPONENTS; do + update $COMPONENT +done diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index c78d943..70608d5 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -1,23 +1,27 @@ FROM ubuntu:20.04 -ARG SERVICE_VERSION -ARG GIT_BASE=https://github.com/ExaWorks/psij-testing-service/archive/refs/tags +ARG SERVICE_PACKAGE RUN useradd -ms /bin/bash testsrv RUN apt-get update RUN apt-get upgrade -y -RUN DEBIAN_FRONTEND="noninteractive" TZ="UTC" apt-get install -y python3.9 python3-pip mongodb wget mc +RUN DEBIAN_FRONTEND="noninteractive" TZ="UTC" apt-get install -y python3.9 python3-pip mongodb \ + wget mc postfix procmail syslog-ng WORKDIR ~/ -RUN pip install $GIT_BASE/v$SERVICE_VERSION.tar.gz +# needed to get syslog-ng to work inside the container +RUN sed -i 's/system()/system(exclude-kmsg(yes))/g' /etc/syslog-ng/syslog-ng.conf -COPY psi-j-testing-service /etc/init.d -COPY entrypoint.sh / -COPY update-psi-j-testing-service /usr/bin + +COPY $SERVICE_PACKAGE /tmp +RUN pip install /tmp/$SERVICE_PACKAGE +RUN rm /tmp/$SERVICE_PACKAGE + +COPY fs / RUN mkdir /var/log/psi-j-testing-service RUN chown testsrv:testsrv /var/log/psi-j-testing-service -CMD /entrypoint.sh \ No newline at end of file +CMD /entrypoint.sh diff --git a/deployment/docker/build.sh b/deployment/docker/build.sh index e907a6b..f7295a0 100755 --- a/deployment/docker/build.sh +++ b/deployment/docker/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + error() { echo "$@" exit 1 @@ -13,8 +15,19 @@ source ../config echo "Version: $VERSION" +if [ "$DEV" == "1" ]; then + pushd ../.. + python setup.py sdist + popd + cp ../../dist/$PACKAGE_NAME-$SERVICE_VERSION.tar.gz ./$PACKAGE_NAME-$SERVICE_VERSION.tar.gz +else + wget $GIT_BASE/v$SERVICE_VERSION.tar.gz -O ./$PACKAGE_NAME-$SERVICE_VERSION.tar.gz +fi + -docker build --build-arg SERVICE_VERSION=$SERVICE_VERSION --build-arg GIT_BASE="$GIT_BASE" -t $IMAGE:$VERSION . -docker push $IMAGE:$VERSION +docker build --build-arg SERVICE_PACKAGE=./$PACKAGE_NAME-$SERVICE_VERSION.tar.gz -t $IMAGE:$VERSION . docker image tag $IMAGE:$VERSION $IMAGE:latest -docker push $IMAGE:latest +if [ "$DEV" == "0" ]; then + docker push $IMAGE:$VERSION + docker push $IMAGE:latest +fi diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/fs/entrypoint.sh similarity index 70% rename from deployment/docker/entrypoint.sh rename to deployment/docker/fs/entrypoint.sh index 8273ae2..58bf407 100755 --- a/deployment/docker/entrypoint.sh +++ b/deployment/docker/fs/entrypoint.sh @@ -1,7 +1,11 @@ #!/bin/bash +set -e + +service syslog-ng start chown -R mongodb:mongodb /var/lib/mongodb service mongodb start +service postfix start service psi-j-testing-service start diff --git a/deployment/docker/psi-j-testing-service b/deployment/docker/fs/etc/init.d/psi-j-testing-service similarity index 91% rename from deployment/docker/psi-j-testing-service rename to deployment/docker/fs/etc/init.d/psi-j-testing-service index bcccb99..5fbb388 100755 --- a/deployment/docker/psi-j-testing-service +++ b/deployment/docker/fs/etc/init.d/psi-j-testing-service @@ -31,9 +31,11 @@ start() { log_progress_msg "already running" errorcode=0 else + # without this, errors get delayed in the log + export PYTHONUNBUFFERED=1 start-stop-daemon --background --start --quiet --pidfile $PIDFILE \ --make-pidfile --chuid $DAEMONUSER \ - --startas /bin/bash -- -c "exec psi-j-testing-service >>/var/log/psi-j-testing-service/service.log 2>&1" + --startas /bin/bash -- -c "exec psi-j-testing-service -c /etc/psij-testing-service/config.json -s /etc/psij-testing-service/secrets.json >>/var/log/psi-j-testing-service/service.log 2>&1" errorcode=$? fi return $errorcode diff --git a/deployment/docker/fs/etc/postfix/header_checks b/deployment/docker/fs/etc/postfix/header_checks new file mode 100644 index 0000000..e69de29 diff --git a/deployment/docker/fs/etc/postfix/main.cf b/deployment/docker/fs/etc/postfix/main.cf new file mode 100644 index 0000000..e074e1e --- /dev/null +++ b/deployment/docker/fs/etc/postfix/main.cf @@ -0,0 +1,179 @@ +# Global Postfix configuration file. This file lists only a subset +# of all parameters. For the syntax, and for a complete parameter +# list, see the postconf(5) manual page (command: "man 5 postconf"). +# +# For common configuration examples, see BASIC_CONFIGURATION_README +# and STANDARD_CONFIGURATION_README. To find these documents, use +# the command "postconf html_directory readme_directory", or go to +# http://www.postfix.org/BASIC_CONFIGURATION_README.html etc. +# +# For best results, change no more than 2-3 parameters at a time, +# and test if Postfix still works after every change. + +# COMPATIBILITY +# +# The compatibility_level determines what default settings Postfix +# will use for main.cf and master.cf settings. These defaults will +# change over time. +# +# To avoid breaking things, Postfix will use backwards-compatible +# default settings and log where it uses those old backwards-compatible +# default settings, until the system administrator has determined +# if any backwards-compatible default settings need to be made +# permanent in main.cf or master.cf. +# +# When this review is complete, update the compatibility_level setting +# below as recommended in the RELEASE_NOTES file. +# +# The level below is what should be used with new (not upgrade) installs. +# +compatibility_level = 2 + + +# soft_bounce = no + +# queue_directory = /var/spool/postfix + +command_directory = /usr/sbin + +daemon_directory = /usr/lib/postfix/sbin + +data_directory = /var/lib/postfix + +# mail_owner = postfix + +# default_privs = nobody + +# myhostname = + +# mydomain = + +# myorigin = /etc/mailname +myorigin = $myhostname +# myorigin = $mydomain + +inet_interfaces = all +# inet_interfaces = $myhostname +# inet_interfaces = $myhostname, localhost + +# proxy_interfaces = +# proxy_interfaces = 1.2.3.4 + +mydestination = $myhostname, localhost.$mydomain, localhost +# mydestination = $myhostname, localhost.$mydomain, localhost, $mydomain +# mydestination = $myhostname, localhost.$mydomain, localhost, $mydomain, +# mail.$mydomain, www.$mydomain, ftp.$mydomain + +# local_recipient_maps = unix:passwd.byname $alias_maps +# local_recipient_maps = proxy:unix:passwd.byname $alias_maps +local_recipient_maps = $alias_maps + +unknown_local_recipient_reject_code = 550 + +# mynetworks_style = class +# mynetworks_style = subnet +mynetworks_style = host + +# mynetworks = 168.100.189.0/28, 127.0.0.0/8 +# mynetworks = $config_directory/mynetworks +# mynetworks = hash:/etc/postfix/network_table +mynetworks = 127.0.0.0/8 + +# relay_domains = $mydestination + +# relayhost = $mydomain +# relayhost = [gateway.my.domain] +# relayhost = [mailserver.isp.tld] +# relayhost = uucphost +# relayhost = [an.ip.add.ress] + +# relay_recipient_maps = hash:/etc/postfix/relay_recipients + +# in_flow_delay = 1s + +alias_maps = hash:/etc/aliases +# alias_maps = hash:/etc/aliases +# alias_maps = hash:/etc/aliases, nis:mail.aliases +# alias_maps = netinfo:/aliases + +alias_database = hash:/etc/aliases +# alias_database = dbm:/etc/mail/aliases +# alias_database = hash:/etc/aliases +# alias_database = hash:/etc/aliases, hash:/opt/majordomo/aliases + +# recipient_delimiter = + + +# home_mailbox = Mailbox +# home_mailbox = Maildir/ + +mail_spool_directory = /var/mail +# mail_spool_directory = /var/spool/mail + +mailbox_command = /usr/bin/procmail +# mailbox_command = /usr/bin/procmail -a "$EXTENSION" + +# mailbox_transport = lmtp:unix:/var/imap/socket/lmtp +# mailbox_transport = cyrus + +# fallback_transport = lmtp:unix:/file/name +# fallback_transport = cyrus +# fallback_transport = + +# luser_relay = $user@other.host +# luser_relay = $local@other.host +# luser_relay = admin+$local + +header_checks = regexp:/etc/postfix/header_checks + +# fast_flush_domains = $relay_domains + +# smtpd_banner = $myhostname ESMTP $mail_name +# smtpd_banner = $myhostname ESMTP $mail_name ($mail_version) +smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu) + +# local_destination_concurrency_limit = 2 + +# default_destination_concurrency_limit = 20 + +# debug_peer_level = 2 + +# debug_peer_list = 127.0.0.1 +# debug_peer_list = some.domain + +debugger_command = + PATH=/bin:/usr/bin:/usr/local/bin; export PATH; (echo cont; + echo where) | gdb $daemon_directory/$process_name $process_id 2>&1 + >$config_directory/$process_name.$process_id.log & sleep 5 + +# sendmail_path = + +# newaliases_path = + +# mailq_path = + +# setgid_group = + +# html_directory = + +# manpage_directory = + +# sample_directory = + +# readme_directory = + +inet_protocols = ipv4 + +smtpd_tls_cert_file=/etc/letsencrypt/live/$mydomain/fullchain.pem +smtpd_tls_key_file=/etc/letsencrypt/live/$mydomain/privkey.pem +smtpd_tls_security_level=may + +smtp_tls_CApath=/etc/ssl/certs +smtp_tls_security_level=may +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# May need this for relaying +# relayhost = email-smtp.us-west-2.amazonaws.com:587 +# smtp_sasl_auth_enable = yes +# smtp_sasl_security_options = noanonymous +# smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd +# smtp_tls_note_starttls_offer = yes \ No newline at end of file diff --git a/deployment/docker/fs/etc/psij-testing-service/auth-psij.html b/deployment/docker/fs/etc/psij-testing-service/auth-psij.html new file mode 100644 index 0000000..8c92b45 --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/auth-psij.html @@ -0,0 +1,24 @@ + + + + +

Here is your dashboard authentication key:

+ + 
{id}:{token}
+ +

Please place it in a file named key in the ~/.psij directory + and ensure that it is only visible to you. You can do so by running the following commands + in a terminal:

+ + 
+cat <<EOF >~/.psij/key
+{id}:{token}
+EOF
+
+chmod 600 ~/.psij/key
+ + diff --git a/deployment/docker/fs/etc/psij-testing-service/auth-psij.plain b/deployment/docker/fs/etc/psij-testing-service/auth-psij.plain new file mode 100644 index 0000000..2b2754f --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/auth-psij.plain @@ -0,0 +1,8 @@ +Here is your dashboard authentication key: {id}:{token} + +Please place it in a file named "key" in the "~/.psij" directory and ensure +that it is only visible to you. You can do so by running the following +commands in a terminal: + +echo "{id}:{token}" > ~/.psij/key +chmod 600 ~/.psij/key diff --git a/deployment/docker/fs/etc/psij-testing-service/auth-sdk.html b/deployment/docker/fs/etc/psij-testing-service/auth-sdk.html new file mode 100644 index 0000000..5d7665d --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/auth-sdk.html @@ -0,0 +1,12 @@ + + + + +

Here is your dashboard authentication key:

+ + 
{id}:{token}
+ + diff --git a/deployment/docker/fs/etc/psij-testing-service/auth-sdk.plain b/deployment/docker/fs/etc/psij-testing-service/auth-sdk.plain new file mode 100644 index 0000000..6bb2fc5 --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/auth-sdk.plain @@ -0,0 +1 @@ +Here is your dashboard authentication key: {id}:{token} \ No newline at end of file diff --git a/deployment/docker/fs/etc/psij-testing-service/config-psij.json b/deployment/docker/fs/etc/psij-testing-service/config-psij.json new file mode 100644 index 0000000..33b04b0 --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/config-psij.json @@ -0,0 +1,12 @@ +{ + "project-name": "PSI/J", + "auth-email": { + "fallback-admin-email": , + "from": "noreply@testing.psij.io", + "callback-url": "https://testing.psij.io", + "body": "/etc/psij-testing-service/auth-psij", + "exception-body": "mailtemplates/exception", + "exception-approved": "mailtemplates/exception-approved", + "exception-rejected": "mailtemplates/exception-rejected" + } +} \ No newline at end of file diff --git a/deployment/docker/fs/etc/psij-testing-service/config-sdk.json b/deployment/docker/fs/etc/psij-testing-service/config-sdk.json new file mode 100644 index 0000000..dda6602 --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/config-sdk.json @@ -0,0 +1,12 @@ +{ + "project-name": "ExaWorks SDK", + "auth-email": { + "fallback-admin-email": , + "from": "noreply@sdk.testing.exaworks.org", + "callback-url": "https://sdk.testing.exaworks.org", + "body": "/etc/psij-testing-service/auth-sdk", + "exception-body": "mailtemplates/exception", + "exception-approved": "mailtemplates/exception-approved", + "exception-rejected": "mailtemplates/exception-rejected" + } +} \ No newline at end of file diff --git a/deployment/docker/fs/etc/psij-testing-service/secrets.json.template b/deployment/docker/fs/etc/psij-testing-service/secrets.json.template new file mode 100644 index 0000000..5f4bac0 --- /dev/null +++ b/deployment/docker/fs/etc/psij-testing-service/secrets.json.template @@ -0,0 +1,3 @@ +{ + "reCAPTCHA-secret-key": "" +} \ No newline at end of file diff --git a/deployment/docker/fs/usr/bin/update-psi-j-testing-service b/deployment/docker/fs/usr/bin/update-psi-j-testing-service new file mode 100755 index 0000000..d99da49 --- /dev/null +++ b/deployment/docker/fs/usr/bin/update-psi-j-testing-service @@ -0,0 +1,171 @@ +#!/bin/bash + +set -e + +try() { + N=$1 + shift + + for ((I=1; I<=$N; I++)) do + set +e + "$@" + EC=$? + set -e + if [ "$EC" == "0" ]; then + return 0 + fi + echo -n $I... + done + return 1 +} + +PACKAGE_NAME="psij-testing-service" +URL_BASE=https://github.com/ExaWorks/$PACKAGE_NAME/archive/refs/tags/v +FORCE=0 +DONTASK=0 + + +while [ "$1" != "" ]; do + case "$1" in + --force) + FORCE=1 + shift + ;; + -y) + DONTASK=1 + shift + ;; + --src) + PACKAGE="$2" + shift + shift + ;; + -*) + echo "Unrecognized option $1" + exit 1 + ;; + *) + TYPE="$1" + TARGET_VERSION="$2" + shift + shift + ;; + esac +done + +if [ "$TARGET_VERSION" == "" ] || [ "$TYPE" == "" ]; then + echo "Usage: update-psi-j-testing-service [--force] [--src path] " + exit 1 +fi + +PACKAGE="$URL_BASE$TARGET_VERSION.tar.gz" + +set -e + +if [ "${PACKAGE::1}" == "/" ]; then + # install from source + echo "Installing from source in $PACKAGE" +elif ! wget -q --method=HEAD "$PACKAGE"; then + echo "Error: failed to download $PACKAGE. Please check that the repo has a 'v$TARGET_VERSION' tag" + exit 2 +fi + +CURRENT_VERSION=`pip show $PACKAGE_NAME|grep Version|awk '{print $2}'` + +LOWER_VERSION=`echo -e "$CURRENT_VERSION\n$TARGET_VERSION"| sort -V | head -n 1` + +if [ "$FORCE" == "0" ]; then + if [ "$TARGET_VERSION" == "$LOWER_VERSION" ]; then + if [ "$CURRENT_VERSION" != "$TARGET_VERSION" ]; then + echo "Error: target version ($TARGET_VERSION) is lower than the currently" + echo "installed version ($CURRENT_VERSION). If you are sure you want to " + echo "downgrade, use the --force flag." + exit 3 + fi + fi +fi + +if [ "$DONTASK" == "0" ]; then + echo "Upgrade from $CURRENT_VERSION to $TARGET_VERSION (y/n)?" + + read ANSWER +else + ANSWER="y" +fi + +makeLink() { + LINK_NAME="$1" + if [ ! -h "$LINK_NAME" ] && [ ! -f "$LINK_NAME" ]; then + DIR=`dirname $LINK_NAME` + BASE=`basename $LINK_NAME` + NAME="${BASE%.*}" + EXT="${BASE##*.}" + ln -s "$NAME-$TYPE.$EXT" "$LINK_NAME" + fi +} + + +UPGRADES="0.4.0" + +doUpgrade_0_4_0() { + echo "Warning! updates through 0.4.0 need an additional volume attached to the container." + echo "It is not generally possible to add a volume to an existing container. You should " + echo "consider re-deploying the containers from scratch." + read -p "Press a key..." + + HOST_NAME=`cat /etc/hostname | tr -d '\n'` + DOMAIN_NAME=${HOST_NAME%%.*} + apt-get install -y postfix procmail syslog-ng + cp -n /tmp/fs/etc/postfix/main.cf /etc/postfix + sed -i "s/\$myhostname/$HOST_NAME/g" /etc/postfix/main.cf + sed -i "s/\$mydomain/$DOMAIN_NAME/g" /etc/postfix/main.cf + + + sed -i 's/system()/system(exclude-kmsg(yes))/g' /etc/syslog-ng/syslog-ng.conf + cp -n /tmp/fs/etc/postfix/header_checks /etc/postfix + if [ -f /tmp/fs/etc/postfix/sasl_passwd ]; then + cp -n /tmp/fs/etc/postfix/sasl_passwd /etc/postfix + chmod 600 /etc/postfix/sasl_passwd + postmap -v hash:/etc/postfix/sasl_passwd + fi + + service syslog-ng start + service postfix start + mkdir -p /etc/psij-testing-service + cp -n /tmp/fs/etc/psij-testing-service/* /etc/psij-testing-service + if [ ! -f /etc/psij-testing-service/secrets.json ]; then + echo "Error. No secrets.json file found for the service." + exit 1 + fi +} + +doUpgrades() { + D=`echo -e "$CURRENT_VERSION\n$TARGET_VERSION" | sort -V | tail -n 1` + if [ "$D" == "$TARGET_VERSION" ]; then + for UPGRADE in $UPGRADES; do + T=`echo -e "$CURRENT_VERSION\n$UPGRADE\n$TARGET_VERSION" | sort -V | head -n 2 | tail -n 1` + if [ "$UPGRADE" == "$T" ]; then + echo "Performing $UPGRADE upgrades..." + SUFFIX=`echo $UPGRADE | tr "." "_"` + "doUpgrade_$SUFFIX" + fi + done + else + # downgrading + echo Skipping upgrades + fi +} + +if [ "$ANSWER" == "y" ]; then + pip install "$PACKAGE" + + PACKAGE_LOC=`pip show $PACKAGE_NAME | grep "Location: " | sed 's/Location: //'` + makeLink "$PACKAGE_LOC/psij/web/instance/customization.js" + makeLink "/etc/psij-testing-service/config.json" + + try 3 timeout 10s service psi-j-testing-service stop + doUpgrades + try 3 timeout 10s service psi-j-testing-service start +else + echo "OK. Bailing out..." +fi diff --git a/deployment/docker/update-psi-j-testing-service b/deployment/docker/update-psi-j-testing-service deleted file mode 100755 index a84cea7..0000000 --- a/deployment/docker/update-psi-j-testing-service +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -try() { - N=$1 - shift - - for ((I=1; I<=$N; I++)) do - set +e - "$@" - EC=$? - set -e - if [ "$EC" == "0" ]; then - return 0 - fi - echo -n $I... - done - return 1 -} - -PACKAGE_NAME="psij-testing-service" -URL_BASE=https://github.com/ExaWorks/$PACKAGE_NAME/archive/refs/tags/v -FORCE=0 -DONTASK=0 - -while [ "$1" != "" ]; do - case "$1" in - --force) - FORCE=1 - shift - ;; - -y) - DONTASK=1 - shift - ;; - -*) - echo "Unrecognized option $1" - exit 1 - ;; - *) - TYPE="$1" - TARGET_VERSION="$2" - shift - shift - ;; - esac -done - -if [ "$TARGET_VERSION" == "" ] || [ "$TYPE" == "" ]; then - echo "Usage: update-psi-j-testing-service [--force] " - exit 1 -fi - -set -e - -PACKAGE="$URL_BASE$TARGET_VERSION.tar.gz" -if [ "${TARGET_VERSION::1}" == "/" ]; then - # install from source - echo "Installing from source in $TARGET_VERSION" - PACKAGE="$TARGET_VERSION" -elif ! wget -q --method=HEAD "$URL_BASE$TARGET_VERSION.tar.gz"; then - echo "Error: no such version. Please check that the repo has a 'v$TARGET_VERSION' tag" - exit 2 -fi - -CURRENT_VERSION=`pip show $PACKAGE_NAME|grep Version|awk '{print $2}'` - -LOWER_VERSION=`echo -e "$CURRENT_VERSION\n$TARGET_VERSION"| sort -V | head -n 1` - -if [ "$FORCE" == "0" ]; then - if [ "$TARGET_VERSION" == "$LOWER_VERSION" ]; then - if [ "$CURRENT_VERSION" != "$TARGET_VERSION" ]; then - echo "Error: target version ($TARGET_VERSION) is lower than the currently" - echo "installed version ($CURRENT_VERSION). If you are sure you want to " - echo "downgrade, use the --force flag." - exit 3 - fi - fi -fi - -if [ "$DONTASK" == "0" ]; then - echo "Upgrade from $CURRENT_VERSION to $TARGET_VERSION (y/n)?" - - read ANSWER -else - ANSWER="y" -fi - -if [ "$ANSWER" == "y" ]; then - pip install "$PACKAGE" - PACKAGE_LOC=`pip show $PACKAGE_NAME | grep "Location: " | sed 's/Location: //'` - LINK_NAME="$PACKAGE_LOC/psij/web/instance/customization.js" - if [ ! -e "$LINK_NAME" ]; then - ln -s "customization-$TYPE.js" "$LINK_NAME" - fi - try 3 timeout 10s service psi-j-testing-service restart -else - echo "OK. Bailing out..." -fi diff --git a/deployment/readme.txt b/deployment/readme.txt index 66c5611..40802f4 100644 --- a/deployment/readme.txt +++ b/deployment/readme.txt @@ -12,7 +12,15 @@ To make a release of the service: To deploy the service(s): - make a release as above for deployment -- edit config and fill in appropriate values if necessary +- check out the psij-testing-service repo on the target machine +- edit the following and populate with appropriate values: + - config + - deploy/DOMAIN_NAME + - docker/fs/etc; make sure you: + - create a secrets.json in psij-testing-service + - the config-*.json files are valid and have proper values + - web/instance; add reCapthcaSiteKey + - run deploy.sh from the deploy directory. This will: - install or upgrade nginx - start or upgrade the PSI/J and SDK containers to the version diff --git a/deployment/web/instance/customization-psij.js b/deployment/web/instance/customization-psij.js new file mode 100644 index 0000000..dc7e26a --- /dev/null +++ b/deployment/web/instance/customization-psij.js @@ -0,0 +1,19 @@ +CUSTOMIZATION = { + projectName: "PSI/J", + title: "PSI/J Testing Dashboard", + // If not specified, relative links are used, so it works on any domain + backendURL: "", + reCaptchaSiteKey: "", + testsTable: { + tests: [ + "test_simple_job[local:single]", + "test_parallel_jobs[local:single]" + ], + testNames: [ + "Basic test", + "Parallel jobs" + ], + suiteName: "Test suite" + } +}; + diff --git a/deployment/web/instance/customization-sdk.js b/deployment/web/instance/customization-sdk.js new file mode 100644 index 0000000..bcdb67b --- /dev/null +++ b/deployment/web/instance/customization-sdk.js @@ -0,0 +1,23 @@ +CUSTOMIZATION = { + projectName: "SDK", + title: "Exascale dashboard testing service", + backendURL: "https://sdk.testing.exaworks.org/", + reCaptchaSiteKey: "", + testsTable: { + tests: [ + "flux", + "parsl", + "parsl-flux", + "rp", + "swift-t" + ], + testNames: [ + "Flux", + "Parsl", + "Parsl-Flux", + "Radical", + "Swift/T" + ], + suiteName: "Test suite" + } +}; diff --git a/deployment/web/psij/customization.js b/deployment/web/psij/customization.js deleted file mode 100644 index a0b55df..0000000 --- a/deployment/web/psij/customization.js +++ /dev/null @@ -1,3 +0,0 @@ -CUSTOMIZATION = { - projectName: "PSI/J" -} \ No newline at end of file diff --git a/deployment/web/sdk/customization.js b/deployment/web/sdk/customization.js deleted file mode 100644 index c5c7568..0000000 --- a/deployment/web/sdk/customization.js +++ /dev/null @@ -1,3 +0,0 @@ -CUSTOMIZATION = { - projectName: "SDK" -} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ce0f0d9..539b70b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ cherrypy mongoengine >= 0.23.1 -python-dateutil >= 2.8.1 \ No newline at end of file +python-dateutil >= 2.8.1 +bcrypt == 3.2.2 +requests \ No newline at end of file diff --git a/setup.py b/setup.py index f9d446e..acb4fed 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def get_files(root: str) -> List[str]: package_data={ '': ['README.md', 'LICENSE', 'RELEASE'], - 'psij': get_files('src/psij/web') + 'psij': get_files('src/psij') }, scripts=[], diff --git a/src/psij/testing/config.json b/src/psij/testing/config.json new file mode 100644 index 0000000..1b4705a --- /dev/null +++ b/src/psij/testing/config.json @@ -0,0 +1,12 @@ +{ + "project-name": "Example", + "auth-email": { + "fallback-admin-email": "admin@example.org", + "from": "noreply@example.org", + "callback-url": "https://example.org", + "body": "mailtemplates/auth", + "exception-body": "mailtemplates/exception", + "exception-approved": "mailtemplates/exception-approved", + "exception-rejected": "mailtemplates/exception-rejected" + } +} \ No newline at end of file diff --git a/src/psij/testing/mailtemplates/auth.html b/src/psij/testing/mailtemplates/auth.html new file mode 100644 index 0000000..5d7665d --- /dev/null +++ b/src/psij/testing/mailtemplates/auth.html @@ -0,0 +1,12 @@ + + + + +

Here is your dashboard authentication key:

+ + 
{id}:{token}
+ + diff --git a/src/psij/testing/mailtemplates/auth.plain b/src/psij/testing/mailtemplates/auth.plain new file mode 100644 index 0000000..6bb2fc5 --- /dev/null +++ b/src/psij/testing/mailtemplates/auth.plain @@ -0,0 +1 @@ +Here is your dashboard authentication key: {id}:{token} \ No newline at end of file diff --git a/src/psij/testing/mailtemplates/exception-approved.html b/src/psij/testing/mailtemplates/exception-approved.html new file mode 100644 index 0000000..cc86874 --- /dev/null +++ b/src/psij/testing/mailtemplates/exception-approved.html @@ -0,0 +1,11 @@ + + + + +

Your request to allow an exception for has been approved.

+

Please go to {authpage} to request an authentication key.

+ + diff --git a/src/psij/testing/mailtemplates/exception-approved.plain b/src/psij/testing/mailtemplates/exception-approved.plain new file mode 100644 index 0000000..993cab5 --- /dev/null +++ b/src/psij/testing/mailtemplates/exception-approved.plain @@ -0,0 +1,2 @@ +Your request to allow an exception for {email} has been approved. +Please go to {authpage} to request an authentication key. \ No newline at end of file diff --git a/src/psij/testing/mailtemplates/exception-rejected.html b/src/psij/testing/mailtemplates/exception-rejected.html new file mode 100644 index 0000000..ef061aa --- /dev/null +++ b/src/psij/testing/mailtemplates/exception-rejected.html @@ -0,0 +1,10 @@ + + + + +

Your request to allow an exception for has been rejected.

+ + diff --git a/src/psij/testing/mailtemplates/exception-rejected.plain b/src/psij/testing/mailtemplates/exception-rejected.plain new file mode 100644 index 0000000..8ee78fe --- /dev/null +++ b/src/psij/testing/mailtemplates/exception-rejected.plain @@ -0,0 +1 @@ +Your request to allow an exception for {email} has been rejected. diff --git a/src/psij/testing/mailtemplates/exception.html b/src/psij/testing/mailtemplates/exception.html new file mode 100644 index 0000000..9fb5344 --- /dev/null +++ b/src/psij/testing/mailtemplates/exception.html @@ -0,0 +1,23 @@ + + + + +

Request for email exception:

+ + + +

Reason provided by user:

+ +

{reason}

+ + Reject: {rejectUrl} + Approve: {approveUrl} + + diff --git a/src/psij/testing/mailtemplates/exception.plain b/src/psij/testing/mailtemplates/exception.plain new file mode 100644 index 0000000..2d36671 --- /dev/null +++ b/src/psij/testing/mailtemplates/exception.plain @@ -0,0 +1,10 @@ +Request for email exception. + +Email: {email} + +Reason provided by user: + +{reason} + +Reject: {rejectUrl} +Approve: {approveUrl} diff --git a/src/psij/testing/secrets.json.template b/src/psij/testing/secrets.json.template new file mode 100644 index 0000000..5f4bac0 --- /dev/null +++ b/src/psij/testing/secrets.json.template @@ -0,0 +1,3 @@ +{ + "reCAPTCHA-secret-key": "" +} \ No newline at end of file diff --git a/src/psij/testing/service.py b/src/psij/testing/service.py index cfda04f..5dc8bcf 100644 --- a/src/psij/testing/service.py +++ b/src/psij/testing/service.py @@ -1,18 +1,28 @@ import argparse +import os +import smtplib +import tempfile +import traceback +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +import bcrypt import datetime import json -import logging +import secrets import sys from pathlib import Path -from typing import Optional, Dict, cast +from typing import Optional, Dict, cast, Tuple, Any, List import cherrypy +import requests from bson import ObjectId -from mongoengine import Document, StringField, DateTimeField, connect, DictField, BooleanField, \ - IntField +from mongoengine import Document, StringField, DateTimeField, connect, DictField, \ + IntField, NotUniqueError - -CODE_DB_VERSION = 2 +CODE_DB_VERSION = 4 +EMAIL_BLOCKLIST_URL = 'https://raw.githubusercontent.com/disposable-email-domains/' \ + 'disposable-email-domains/master/disposable_email_blocklist.conf' def upgrade_0_to_1() -> None: @@ -27,9 +37,28 @@ def upgrade_1_to_2() -> None: Test.create_index(['site_id', 'run_id', 'branch']) +def upgrade_2_to_3() -> None: + # Mongoengine automatically creates indexes on unique fields? + # Auth.create_index(['key_id']) + # AuthExceptionRequests.create_index(['req_id']) + # AuthDisabledDomains.create_index(['domain']) + AuthAllowedEmails.create_index(['email']) + + +def _add_disabled_domain(domain: str) -> None: + AuthDisabledDomains(domain=domain).save() + + +def upgrade_3_to_4() -> None: + for domain in ['gmail.com', 'yahoo.com']: + _add_disabled_domain(domain) + + DB_UPGRADES = { 0: upgrade_0_to_1, - 1: upgrade_1_to_2 + 1: upgrade_1_to_2, + 2: upgrade_2_to_3, + 3: upgrade_3_to_4 } class Version(Document): @@ -74,6 +103,40 @@ class RunEnv(Document): skipped_count = IntField(default=0) +class Auth(Document): + key_id = StringField(required=True, unique=True) + hash = StringField(required=True) + email = StringField(required=True) + last_used = DateTimeField(required=True) + # the rounds for bcrypt; just default for now, but if that changes, we need to distinguish + # between entries that were encoded with one round vs something else + rounds = IntField(required=True, default=1) + + +# email domains for which we do not allow registration +class AuthDisabledDomains(Document): + domain = StringField(required=True, unique=True) + + +# exceptions to AuthDisabledDomains +class AuthAllowedEmails(Document): + email = StringField(required=True) + approved_by = StringField(required=True) + approved_on = DateTimeField(required=True) + approved_by_ip = StringField(required=True) + + +# send admin requests to these emails +class AuthAdminEmails(Document): + email = StringField(required=True, unique=True) + + +class AuthExceptionRequests(Document): + req_id = StringField(required=True, unique=True) + email = StringField(required=True) + approver_email = StringField(required=True) + + def strtime(d): return d.strftime('%a, %b %d, %Y - %H:%M') @@ -101,9 +164,23 @@ def add_cors_headers(): headers['Access-Control-Allow-Headers'] = 'Content-Type, Accept' +class AuthError(Exception): + def __init__(self, error: str, email: str = '', banned_domain: bool = False, domain: str = '') -> None: + self.error = error + self.email = email + self.banned_domain = banned_domain + self.domain = domain + + def to_object(self): + return {'success': False, 'error': self.error, 'email': self.email, + 'bannedDomain': self.banned_domain, 'domain': self.domain} + + class TestingAggregatorApp(object): - def __init__(self): + def __init__(self, config: Dict[str, Any], secrets: Dict[str, object]) -> None: self.seq = 0 + self.config = config + self.secrets = secrets @cherrypy.expose def index(self) -> None: @@ -116,14 +193,17 @@ def result(self) -> None: if not 'id' in json: raise cherrypy.HTTPError(400, 'Missing id') if not 'key' in json: - raise cherrypy.HTTPError(400, 'Missing key') + raise cherrypy.HTTPError(400, 'Missing key. Please go to /auth.html to request a key.') site_id = json['id'] key = json['key'] data = json['data'] - site = self._check_authorized(site_id, key) - if not site: - raise cherrypy.HTTPError(403, 'This ID is associated with another key') + if ':' in key: + if not self._check_authorized(site_id, key): + raise cherrypy.HTTPError(403, 'Invalid key. Please go to /auth.html to request a new key.') + else: + if not self._check_authorized_legacy(site_id, key): + raise cherrypy.HTTPError(403, 'This ID is associated with another key') try: self.seq += 1 @@ -133,7 +213,7 @@ def result(self) -> None: function = data['function'] if module == '_conftest': if function == '_discover_environment': - self._save_environment(site, data) + self._save_environment(site_id, data) if function == '_end': self._end_tests(site_id, data) @@ -142,7 +222,7 @@ def result(self) -> None: cherrypy.log('Request json: %s' % json) raise - def _update_totals(self, site_id, data: Dict[str, object]) -> None: + def _update_totals(self, site_id: str, data: Dict[str, Any]) -> None: run_id = data['run_id'] branch = data['branch'] @@ -150,6 +230,7 @@ def _update_totals(self, site_id, data: Dict[str, object]) -> None: failed = False skipped = False for k, v in results.items(): + if v['status'] == 'failed': failed = True if k == 'call' and v['status'] == 'skipped': @@ -171,41 +252,43 @@ def _save_test(self, site_id: str, data: Dict[str, object]) -> None: data['site_id'] = site_id Test(**data).save() - def _save_environment(self, site: Site, data: Dict[str, object]) -> None: + def _save_environment(self, site_id: str, data: Dict[str, object]) -> None: env = cast(Dict[str, object], data['extras']) config = cast(Dict[str, object], env['config']) del env['config'] maintainer_email = config['maintainer_email'] + site = Site.objects(site_id=site_id).first() + if site is None: + self._update(Site(site_id=site_id, key='', ip=cherrypy.request.remote.ip)) if maintainer_email: site.crt_maintainer_email = maintainer_email site.save() - run_env = RunEnv(run_id=data['run_id'], site_id=site.site_id, config=config, env=env, + run_env = RunEnv(run_id=data['run_id'], site_id=site_id, config=config, env=env, run_start_time=env['start_time'], branch=env['git_branch']) run_env.save() - def _check_authorized(self, id: str, key: str) -> Optional[Site]: + def _check_authorized(self, site_id:str, key: str) -> bool: + id, token = self._split_key(key) + return self._verify_key(id, token) + + def _check_authorized_legacy(self, site_id: str, key: str) -> bool: entries = Site.objects(site_id=id) entry = entries.first() if entry: entry.ip = cherrypy.request.remote.ip if key == entry.key: - return self._update(entry) + self._update(entry) + return True else: - now = datetime.datetime.utcnow() - diff = now - entry.last_seen - if diff >= datetime.timedelta(days=7): - entry.key = key # update with the new key - return self._update(entry) - else: - return None + # we do not allow ad-hoc keys any more + return False else: # nothing yet - return self._update(Site(site_id=id, key=key, ip=cherrypy.request.remote.ip)) + return False - def _update(self, entry: Site) -> Site: + def _update(self, entry: Site) -> None: entry.last_seen = datetime.datetime.utcnow() entry.save() - return entry @cherrypy.expose @cherrypy.tools.json_out() @@ -220,7 +303,7 @@ def summary(self, inactiveTimeout: str = "10") -> object: except ValueError: iInactiveTimeout = 0 if iInactiveTimeout <= 0: - date_limit = datetime.date.min + date_limit = datetime.datetime.min time_limit = datetime.datetime.combine(date_limit, datetime.datetime.min.time()) resp = [] @@ -236,7 +319,7 @@ def summary(self, inactiveTimeout: str = "10") -> object: # now find all envs/branches with this run id envs = RunEnv.objects(site_id=site.site_id, run_id=run_id) - branches = [] + branches: List[Dict[str, Any]] = [] site_data = { 'site_id': site.site_id, 'run_id': run_id, @@ -280,7 +363,7 @@ def summary(self, inactiveTimeout: str = "10") -> object: month = date_start.month if month not in site_data['months']: - month_data = {} # type Dict[int, object] + month_data: Dict[int, object] = {} site_data['months'][month] = month_data else: month_data = site_data['months'][month] @@ -310,8 +393,6 @@ def summary(self, inactiveTimeout: str = "10") -> object: 'name': k }) - - date_start = date_start + datetime.timedelta(days=-1) add_cors_headers() @@ -323,12 +404,12 @@ def site(self, site_id) -> object: s = Site.objects(site_id=site_id).first() resp = {} resp['site_id'] = site_id - test_runs = [] + test_runs: List[Dict[str, object]] = [] resp['test_runs'] = test_runs runs = RunEnv.objects(site_id=site_id).order_by('-run_start_time', '+branch')[:100] - seen = {} + seen: Dict[str, Dict[str, Any]] = {} for run in runs: if run.run_id in seen: run_set = seen[run.run_id] @@ -353,16 +434,16 @@ def site(self, site_id) -> object: @cherrypy.tools.json_out() def run(self, site_id, run_id) -> object: s = Site.objects(site_id=site_id).first() - resp = {} + resp: Dict[str, object] = {} resp['site_id'] = site_id resp['run_id'] = run_id - branches = [] + branches: List[Dict[str, object]] = [] resp['branches'] = branches runs = RunEnv.objects(site_id=site_id, run_id=run_id).order_by('+branch') for run in runs: - test_list = [] + test_list: List[Dict[str, object]] = [] branch = run.to_mongo().to_dict() branch['tests'] = test_list branch['name'] = run.branch @@ -395,7 +476,7 @@ def tests(self, sites_to_get, tests_to_match) -> object: test_start_time__gt=time_limit, test_name__in=tests_to_match).order_by('-test_start_time') - resp = {} + resp: Dict[str, Dict[str, object]] = {} for test in tests: if test.site_id not in resp: @@ -411,10 +492,231 @@ def tests(self, sites_to_get, tests_to_match) -> object: add_cors_headers() return resp + @cherrypy.expose + @cherrypy.tools.json_out() + def authRequest(self, email: str, ctoken: str) -> object: + try: + self._verify_captcha_token(ctoken) + ix = email.find('@') + if ix == -1: + raise AuthError('The email you provided has an incorrect syntax.', email) + domain = email[ix + 1:] + + if AuthDisabledDomains.objects(domain=domain).count() > 0: + if AuthAllowedEmails.objects(email=email).count() > 0: + self._perform_auth_request(email) + else: + raise AuthError('Invalid domain', email, banned_domain=True, domain=domain) + else: + self._perform_auth_request(email) + except AuthError as err: + return err.to_object() + except Exception: + traceback.print_exc() + return AuthError('Internal error', email=email).to_object() + + return {'success': True} + + def _verify_captcha_token(self, ctoken: str) -> None: + r = requests.post('https://www.google.com/recaptcha/api/siteverify', + data = {'secret': self.secrets['reCAPTCHA-secret-key'], + 'response': ctoken}) + if r.status_code != 200: + print(r.json()) + raise AuthError('reCAPTCHA verify error', email='') + rj = r.json() + if not rj['success']: + raise AuthError('reCAPTCHA verify error', email='') + + def _perform_auth_request(self, email: str) -> None: + salt = bcrypt.gensalt() + token = secrets.token_hex(24) + encrypted_token = bcrypt.hashpw(token.encode('ascii'), salt) + + for tries in range(3): + id = secrets.token_hex(8) + try: + Auth.save(Auth(key_id=id, email=email, hash=encrypted_token, + last_used=datetime.datetime.utcnow())) + return self._send_key_email(email, id, token) + except NotUniqueError: + pass + + raise AuthError('Failed to generate authentication key', email) + + def _load_email_body(self, file_name: str) -> str: + if file_name[0] == '/': + path = file_name + else: + dir = os.path.dirname(__file__) + path = dir + '/' + file_name + + with open(path) as f: + return f.read() + + def _load_email_bodies(self, file_prefix: str, params: Dict[str, str]) -> MIMEMultipart: + body_plain = self._load_email_body(file_prefix + '.plain').format(**params) + body_html = self._load_email_body(file_prefix + '.html').format(**params) + msg = MIMEMultipart('alternative') + msg.attach(MIMEText(body_plain, 'plain')) + msg.attach(MIMEText(body_html, 'html')) + return msg + + + def _send_key_email(self, email: str, id: str, token: str) -> None: + + msg = self._load_email_bodies(self.config['auth-email']['body'], {'id': id, 'token': token}) + + source = self.config['auth-email']['from'] + + msg['Subject'] = 'Your ' + self.config['project-name'] + ' testing dashboard key' + msg['From'] = source + msg['To'] = email + + self._send_email(source, email, msg) + + def _send_email(self, source: str, email: str, msg: MIMEMultipart) -> None: + smtp = smtplib.SMTP('localhost') + smtp.sendmail(source, email, msg.as_string()) + smtp.quit() + + @cherrypy.expose + @cherrypy.tools.json_out() + def authRevoke(self, project: str, key: str, ctoken: str) -> object: + try: + self._verify_captcha_token(ctoken) + + id, token = self._split_key(key) + if not self._verify_key(id, token): + raise AuthError('Invalid key') + Auth.objects(key_id=id).delete() + except AuthError as err: + return err.to_object() + except Exception: + traceback.print_exc() + return AuthError('Internal error').to_object() + + return {'success': True} + + def _verify_key(self, id: str, key: str) -> bool: + auth = Auth.objects(key_id=id).first() + + encrypted_key = bcrypt.hashpw(key.encode('ascii'), auth.hash.encode('ascii')) + if encrypted_key.decode('ascii') == auth.hash: + auth.update(last_used=datetime.datetime.utcnow()) + return True + else: + return False + + def _split_key(self, key: str) -> Tuple[str, str]: + ix = key.find(':') + if ix == -1: + raise AuthError('Invalid code') + + return key[:ix], key[ix + 1:] + + @cherrypy.expose + @cherrypy.tools.json_out() + def authExceptionRequest(self, email: str, reason: str) -> object: + try: + self._send_exception_emails(email, reason) + except AuthError as err: + return err.to_object() + except Exception: + traceback.print_exc() + return AuthError('Internal error').to_object() + + return {'success': True} + + def _send_exception_emails(self, email: str, reason: str) -> None: + emails = [o.email for o in AuthAdminEmails.objects()] + if len(emails) == 0: + emails = [self.config['auth-email']['fallback-admin-email']] + for admin in emails: + id = secrets.token_hex(16) + AuthExceptionRequests(req_id=id, email=email, approver_email=admin).save() + self._send_exception_email(admin, email, reason, id) + + def _send_exception_email(self, to: str, email: str, reason: str, id: str) -> None: + + approve_url = self.config['auth-email']['callback-url'] \ + + '/exception-control.html?action=approve&req_id=' + id + reject_url = self.config['auth-email']['callback-url'] \ + + '/exception-control.html?action=reject&req_id=' + id + project_name = self.config['project-name'] + + msg = self._load_email_bodies(self.config['auth-email']['exception-body'], + {'project': project_name, 'email': email, 'reason': reason, + 'approveUrl': approve_url, 'rejectUrl': reject_url}) + + source = self.config['auth-email']['from'] + + msg['Subject'] = project_name + ' email exception request' + msg['From'] = source + msg['To'] = to + + self._send_email(source, to, msg) + + @cherrypy.expose + @cherrypy.tools.json_out() + def authExceptionAction(self, req_id: str, action: str) -> object: + try: + reqs = AuthExceptionRequests.objects(req_id=req_id) + if len(reqs) == 0: + raise AuthError('Exception request not found') + req = reqs.first() + if action == 'approve': + headers = cherrypy.request.headers + if 'X-Forwarded-For' in headers: + ip = headers['X-Forwarded-For'] + else: + ip = cherrypy.request.remote.ip + AuthAllowedEmails(email=req.email, approved_by=req.approver_email, + approved_on=datetime.datetime.utcnow(), + approved_by_ip=ip).save() + self._send_exception_confirmation_email(req.email, action) + req.delete() + except AuthError as err: + return err.to_object() + except Exception: + traceback.print_exc() + return AuthError('Internal error').to_object() + + return {'success': True} + + def _send_exception_confirmation_email(self, to: str, action: str) -> None: + authpage = self.config['auth-email']['callback-url'] + '/auth.html' + project_name = self.config['project-name'] + if action == 'approve': + msg = self._load_email_bodies(self.config['auth-email']['exception-approved'], + {'email': to, 'authpage': authpage}) + msg['Subject'] = project_name + ' email exception approved' + else: + msg = self._load_email_bodies(self.config['auth-email']['exception-rejected'], + {'email': to, 'authpage': authpage}) + msg['Subject'] = project_name + ' email exception rejected' + + source = self.config['auth-email']['from'] + + msg['From'] = source + msg['To'] = to + + self._send_email(source, to, msg) class Server: - def __init__(self, port: int = 9909) -> None: + def __init__(self, port: int = 9909, config_path: str = 'config.json', + secrets_path: str = 'secrets.json') -> None: self.port = port + self.config = self._read_file(config_path) + self.secrets = self._read_file(secrets_path) + + def _read_file(self, path: str) -> Dict[str, object]: + if path[0] == '/': + abs_path = path + else: + abs_path = os.path.dirname(__file__) + '/' + path + with open(abs_path, 'r') as f: + return json.load(f) def start(self) -> None: print('webpath: %s' % (Path().absolute() / 'web')) @@ -429,7 +731,7 @@ def json_handler(*args, **kwargs): 'server.socket_port': self.port, 'server.socket_host': '0.0.0.0' }) - cherrypy.quickstart(TestingAggregatorApp(), '/', { + cherrypy.quickstart(TestingAggregatorApp(self.config, self.secrets), '/', { '/': { 'tools.staticdir.root': str(Path(__file__).parent.parent.absolute() / 'web'), 'tools.staticdir.on': True, @@ -461,13 +763,30 @@ def check_db() -> None: v = upgrade_db(v) +def update_email_blocklist(): + print('updating blocklist') + r = requests.get(EMAIL_BLOCKLIST_URL) + r.raise_for_status() + for line in r.content.splitlines(): + try: + AuthDisabledDomains(domain=line).save() + except NotUniqueError: + pass + + def main() -> None: check_db() + update_email_blocklist() parser = argparse.ArgumentParser(description='Starts test aggregation server') parser.add_argument('-p', '--port', action='store', type=int, default=9909, help='The port on which to start the server.') + parser.add_argument('-c', '--config', action='store', type=str, default='config.json', + help='A configuration file. A relative path points to a file inside the ' + 'source package.') + parser.add_argument('-s', '--secrets', action='store', type=str, default='secrets.json', + help='A file containing authentication keys/tokens.') args = parser.parse_args(sys.argv[1:]) - server = Server(args.port) + server = Server(args.port, args.config, args.secrets) server.start() diff --git a/src/psij/web/auth.html b/src/psij/web/auth.html new file mode 100644 index 0000000..da7be1b --- /dev/null +++ b/src/psij/web/auth.html @@ -0,0 +1,288 @@ + + + Authentication key management + + + + + + + + + + + + + + + +
+
+ + + Request + Revoke + + + + + + Request authentication key + +
Enter your email below to receive an authentication key that + will allow you to upload test results to this dashboard. The email you enter here + will only be used as part of the process of obtaining a key.
+ + + + + + + + + + + +
+ + + + Request key + + + + + + + Requesting key... + + + + + + + + + + + + Revoke authentication key + +
Enter an authentication key to revoke. After revocation, test + uploads using this authentication key will be rejected.
+ + + + + + + + + + + +
+ + + + Revoke key + + + + + + + + + + + + + +
+
+ + + + + + \ No newline at end of file diff --git a/src/psij/web/css/style.css b/src/psij/web/css/style.css index 12dc788..086eb1b 100644 --- a/src/psij/web/css/style.css +++ b/src/psij/web/css/style.css @@ -700,3 +700,15 @@ td.run-id { padding: 0; } +#auth .sep { + display: block; + height: 36pt; +} + +#auth .theme--light.v-application { + background-color: #f0f0f0; +} + +#auth .v-tabs { + flex: none; +} \ No newline at end of file diff --git a/src/psij/web/exception-control.html b/src/psij/web/exception-control.html new file mode 100644 index 0000000..5cc3e07 --- /dev/null +++ b/src/psij/web/exception-control.html @@ -0,0 +1,106 @@ + + + Authentication code management + + + + + + + + + + + + + + +
+
+ + + + + +
+
+ + + + + + \ No newline at end of file diff --git a/src/psij/web/instance/customization-psij.js b/src/psij/web/instance/customization-psij.js index 9f161e7..dc7e26a 100644 --- a/src/psij/web/instance/customization-psij.js +++ b/src/psij/web/instance/customization-psij.js @@ -1,7 +1,9 @@ CUSTOMIZATION = { projectName: "PSI/J", title: "PSI/J Testing Dashboard", - backendURL: "https://testing.psij.io/", + // If not specified, relative links are used, so it works on any domain + backendURL: "", + reCaptchaSiteKey: "", testsTable: { tests: [ "test_simple_job[local:single]", diff --git a/src/psij/web/instance/customization-sdk.js b/src/psij/web/instance/customization-sdk.js index 026d76b..bcdb67b 100644 --- a/src/psij/web/instance/customization-sdk.js +++ b/src/psij/web/instance/customization-sdk.js @@ -2,6 +2,7 @@ CUSTOMIZATION = { projectName: "SDK", title: "Exascale dashboard testing service", backendURL: "https://sdk.testing.exaworks.org/", + reCaptchaSiteKey: "", testsTable: { tests: [ "flux", diff --git a/src/psij/web/js/auth-common.js b/src/psij/web/js/auth-common.js new file mode 100644 index 0000000..c494336 --- /dev/null +++ b/src/psij/web/js/auth-common.js @@ -0,0 +1,27 @@ +function genericSubmit(vue, path, params, successMessage, crtReCaptcha) { + vue.requestSubmitDialog = true; + if (crtReCaptcha) { + grecaptcha.reset(crtReCaptcha); + } + $.post(PS.getURL(path), params, function(data) { + vue.requestSubmitDialog = false; + if (!data) { + vue.showErrorDialog("Server did not return any data. Please try again later."); + } + else if (!data.success) { + if (data.bannedDomain) { + vue.showBannedDomainDialog(data.domain, data.email); + } + else { + vue.showErrorDialog(data.error); + } + } + else { + vue.showSuccessDialog(successMessage); + } + }).fail(function(jqXHR, textStatus, errorThrown) { + console.log(jqXHR.responseText); + vue.requestSubmitDialog = false; + vue.showErrorDialog("An unknown error has occurred. Please try again later."); + }); +} \ No newline at end of file