diff --git a/pi/balena.yml b/balena.yml similarity index 100% rename from pi/balena.yml rename to balena.yml diff --git a/pi/docker-compose.yml b/docker-compose.yml similarity index 97% rename from pi/docker-compose.yml rename to docker-compose.yml index 4ac93eb..18f40ce 100644 --- a/pi/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: - 'resin-data:/logs_shared' api: - build: api + build: server privileged: true restart: always depends_on: @@ -75,7 +75,7 @@ services: - 'resin-data:/logs_shared' queue: - build: queue + build: server privileged: true restart: always depends_on: @@ -90,7 +90,7 @@ services: - 'resin-data:/logs_shared' websocket: - build: websocket + build: server privileged: true restart: always depends_on: @@ -111,7 +111,7 @@ services: - 'resin-data:/logs_shared' ui: - build: ui + build: pi/ui privileged: true restart: always depends_on: diff --git a/pi/push_dev.cmd b/pi/push_dev.cmd deleted file mode 100644 index 4facfc7..0000000 --- a/pi/push_dev.cmd +++ /dev/null @@ -1,56 +0,0 @@ -COPY ui\.env.development ..\client\.env -cd ..\client -call npm run --silent build -cd ..\pi - -echo Creating API container -xcopy /q ..\server\src api\server\src /I /S /Y -del api\server\*.json -copy ..\server\package.json api\server /Y -copy ..\server\package-lock.json api\server /Y -copy ..\server\config_prod.json api\server\config.json /Y -copy ..\server\config_prod.json api\server\config_prod.json /Y -copy ..\server\config_pi.json api\server\config_pi.json /Y -copy ..\server\config_ci.json api\server\config_ci.json /Y -copy ..\server\config_test.json api\server\config_test.json /Y -copy ..\server\config_dev.json api\server\config_dev.json /Y -xcopy /q ..\server\migrations api\migrations /I /S /Y - -echo Creating ALERT container -xcopy /q ..\server\src alert\server\src /I /S /Y -copy ..\server\package.json alert\server /Y -copy ..\server\package-lock.json alert\server /Y -copy ..\server\config_prod.json alert\server\config.json /Y -copy ..\server\config_prod.json alert\server\config_prod.json /Y -copy ..\server\config_pi.json alert\server\config_pi.json /Y -copy ..\server\config_ci.json alert\server\config_ci.json /Y -copy ..\server\config_test.json alert\server\config_test.json /Y -copy ..\server\config_dev.json alert\server\config_dev.json /Y - -echo Creating NOTIFY container -xcopy /q ..\server\src notify\server\src /I /S /Y -copy ..\server\package.json notify\server /Y -copy ..\server\package-lock.json notify\server /Y -copy ..\server\config_prod.json notify\server\config.json /Y -copy ..\server\config_prod.json notify\server\config_prod.json /Y -copy ..\server\config_pi.json notify\server\config_pi.json /Y -copy ..\server\config_ci.json notify\server\config_ci.json /Y -copy ..\server\config_test.json notify\server\config_test.json /Y -copy ..\server\config_dev.json notify\server\config_dev.json /Y - -echo Creating QUEUE container -xcopy /q ..\server\src queue\server\src /I /S /Y -copy ..\server\package.json queue\server /Y -copy ..\server\package-lock.json queue\server /Y - -echo Creating UI container -xcopy /q ..\client\build ui\build /I /S /Y -copy ..\client\package.json ui\client /Y -copy ..\client\package-lock.json ui\client /Y - -echo Creating WEBSOCKET container -xcopy /q ..\server\src websocket\server\src /I /S /Y -copy ..\server\package.json websocket\server /Y -copy ..\server\package-lock.json websocket\server /Y - -balena push bubblesnet4_controller_dev \ No newline at end of file diff --git a/pi/push_prod.cmd b/pi/push_prod.cmd deleted file mode 100644 index eb81662..0000000 --- a/pi/push_prod.cmd +++ /dev/null @@ -1,32 +0,0 @@ -COPY ui\.env.production ..\client\.env -cd ..\client -call npm run --silent build -cd ..\pi - -xcopy /q ..\server\src api\server\src /I /S /Y -del api\server\*.json -copy ..\server\package.json api\server /Y -copy ..\server\package-lock.json api\server /Y -copy ..\server\config_prod.json api\server\config.json /Y -copy ..\server\config_prod.json api\server\config_prod.json /Y -copy ..\server\config_pi.json api\server\config_pi.json /Y -copy ..\server\config_ci.json api\server\config_ci.json /Y -copy ..\server\config_test.json api\server\config_test.json /Y -copy ..\server\config_dev.json api\server\config_dev.json /Y - -xcopy /q ..\server\migrations api\migrations /I /S /Y - -xcopy /q ..\server\src queue\server\src /I /S /Y -copy ..\server\package.json queue\server /Y -copy ..\server\package-lock.json queue\server /Y - -xcopy /q ..\client\build ui\build /I /S /Y -copy ..\client\package.json ui\client /Y -copy ..\client\package-lock.json ui\client /Y - -xcopy /q ..\server\src websocket\server\src /I /S /Y -copy ..\server\package.json websocket\server /Y -copy ..\server\package-lock.json websocket\server /Y - -balena push bubblesnet4_controller_aarch64_prod - diff --git a/pi/push_coral.cmd b/push_coral.cmd similarity index 100% rename from pi/push_coral.cmd rename to push_coral.cmd diff --git a/push_dev.cmd b/push_dev.cmd new file mode 100644 index 0000000..8be0ce8 --- /dev/null +++ b/push_dev.cmd @@ -0,0 +1,32 @@ +COPY ui\.env.production client\.env +cd client +call npm run --silent build +cd .. + +xcopy /q server\src pi\api\server\src /I /S /Y +del api\server\*.json +copy server\package.json pi\api\server /Y +copy server\package-lock.json pi\api\server /Y +copy server\config_prod.json pi\api\server\config.json /Y +copy server\config_prod.json pi\api\server\config_prod.json /Y +copy server\config_pi.json pi\api\server\config_pi.json /Y +copy server\config_ci.json pi\api\server\config_ci.json /Y +copy server\config_test.json pi\api\server\config_test.json /Y +copy server\config_dev.json pi\api\server\config_dev.json /Y + +xcopy /q server\migrations pi\api\migrations /I /S /Y + +xcopy /q server\src pi\queue\server\src /I /S /Y +copy server\package.json pi\queue\server /Y +copy server\package-lock.json pi\queue\server /Y + +xcopy /q client\build pi\ui\build /I /S /Y +copy client\package.json pi\ui\client /Y +copy client\package-lock.json pi\ui\client /Y + +xcopy /q server\src pi\websocket\server\src /I /S /Y +copy server\package.json pi\websocket\server /Y +copy server\package-lock.json pi\websocket\server /Y + +balena push bubblesnet4_controller_dev + diff --git a/push_prod.cmd b/push_prod.cmd new file mode 100644 index 0000000..9b5e3dc --- /dev/null +++ b/push_prod.cmd @@ -0,0 +1,32 @@ +COPY ui\.env.production client\.env +cd client +call npm run --silent build +cd .. + +xcopy /q server\src pi\api\server\src /I /S /Y +del api\server\*.json +copy server\package.json pi\api\server /Y +copy server\package-lock.json pi\api\server /Y +copy server\config_prod.json pi\api\server\config.json /Y +copy server\config_prod.json pi\api\server\config_prod.json /Y +copy server\config_pi.json pi\api\server\config_pi.json /Y +copy server\config_ci.json pi\api\server\config_ci.json /Y +copy server\config_test.json pi\api\server\config_test.json /Y +copy server\config_dev.json pi\api\server\config_dev.json /Y + +xcopy /q server\migrations pi\api\migrations /I /S /Y + +xcopy /q server\src pi\queue\server\src /I /S /Y +copy server\package.json pi\queue\server /Y +copy server\package-lock.json pi\queue\server /Y + +xcopy /q client\build pi\ui\build /I /S /Y +copy client\package.json pi\ui\client /Y +copy client\package-lock.json pi\ui\client /Y + +xcopy /q server\src pi\websocket\server\src /I /S /Y +copy server\package.json pi\websocket\server /Y +copy server\package-lock.json pi\websocket\server /Y + +balena push bubblesnet4_controller_aarch64_prod + diff --git a/server/Dockerfile.template b/server/Dockerfile.template new file mode 100644 index 0000000..7a64caf --- /dev/null +++ b/server/Dockerfile.template @@ -0,0 +1,42 @@ +FROM balenalib/%%BALENA_MACHINE_NAME%%-node:16.15-buster-run + +# Dockerfile boilerplate - certs and timezone +COPY ca-certificates.crt /etc/ssl/certs/ +RUN ln -sf /usr/share/zoneinfo/US/Eastern /etc/localtime + +ENV UDEV=on + +# RUN apt-get update +RUN apt-get update && apt-get install -y apt-transport-https +RUN install_packages vim postgresql-client inetutils-telnet exfat-fuse exfat-utils + +# install migrate tool +RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.2/migrate.linux-armv7.tar.gz | tar xvz +RUN chmod +x migrate + +# COPY usb drive handling rules and scripts over +COPY usb.rules /etc/udev/rules.d/usb.rules +COPY scripts /usr/src/scripts +RUN chmod +x /usr/src/scripts/*.sh + +COPY startservers.sh / +RUN chmod +x /startservers.sh +COPY migrations /go/migrations +COPY conf/postgresql.conf /go/conf/postgresql.conf +COPY conf/pg_hba.conf /go/conf/pg_hba.conf +COPY src /server/src +COPY config_ci.json /server +COPY config_pi.json /server +COPY config_test.json /server +COPY config_dev.json /server +COPY config_prod.json /server/config.json +COPY config.json /server +COPY package.json /server +COPY package-lock.json /server +WORKDIR /server +RUN npm install + +CMD cp /server/config.json /config +CMD cd / +CMD /startservers.sh +# CMD ["bash"] diff --git a/server/start_api.sh b/server/start_api.sh new file mode 100644 index 0000000..de01dc3 --- /dev/null +++ b/server/start_api.sh @@ -0,0 +1,171 @@ +#!/bin/bash + +# +# Copyright (c) John Rodley 2022. +# SPDX-FileCopyrightText: John Rodley 2022. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this +# software and associated documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation the rights to use, copy, +# modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +DB_INITIALIZED="NO" + +if [ -f "$POSTGRESQL_SHARED_DIRECTORY/initialized" ] +then + echo "$POSTGRESQL_SHARED_DIRECTORY/initialized already exists" + DB_INITIALIZED="YES" +else + echo "$POSTGRESQL_SHARED_DIRECTORY/initialized doesn't exist - database will be dropped/rebuilt" +fi + +echo "Copying fresh migrations into shared directory" +mkdir -p "$POSTGRESQL_SHARED_DIRECTORY/migrations" +cp /go/migrations/* "$POSTGRESQL_SHARED_DIRECTORY/migrations" + +if [ ! -d "$POSTGRESQL_SHARED_DIRECTORY/conf" ] +then + mkdir -p "$POSTGRESQL_SHARED_DIRECTORY/conf" +else + echo "conf directory already exists" +fi +cp /go/conf/postgresql.conf "$POSTGRESQL_SHARED_DIRECTORY/conf" +cp /go/conf/pg_hba.conf "$POSTGRESQL_SHARED_DIRECTORY/conf" + +export PGPASSWORD=$POSTGRESQL_POSTGRES_PASSWORD + +# terminate all connections +if [ "$DB_INITIALIZED" = "YES" ] +then + echo "Database already exists, don't drop/recreate" +else + echo "Database doesn't already exist, drop/recreate" + echo "terminate all connections doesn't work yet - skipping" + sudo PGPASSWORD="$POSTGRESQL_POSTGRES_PASSWORD" psql -h "$POSTGRESQL_HOST" -p 5432 -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRESQL_APPLICATION_DBNAME' AND pid <> pg_backend_pid();" "user='$POSTGRESQL_POSTGRES_USER' dbname=$POSTGRESQL_APPLICATION_DBNAME password='$POSTGRESQL_POSTGRES_PASSWORD'" +# create database + echo Drop database + sudo psql -h "$POSTGRESQL_HOST" -p 5432 -c "DROP DATABASE IF EXISTS $POSTGRESQL_SYSTEM_DBNAME" "user=$POSTGRESQL_POSTGRES_USER dbname='$POSTGRESQL_SYSTEM_DBNAME' password='$POSTGRESQL_POSTGRES_PASSWORD'" + echo Create database + sudo PGPASSWORD="$POSTGRESQL_POSTGRES_PASSWORD" psql -h "$POSTGRESQL_HOST" -p 5432 -c "CREATE DATABASE $POSTGRESQL_APPLICATION_DBNAME" "user=$POSTGRESQL_POSTGRES_USER dbname='$POSTGRESQL_SYSTEM_DBNAME' password='$POSTGRESQL_POSTGRES_PASSWORD'" +fi + +echo "Always run migrations from $POSTGRESQL_SHARED_DIRECTORY/migrations - this will create all the tables" +ls -l "$POSTGRESQL_SHARED_DIRECTORY/migrations/*_ca.up.sql" +/migrate -verbose -source "file://$POSTGRESQL_SHARED_DIRECTORY/migrations" -database "postgres://$POSTGRESQL_POSTGRES_USER:$POSTGRESQL_POSTGRES_PASSWORD@$POSTGRESQL_HOST:5432/$POSTGRESQL_APPLICATION_DBNAME?sslmode=disable" up + +# echo "Update ClamAV database" +# sudo freshclam + +if [ "$DB_INITIALIZED" = "YES" ] +then + echo "Database already exists" +else + echo "Database didn't already exist, set initialized flag" + echo Marking database as initialized + date > "$POSTGRESQL_SHARED_DIRECTORY/initialized" +fi + +now=$(date +"%Y.%m.%d_%H.%M.%S") + +if [ -b "/dev/sda1" ] +then + echo Mounting /dev/sda1 into /mnt/backups + sudo mkdir -p /mnt/backups + sudo mount -t exfat -o rw /dev/sda1 /mnt/backups + sudo mkdir -p /mnt/backups/database + sudo mkdir -p /mnt/backups/pictures + sudo mkdir -p /mnt/backups/logs/api/${now} + sudo mkdir -p /mnt/backups/logs/queue/${now} + sudo mkdir -p /mnt/backups/logs/ui/${now} + sudo mkdir -p /mnt/backups/logs/websocket/${now} + sudo mkdir -p $POSTGRESQL_SHARED_DIRECTORY/logs/queue/${now} + sudo mkdir -p $POSTGRESQL_SHARED_DIRECTORY/logs/ui/${now} + sudo mkdir -p $POSTGRESQL_SHARED_DIRECTORY/logs/websocket/${now} + + echo Moving logs to backup + sudo mv /server/src/bubblesnet.log /mnt/backups/logs/api/${now} + sudo mv $LOGS_SHARED_DIRECTORY/logs/queue /mnt/backups/logs/queue/${now} + sudo mv $LOGS_SHARED_DIRECTORY/logs/ui /mnt/backups/logs/ui/${now} + sudo mv $LOGS_SHARED_DIRECTORY/logs/websocket /mnt/backups/logs/websocket/${now} + + echo Moving pictures to backup drive + sudo mv /server/src/public/*.jpg /mnt/backups/pictures + + echo Backing database "$POSTGRESQL_APPLICATION_DBNAME" up to mounted backup drive + echo "****************************************" + echo "**** ********" + echo "**** *******" + echo "**** DATABASE BACKING UP NOW ****" + + echo *:*:*:*:$POSTGRESQL_POSTGRES_PASSWORD > ~/.pgpass + export PGPASSFILE=~/.pgpass + chmod 600 ~/.pgpass + mkdir /mnt/backups/database/${now} + sudo pg_dump --no-password -h "$POSTGRESQL_HOST" -p 5432 -U postgres "$POSTGRESQL_APPLICATION_DBNAME" > /mnt/backups/database/${now}/"$POSTGRESQL_APPLICATION_DBNAME".bak 2>> /mnt/backups/database/${now}/dbname.bak.log + echo "**** *******" + echo "****************************************" + cat /mnt/backups/database/${now}/dbname.bak.log + +else + echo "/dev/sda1 doesn't exist, not mounting" + echo "****************************************" + echo "****************************************" + echo "**** ********" + echo "**** DB BACKUPS NOT SAVED! *************" + echo "**** LOGS AND PICTURES NOT SAVED !! ****" + echo "**** *******" + echo "****************************************" + echo "****************************************" + echo "****************************************" + echo "****************************************" +fi + +if [ "$BALENA_DEVICE_TYPE" = "coral" ] +then + echo Turning coral dev board fan on + sudo echo "disabled" > /sys/devices/virtual/thermal/thermal_zone0/mode + sudo echo 8600 > /sys/devices/platform/gpio_fan/hwmon/hwmon0/fan1_target +fi + +echo Setting timezone +sudo ln -sf /usr/share/zoneinfo/US/Eastern /etc/localtime + +echo "Starting API" +cd /server || exit + +# Start the first process +node src/api-server.js & + +# Wait for any process to exit +wait -n + +exit_status=$? + +if [ -b "/dev/sda1" ] +then + echo Moving API logs to mounted storage + mv /server/src/*.log /mnt/backups/logs/api/${now} + echo Umounting mounted storage + sudo umount /dev/sda1 +fi + +echo Executing sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s +sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s + +echo Exiting with exit status $exit_status + +exit $exit_status diff --git a/server/start_queue.sh b/server/start_queue.sh new file mode 100644 index 0000000..dfc938f --- /dev/null +++ b/server/start_queue.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# +# Copyright (c) John Rodley 2022. +# SPDX-FileCopyrightText: John Rodley 2022. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this +# software and associated documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation the rights to use, copy, +# modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +cd /server + +echo Setting timezone +sudo ln -sf /usr/share/zoneinfo/US/Eastern /etc/localtime + +echo Backing up log files +now=$(date +"%Y.%m.%d_%H.%M.%S") +sudo mkdir -p $LOGS_SHARED_DIRECTORY/logs/queue/${now} +sudo mv /server/src/*.log $LOGS_SHARED_DIRECTORY/logs/queue/${now} + +# Start the second process +node src/queue-server.js & + +# Wait for any process to exit +wait -n + +exit_status=$? + +echo Executing sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s +sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s + +echo Exiting with exit status $exit_status + +# Exit with status of process that exited first +exit $exit_status \ No newline at end of file diff --git a/server/start_websocket.sh b/server/start_websocket.sh new file mode 100644 index 0000000..479e2c5 --- /dev/null +++ b/server/start_websocket.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# +# Copyright (c) John Rodley 2022. +# SPDX-FileCopyrightText: John Rodley 2022. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this +# software and associated documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation the rights to use, copy, +# modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +echo Setting timezone +sudo ln -sf /usr/share/zoneinfo/US/Eastern /etc/localtime + +echo Backing up log files +now=$(date +"%Y.%m.%d_%H.%M.%S") +sudo mkdir -p $LOGS_SHARED_DIRECTORY/logs/websocket/${now} +sudo mv /server/src/*.log $LOGS_SHARED_DIRECTORY/logs/websocket/${now} + +cd /server + +# Start the third process +node src/ws-server.js & + +# Wait for any process to exit +wait -n + +exit_status=$? + +echo Executing sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s +sleep "$SLEEP_ON_EXIT_FOR_DEBUGGING"s + +echo Exiting with exit status $exit_status + +# Exit with status of process that exited first +exit $exit_status \ No newline at end of file