diff --git a/.env.example.WIP b/.env.example.WIP deleted file mode 100644 index 461bdcde..00000000 --- a/.env.example.WIP +++ /dev/null @@ -1,51 +0,0 @@ -ENVIRONMENT=production -API_PORT=8030 -SECRET_KEY=test - -RUN_MIGRATION=True -RUN_COMPILE_MESSAGES=True -RUN_COLLECT_STATIC=True -RUN_LOAD_DUMMY_DATA=False -RUN_CREATE_SUPER_USER=False - -# django -DEBUG=True -ENABLE_DEBUG_TOOLBAR=False - -ALLOWED_HOSTS=localhost -CSRF_TRUSTED_ORIGINS=http://localhost -CORS_ALLOWED_ORIGINS=http://localhost -CORS_ALLOWED_ORIGIN_REGEXES=http://localhost* - -SITE_URL= -EMAIL_HOST= -EMAIL_PORT=25 -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= -EMAIL_USE_TLS=False -EMAIL_USE_SSL=False - -NO_REPLY_EMAIL= -DEFAULT_FROM_EMAIL= - -# admin -DJANGO_ADMIN_EMAIL=a@a.co -DJANGO_ADMIN_PASSWORD=a - -# database -POSTGRES_USER=postgres -POSTGRES_PASSWORD=secret -POSTGRES_DB=seismic_site -DATABASE_URL=postgres://postgres:secret@db/postgres - -BACKGROUND_WORKERS=1 - -# client -REACT_APP_DJANGO_SITE_URL=http://localhost -REACT_APP_DJANGO_PORT=8030 -REACT_APP_DJANGO_API_ENDPOINT=api/v1 - -# external api keys -REACT_APP_CAPTCHA_API_KEY= -HERE_MAPS_API_KEY= -REACT_APP_HERE_MAPS_API_KEY= diff --git a/.env.example.dev b/.env.example.dev index c98c5ccb..e24c1ef0 100644 --- a/.env.example.dev +++ b/.env.example.dev @@ -3,11 +3,11 @@ ENVIRONMENT=development API_PORT=8030 SECRET_KEY=test -RUN_MIGRATION=yes -RUN_COMPILEMESSAGES=yes -RUN_LOAD_DUMMY_DATA=no -RUN_COLLECT_STATIC=no -RUN_DEV_SERVER=yes +RUN_MIGRATION=True +RUN_COMPILE_MESSAGES=True +RUN_COLLECT_STATIC=False +RUN_LOAD_DUMMY_DATA=False +RUN_CREATE_SUPER_USER=True # django DEBUG=True @@ -25,10 +25,8 @@ NO_REPLY_EMAIL= DEFAULT_FROM_EMAIL= # admin -SUPER_ADMIN_EMAIL=a@a.co -SUPER_ADMIN_PASS=a -SUPER_ADMIN_FIRST_NAME=abc -SUPER_ADMIN_LAST_NAME=dee +DJANGO_ADMIN_EMAIL=a@a.co +DJANGO_ADMIN_PASSWORD=a # database POSTGRES_USER=postgres diff --git a/.env.example.prod b/.env.example.prod index e7f61727..818b6f11 100644 --- a/.env.example.prod +++ b/.env.example.prod @@ -1,52 +1,50 @@ # api deployment ENVIRONMENT=production API_PORT=8030 -SECRET_KEY= +SECRET_KEY=test -RUN_MIGRATION=yes -RUN_COMPILEMESSAGES=yes -RUN_LOAD_DUMMY_DATA=no -RUN_COLLECT_STATIC=yes -RUN_DEV_SERVER=no +RUN_MIGRATION=True +RUN_COMPILE_MESSAGES=True +RUN_COLLECT_STATIC=True +RUN_LOAD_DUMMY_DATA=False +RUN_CREATE_SUPER_USER=True # django DEBUG=False ENABLE_DEBUG_TOOLBAR=False -ALLOWED_HOSTS=localhost -CSRF_TRUSTED_ORIGINS=http://localhost -CORS_ALLOWED_ORIGINS=http://localhost -CORS_ALLOWED_ORIGIN_REGEXES=http://localhost* - SITE_URL= EMAIL_HOST= -EMAIL_PORT= +EMAIL_PORT=25 EMAIL_HOST_USER= EMAIL_HOST_PASSWORD= -EMAIL_USE_TLS= -EMAIL_USE_SSL= +EMAIL_USE_TLS=False +EMAIL_USE_SSL=False NO_REPLY_EMAIL= DEFAULT_FROM_EMAIL= +ALLOWED_HOSTS=localhost +CSRF_TRUSTED_ORIGINS=http://localhost +CORS_ALLOWED_ORIGINS=http://localhost +CORS_ALLOWED_ORIGIN_REGEXES=http://localhost* + # admin -SUPER_ADMIN_EMAIL= -SUPER_ADMIN_PASS= -SUPER_ADMIN_FIRST_NAME= -SUPER_ADMIN_LAST_NAME= +DJANGO_ADMIN_EMAIL=a@a.co +DJANGO_ADMIN_PASSWORD=a # database -POSTGRES_USER= -POSTGRES_PASSWORD= -POSTGRES_DB= -DATABASE_URL=postgres://user:password@netloc/database +POSTGRES_USER=postgres +POSTGRES_PASSWORD=secret +POSTGRES_DB=seismic_site +DATABASE_URL=postgres://postgres:secret@db/postgres -GUNICORN_PORT=5000 -GUNICORN_WORKERS=10 +BACKGROUND_WORKERS=2 +GUNICORN_WORKERS=2 # client -REACT_APP_DJANGO_SITE_URL= -REACT_APP_DJANGO_PORT= +REACT_APP_DJANGO_SITE_URL=http://localhost +REACT_APP_DJANGO_PORT=8030 REACT_APP_DJANGO_API_ENDPOINT=api/v1 # external api keys diff --git a/.gitignore b/.gitignore index d91f0e69..531482a0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ api/public/* # dotenv environment variables file .env +.env.dev .env.test .env.prod .env.* diff --git a/Makefile b/Makefile index d995e652..634da8b0 100644 --- a/Makefile +++ b/Makefile @@ -3,45 +3,36 @@ help: ## Display a help message detailing commands a @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @echo "" -## [DEV ENV SETUP] -install-docker-ubuntu: ## installs docker and docker-compose on Ubuntu - sudo apt-get remove docker docker-engine docker.io containerd runc - sudo apt-get update - sudo apt-get -y install apt-transport-https ca-certificates curl gnupg-agent software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo apt-key fingerprint 0EBFCD88 - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(shell lsb_release -cs) stable" || { echo "$(shell lsb_release -cs) is not yet supported by docker.com."; exit 1; } - sudo apt-get update - sudo apt-get install -y docker-ce gettext - sudo curl -L "https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-$(shell uname -s)-$(shell uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - -install-docker-osx: ## installs homebrew (you can skip this at runtime), docker and docker-compose on OSX - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - brew update - brew cask install docker - brew install docker-compose gettext - -build: ## builds the container - docker-compose build --pull - docker-compose up -d - build-dev: ## builds the container with the development flag - docker-compose build --build-arg ENVIRONMENT=development --pull - docker-compose up -d + docker compose build + docker compose up -d + +build-prod: ## builds the container with the production flag + docker compose -f docker-compose.prod.yml build \ + --build-arg $$(cat .env.prod | grep ENVIRONMENT) \ + --build-arg $$(cat .env.prod | grep REACT_APP_CAPTCHA_API_KEY) \ + --build-arg $$(cat .env.prod | grep REACT_APP_HERE_MAPS_API_KEY) \ + --build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_SITE_URL) \ + --build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_PORT) \ + --build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_API_ENDPOINT) + docker compose -f docker-compose.prod.yml up -d superuser: ## creates a superuser for the API - docker-compose exec api ./manage.py createsuperuser + docker compose exec api ./manage.py createsuperuser init-db: superuser ## sets up the database and fixtures - docker-compose exec api ./manage.py loaddata statistics - docker-compose exec api ./manage.py loaddata proximal_utilities - docker-compose exec api ./manage.py loaddata work_performed - docker-compose exec api ./manage.py loaddata buildings + docker compose exec api ./manage.py loaddata statistics + docker compose exec api ./manage.py loaddata proximal_utilities + docker compose exec api ./manage.py loaddata work_performed + docker compose exec api ./manage.py loaddata buildings + +drop-db-dev: ## drops the containers and removes the database for the development environment + docker compose down -v -t 60 + +drop-db-prod: ## drops the containers and removes the database for the production environment + docker compose -f docker-compose.prod.yml down -v -t 60 -drop-db: ## drops the database - docker-compose down -t 60 - docker volume rm seismic-risc_pgdata +drop-db: drop-db-dev drop-db-prod ## drops the containers and removes the database for both environments redo-db: drop-db init-db ## drops the database, then sets up the database and fixtures @@ -53,43 +44,43 @@ requirements-update: ## run pip compile and rebuild the requirement docker compose run --rm --no-deps --entrypoint "bash -c" api "cd /code && pip-compile --resolver=backtracking -r -U -o requirements.txt requirements.in && pip-compile --resolver=backtracking -r -U -o requirements-dev.txt requirements-dev.in && chmod a+r requirements.txt && chmod a+r requirements-dev.txt" migrations: ## generate migrations in a clean container - docker-compose exec api ./manage.py makemigrations + docker compose exec api ./manage.py makemigrations migrate: ## apply migrations in a clean container - docker-compose exec api ./manage.py migrate + docker compose exec api ./manage.py migrate makemessages: ## generate the strings marked for translation - docker-compose exec api ./manage.py makemessages -a + docker compose exec api ./manage.py makemessages -a compilemessages: ## compile the translations - docker-compose exec api ./manage.py compilemessages + docker compose exec api ./manage.py compilemessages messages: makemessages compilemessages collectstatic: - docker-compose exec api ./manage.py collectstatic --no-input + docker compose exec api ./manage.py collectstatic --no-input pyshell: ## start a django shell - docker-compose exec api ./manage.py shell + docker compose exec api ./manage.py shell black: ## run the Black formatter on the Python code black --line-length 120 --target-version py311 --exclude migrations ./api ## [TEST] test: ## run all tests - docker-compose run --rm api "pytest" + docker compose run --rm api "pytest" test-pdb: ## run tests and enter debugger on failed assert or error - docker-compose run --rm api "pytest --pdb" + docker compose run --rm api "pytest --pdb" test-lf: ## rerun tests that failed last time - docker-compose run --rm api "pytest --lf" + docker compose run --rm api "pytest --lf" ## [CLEAN] clean: clean-docker clean-py ## remove all build, test, coverage and Python artifacts clean-docker: ## stop docker containers and remove orphaned images and volumes - docker-compose down -t 60 + docker compose down -t 60 docker system prune -f clean-py: ## remove Python test, coverage, file artifacts, and compiled message files diff --git a/README.md b/README.md index 090bb411..9385352f 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ Make sure to check the [Environment variables](#environment-variables) section for info on how to set up the keys before you run the following commands: ```shell -cp .env.example.dev .env +cp .env.example.dev .env.dev # build the development container make build-dev ``` @@ -193,7 +193,7 @@ make build-dev If you didn't set up the `RUN_LOAD_DUMMY_DATA` variable, you can add dummy data to the database with the following command: ```shell -make init-db +make build-dev ``` If the `RUN_LOAD_DUMMY_DATA` was `yes`, then you should have dummy data but will have to create a superuser: @@ -252,7 +252,7 @@ Runs the application in the development mode. #### External services API keys -In order to have a fully functional project, you have to get two API keys: HERE Maps API Key and hCAPTCHA API Key. +To have a fully functional project, you have to get two API keys: HERE Maps API Key and hCAPTCHA API Key. ##### HERE Maps API Key @@ -309,13 +309,13 @@ docker-compose exec api some_container_command docker-compose exec client some_container_command ``` -In order to see all available commands, run: +To see all available commands, run: ```shell make help ``` -### Starting the project without docker +### Starting the project without Docker #### Windows platform @@ -340,7 +340,7 @@ make help provide or change. Double check database config line in .env. It has to follow this pattern: `postgres://USER:PASSWORD@HOST:PORT/NAME` -3. Run following in order to set the needed environment variables: +3. Run following to set the needed environment variables: ```shell activate_dev_env.bat @@ -358,7 +358,7 @@ make help python api/manage.py migrate --no-input ``` -6. Create admin user (user to login into admin pannel): +6. Create admin user (user to login into admin panel): ```shell python api/manage.py createsuperuser @@ -407,7 +407,7 @@ Check functionality at http://localhost:3000. ### Development -When creating new models in Django, in order to make sure they are generated in a clean environment, it is recommended +When creating new models in Django, to make sure they are generated in a clean environment, it is recommended to generate the migration files using the `make` command: ```shell @@ -497,7 +497,7 @@ make test ## Production -In order to get the container ready for production use, we need to first build it: +To get the container ready for production use, we need to first build it: ```shell docker build -t seismic-risc:latest ./api diff --git a/api/Dockerfile b/api/Dockerfile index 6bc6e14b..47f1ba2b 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -2,10 +2,15 @@ FROM python:3.11.6-slim-bookworm ENV PYTHONUNBUFFERED=1 -ENV RUN_MIGRATION=yes -ENV RUN_COMPILE_MESSAGES=no -ENV RUN_COLLECT_STATIC=no -ENV RUN_CREATE_SUPER_USER=yes +ENV ENVIRONMENT=production +ENV DEBUG ${DEBUG:-False} + +ENV RUN_MIGRATION=True +ENV RUN_COMPILE_MESSAGES=True +ENV RUN_COLLECT_STATIC=True +ENV RUN_CREATE_SUPER_USER=True + +ENV ENABLE_DEBUG_TOOLBAR=False ENV IS_CONTAINERIZED=True @@ -27,8 +32,6 @@ RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \ tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz -ENTRYPOINT ["/init"] - COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default COPY docker/s6-rc.d /etc/s6-overlay/s6-rc.d @@ -40,7 +43,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" # install the backend libraries: -COPY ./api/setup.cfg ./api/requirements.txt ./api/requirements-dev.txt /var/www/seismic/api/ +COPY ./api/setup.cfg ./api/requirements.txt /var/www/seismic/api/ WORKDIR /var/www/seismic/api/ RUN python3 -m pip install --upgrade pip setuptools && \ python3 -m pip install -r ./requirements.txt @@ -56,4 +59,6 @@ ENV PATH=/root/.local/bin:$PATH WORKDIR /var/www/seismic/ +ENTRYPOINT ["/init"] + EXPOSE 80 diff --git a/api/Dockerfile.dev b/api/Dockerfile.dev new file mode 100644 index 00000000..cab4f663 --- /dev/null +++ b/api/Dockerfile.dev @@ -0,0 +1,64 @@ +FROM python:3.11.6-slim-bookworm + +ENV PYTHONUNBUFFERED=1 + +ENV ENVIRONMENT ${ENVIRONMENT:-development} +ENV DEBUG ${DEBUG:-True} + +ENV RUN_MIGRATION=True +ENV RUN_COMPILE_MESSAGES=False +ENV RUN_COLLECT_STATIC=False +ENV RUN_CREATE_SUPER_USER=True + +ENV ENABLE_DEBUG_TOOLBAR=True + +ENV IS_CONTAINERIZED=True + +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + nginx gcc xz-utils gettext build-essential postgresql-client libpq-dev + + +ARG S6_OVERLAY_VERSION=3.1.2.1 +ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME 0 + +ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp +ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp +RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \ + tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz + + +COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default +COPY docker/s6-rc.d /etc/s6-overlay/s6-rc.d + + +# Python virtualenv paths +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + + +# install the backend libraries: +COPY ./api/setup.cfg ./api/requirements-dev.txt /var/www/seismic/api/ +WORKDIR /var/www/seismic/api/ +RUN python3 -m pip install --upgrade pip setuptools && \ + python3 -m pip install -r ./requirements-dev.txt + + +# copy the backend source code: +COPY ./api/ /var/www/seismic/api/ + + +# Make sure scripts in .local are usable: +ENV PATH=/root/.local/bin:$PATH + + +WORKDIR /var/www/seismic/ + +ENTRYPOINT ["/init"] + +EXPOSE 80 diff --git a/api/buildings/management/commands/seed_superuser.py b/api/buildings/management/commands/seed_superuser.py deleted file mode 100644 index b501e9b8..00000000 --- a/api/buildings/management/commands/seed_superuser.py +++ /dev/null @@ -1,35 +0,0 @@ -from django.conf import settings -from django.contrib.auth.models import User -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - def handle(self, *args, **kwargs): - status = self.create_superuser() - - if status == 0: - self.stdout.write(self.style.SUCCESS("Super admin has been created")) - else: - self.stdout.write(self.style.SUCCESS("Super admin already exists")) - - @staticmethod - def create_superuser(): - admin_user = User.objects.filter(is_superuser=True) - - if admin_user: - return -1 - - superuser = User() - - superuser.is_active = True - superuser.is_superuser = True - superuser.is_staff = True - - superuser.username = superuser.email = settings.SUPER_ADMIN_EMAIL - superuser.first_name = settings.SUPER_ADMIN_FIRST_NAME - superuser.last_name = settings.SUPER_ADMIN_LAST_NAME - superuser.set_password(settings.SUPER_ADMIN_PASS) - - superuser.save() - - return 0 diff --git a/api/docker-entrypoint b/api/docker-entrypoint deleted file mode 100755 index 9ce8e05e..00000000 --- a/api/docker-entrypoint +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -API_PORT=${API_PORT:-"8030"} - -RUN_MIGRATION=${RUN_MIGRATION:-"yes"} -RUN_COMPILEMESSAGES=${RUN_COMPILEMESSAGES:-"yes"} -RUN_COLLECT_STATIC=${RUN_COLLECT_STATIC:-"yes"} -RUN_LOAD_DUMMY_DATA=${RUN_LOAD_DUMMY_DATA:-"no"} -RUN_DEV_SERVER=${RUN_DEV_SERVER:-"no"} - -echo "Run basic health checks" -./manage.py check - -if [[ ${RUN_MIGRATION} = "yes" ]] ; then - echo "Run database migrations" - ./manage.py migrate - ./manage.py createcachetable -fi - -./manage.py seed_superuser - -if [[ ${RUN_COMPILEMESSAGES} = "yes" ]] ; then - echo "Compile i18n messages" - ./manage.py compilemessages -fi - -if [[ ${RUN_LOAD_DUMMY_DATA} = "yes" ]] ; then - echo "Load dummy data in the database" - ./manage.py loaddata proximal_utilities - ./manage.py loaddata work_performed -fi - -if [[ ${RUN_COLLECT_STATIC} = "yes" ]] ; then - echo "Collect static files (this will take a while)" - ./manage.py collectstatic --no-input -fi - -if [[ ${RUN_DEV_SERVER} = "yes" ]] ; then - echo "Start web server on ${API_PORT}" - ./manage.py runserver "0.0.0.0:${API_PORT}" -else - gunicorn seismic_site.wsgi --bind "0.0.0.0:${API_PORT}" --log-level "${LOG_LEVEL:-info}" -k gevent -w "${GUNICORN_WORKERS}" --timeout 300 -fi diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index 3dca5f3a..ecab8c7c 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -4,6 +4,8 @@ # # pip-compile --output-file=requirements-dev.txt requirements-dev.in # +appnope==0.1.3 + # via ipython asgiref==3.7.2 # via # -r requirements.txt @@ -19,6 +21,8 @@ backcall==0.2.0 # via ipython black==23.9.1 # via -r requirements-dev.in +blessed==1.20.0 + # via -r requirements.txt boto3==1.28.61 # via -r requirements.txt botocore==1.31.61 @@ -34,6 +38,8 @@ click==8.1.7 # pip-tools coverage[toml]==7.3.2 # via pytest-cov +croniter==2.0.1 + # via -r requirements.txt decorator==5.1.1 # via ipython defusedxml==0.7.1 @@ -56,6 +62,8 @@ django==4.2.6 # django-import-export # django-jazzmin # django-jquery + # django-picklefield + # django-q2 # django-storages # djangorestframework # drf-spectacular @@ -73,6 +81,12 @@ django-jazzmin==2.6.0 # via -r requirements.txt django-jquery==3.1.0 # via -r requirements.txt +django-picklefield==3.1 + # via + # -r requirements.txt + # django-q2 +django-q2==1.6.1 + # via -r requirements.txt django-storages==1.14.2 # via -r requirements.txt djangorestframework==3.14.0 @@ -167,6 +181,8 @@ pluggy==1.3.0 # via pytest prompt-toolkit==3.0.39 # via ipython +psutil==5.9.6 + # via -r requirements.txt psycopg2==2.9.9 # via -r requirements.txt ptyprocess==0.7.0 @@ -197,10 +213,12 @@ python-dateutil==2.8.2 # via # -r requirements.txt # botocore + # croniter # faker pytz==2023.3.post1 # via # -r requirements.txt + # croniter # djangorestframework pyyaml==6.0.1 # via @@ -225,6 +243,7 @@ six==1.16.0 # via # -r requirements.txt # asttokens + # blessed # python-dateutil sqlparse==0.4.4 # via @@ -256,7 +275,10 @@ urllib3==1.26.17 # -r requirements.txt # botocore wcwidth==0.2.8 - # via prompt-toolkit + # via + # -r requirements.txt + # blessed + # prompt-toolkit werkzeug==3.0.0 # via -r requirements-dev.in wheel==0.41.2 diff --git a/api/requirements.in b/api/requirements.in index 7f163286..90289ecd 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -10,10 +10,10 @@ django-import-export==3.3.1 django-jazzmin==2.6.0 # job scheduler -django-q2~=1.5.5 +django-q2~=1.6.1 blessed~=1.20.0 # optional requirement for django-q2 -psutil~=5.9.5 # optional requirement for django-q2 -croniter~=1.4.1 # optional requirement for django-q2 +psutil~=5.9.6 # optional requirement for django-q2 +croniter~=2.0.1 # optional requirement for django-q2 # REST framework djangorestframework==3.14.0 diff --git a/api/requirements.txt b/api/requirements.txt index e405c093..59c66efc 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -18,7 +18,7 @@ botocore==1.31.61 # via # boto3 # s3transfer -croniter==1.4.1 +croniter==2.0.1 # via -r requirements.in defusedxml==0.7.1 # via odfpy @@ -51,7 +51,7 @@ django-jquery==3.1.0 # via -r requirements.in django-picklefield==3.1 # via django-q2 -django-q2==1.5.5 +django-q2==1.6.1 # via -r requirements.in django-storages==1.14.2 # via -r requirements.in @@ -89,7 +89,7 @@ packaging==23.2 # via gunicorn pillow==10.0.1 # via -r requirements.in -psutil==5.9.5 +psutil==5.9.6 # via -r requirements.in psycopg2==2.9.9 # via -r requirements.in @@ -98,7 +98,9 @@ python-dateutil==2.8.2 # botocore # croniter pytz==2023.3.post1 - # via djangorestframework + # via + # croniter + # djangorestframework pyyaml==6.0.1 # via # drf-spectacular diff --git a/api/seismic_site/settings/base.py b/api/seismic_site/settings/base.py index ad7c1541..1a0bf8b7 100644 --- a/api/seismic_site/settings/base.py +++ b/api/seismic_site/settings/base.py @@ -17,8 +17,8 @@ env = environ.Env( # set casting, default value DEBUG=(bool, False), - ENVIRONMENT=(str, "dev"), - ENABLE_DEBUG_TOOLBAR=(bool, True), + ENVIRONMENT=(str, "production"), + ENABLE_DEBUG_TOOLBAR=(bool, False), LANGUAGE_CODE=(str, "en"), NO_REPLY_EMAIL=(str, "noreply@code4.ro"), DEFAULT_FROM_EMAIL=(str, "noreply@code4.ro"), @@ -35,11 +35,6 @@ AWS_STORAGE_BUCKET_NAME=(str, ""), AWS_SUBDOMAIN=(str, "s3.amazonaws.com"), AWS_S3_REGION_NAME=(str, ""), - # django admin - SUPER_ADMIN_PASS=(str, ""), - SUPER_ADMIN_EMAIL=(str, ""), - SUPER_ADMIN_FIRST_NAME=(str, ""), - SUPER_ADMIN_LAST_NAME=(str, ""), BACKGROUND_WORKERS=(int, 1), ) @@ -72,6 +67,7 @@ "corsheaders", "django_q", # project apps + "utils", "buildings", "static_custom", # api documentation @@ -93,7 +89,7 @@ SITE_ID = 1 -ENABLE_DEBUG_TOOLBAR = env("ENABLE_DEBUG_TOOLBAR") +ENABLE_DEBUG_TOOLBAR = env.bool("ENABLE_DEBUG_TOOLBAR") ROOT_URLCONF = "seismic_site.urls" @@ -478,4 +474,4 @@ "poll": 2, "guard_cycle": 3, "catch_up": False, -} \ No newline at end of file +} diff --git a/api/seismic_site/settings/development.py b/api/seismic_site/settings/development.py index 0af947f6..fca28a5d 100644 --- a/api/seismic_site/settings/development.py +++ b/api/seismic_site/settings/development.py @@ -5,11 +5,6 @@ CORS_ORIGIN_ALLOW_ALL = True SECRET_KEY = "secret" -SUPER_ADMIN_PASS = env("SUPER_ADMIN_PASS", default="pass") -SUPER_ADMIN_EMAIL = env("SUPER_ADMIN_EMAIL", default="a@a.co") -SUPER_ADMIN_FIRST_NAME = env("SUPER_ADMIN_FIRST_NAME", default="First") -SUPER_ADMIN_LAST_NAME = env("SUPER_ADMIN_LAST_NAME", default="Last") - EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS diff --git a/api/seismic_site/settings/production.py b/api/seismic_site/settings/production.py index 1f0a4179..ab75b002 100644 --- a/api/seismic_site/settings/production.py +++ b/api/seismic_site/settings/production.py @@ -4,11 +4,6 @@ SECRET_KEY = env.str("SECRET_KEY") # noqa -SUPER_ADMIN_PASS = env("SUPER_ADMIN_PASS") -SUPER_ADMIN_EMAIL = env("SUPER_ADMIN_EMAIL") -SUPER_ADMIN_FIRST_NAME = env("SUPER_ADMIN_FIRST_NAME") -SUPER_ADMIN_LAST_NAME = env("SUPER_ADMIN_LAST_NAME") - EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" EMAIL_USE_TLS = True EMAIL_CONFIG = env.email_url("EMAIL_URL", default="smtp://user:password@localhost:25") diff --git a/api/seismic_site/settings/test.py b/api/seismic_site/settings/test.py index 031ac620..38a42a57 100644 --- a/api/seismic_site/settings/test.py +++ b/api/seismic_site/settings/test.py @@ -4,11 +4,6 @@ SECRET_KEY = "test_secret" SITE_URL = "http://localhost" -SUPER_ADMIN_PASS = env("SUPER_ADMIN_PASS", default="pass") -SUPER_ADMIN_EMAIL = env("SUPER_ADMIN_EMAIL", default="a@a.co") -SUPER_ADMIN_FIRST_NAME = env("SUPER_ADMIN_FIRST_NAME", default="First") -SUPER_ADMIN_LAST_NAME = env("SUPER_ADMIN_LAST_NAME", default="Last") - EMAIL_BACKEND = "django.core.mail.backends.dummy.EmailBackend" TEST_RUNNER = "tests.runner.PytestTestRunner" diff --git a/api/utils/__init__.py b/api/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/utils/apps.py b/api/utils/apps.py new file mode 100644 index 00000000..4faf4f6f --- /dev/null +++ b/api/utils/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class UtilsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "utils" diff --git a/api/utils/management/commands/seed_superuser.py b/api/utils/management/commands/seed_superuser.py new file mode 100644 index 00000000..05437c07 --- /dev/null +++ b/api/utils/management/commands/seed_superuser.py @@ -0,0 +1,67 @@ +import logging + +import environ +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Command to create a superuser" + + def add_arguments(self, parser): + parser.add_argument( + "--email", + type=str, + help="Email of the superuser", + required=True, + ) + parser.add_argument( + "--username", + type=str, + help="Username of the superuser (default: email)", + required=False, + ) + parser.add_argument( + "--first_name", + type=str, + help="First name of the superuser", + required=False, + ) + parser.add_argument( + "--last_name", + type=str, + help="Last name of the superuser", + required=False, + ) + + def handle(self, *args, **kwargs): + user_model = get_user_model() + env = environ.Env() + + admin_email = kwargs.get("email") + admin_username = kwargs.get("username") or admin_email + admin_first_name = kwargs.get("first_name") or "" + admin_last_name = kwargs.get("last_name") or "" + + if user_model.objects.filter(email=admin_email).exists(): + logger.info("Super admin already exists") + return 0 + + superuser = user_model( + email=admin_email, + username=admin_username, + first_name=admin_first_name, + last_name=admin_last_name, + is_active=True, + is_superuser=True, + is_staff=True, + ) + superuser.set_password(env.str("DJANGO_ADMIN_PASSWORD")) + + superuser.save() + + logger.info("Super admin created successfully") + + return 0 diff --git a/client/Dockerfile b/client/Dockerfile index e221634c..72f4ebf5 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,25 +1,35 @@ -# base image -FROM node:18.18-bookworm-slim +FROM node:18.18-bookworm-slim as frontend -ARG ENVIRONMENT +ARG REACT_APP_DJANGO_SITE_URL +ARG REACT_APP_DJANGO_PORT +ARG REACT_APP_DJANGO_API_ENDPOINT -ENV ENVIRONMENT ${ENVIRONMENT:-production} -ENV NODE_ENV ${ENVIRONMENT} +ENV ENVIRONMENT="development" +ENV NODE_ENV="development" -# set working directory -WORKDIR /code +ENV REACT_APP_DJANGO_SITE_URL ${REACT_APP_DJANGO_SITE_URL} +ENV REACT_APP_DJANGO_PORT ${REACT_APP_DJANGO_PORT} +ENV REACT_APP_DJANGO_API_ENDPOINT ${REACT_APP_DJANGO_API_ENDPOINT} -# add `/code/node_modules/.bin` to $PATH -ENV PATH /code/node_modules/.bin:${PATH} +WORKDIR /var/www/seismic/frontend -# install and cache app dependencies -COPY ./package*.json ./ -RUN if [ "${ENVIRONMENT}" = "production" ]; then npm install --production; else npm install; fi +COPY ./ . -COPY ./docker-entrypoint / -COPY ./ /code/ +# install the frontend libraries: +RUN npm ci && \ + npm run build -ENTRYPOINT ["/docker-entrypoint"] + +FROM nginx:1.25.2-bookworm + +ENV ENVIRONMENT="production" +ENV NODE_ENV="production" + +COPY --from=frontend /var/www/seismic/frontend/build/ /usr/share/nginx/html/ + +WORKDIR /usr/share/nginx/html/ + +EXPOSE 80 # start app -CMD ["npm", "start"] +CMD [ "nginx", "-g", "daemon off;" ] diff --git a/client/Dockerfile.dev b/client/Dockerfile.dev new file mode 100644 index 00000000..f7b927e4 --- /dev/null +++ b/client/Dockerfile.dev @@ -0,0 +1,25 @@ +# base image +FROM node:18.18-bookworm-slim + +ARG ENVIRONMENT + +ENV ENVIRONMENT ${ENVIRONMENT} +ENV NODE_ENV ${ENVIRONMENT} + +# set working directory +WORKDIR /code + +# add `/code/node_modules/.bin` to $PATH +ENV PATH /code/node_modules/.bin:${PATH} + +# install and cache app dependencies +COPY ./package*.json ./ +RUN npm i + +COPY ./docker-entrypoint / +COPY ./ /code/ + +EXPOSE 3000 + +# start app +CMD ["npm", "start"] diff --git a/client/docker-entrypoint b/client/docker-entrypoint old mode 100755 new mode 100644 diff --git a/db/Dockerfile b/db/Dockerfile deleted file mode 100644 index b3dcf9a1..00000000 --- a/db/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM postgres:15.2-alpine3.17 - -RUN apt update -y && apt install -y --no-install-recommends postgresql-contrib -COPY ./docker-entrypoint-initdb.d / diff --git a/db/docker-entrypoint-initdb.d/001-init.sql b/db/docker-entrypoint-initdb.d/001-init.sql deleted file mode 100644 index 588aec00..00000000 --- a/db/docker-entrypoint-initdb.d/001-init.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_trgm; diff --git a/dk-cp.base.yml b/dk-cp.base.yml new file mode 100644 index 00000000..a5b42be7 --- /dev/null +++ b/dk-cp.base.yml @@ -0,0 +1,23 @@ +version: "3.8" + +services: + db_base: + image: postgres:15.4-bookworm + env_file: + - .env + volumes: + - pgdata:/var/lib/postgresql/data + + api_base: + env_file: + - .env + depends_on: + - db + ports: + - "${API_PORT:-8030}:8000" + + client_base: + env_file: + - .env + depends_on: + - api diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..1d03630c --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,46 @@ +version: "3.8" +name: seismic-prod + +services: + + db: + container_name: seismic_db-prod + extends: + file: dk-cp.base.yml + service: db_base + + api: + container_name: seismic_api-prod + extends: + file: dk-cp.base.yml + service: api_base + build: + context: . + dockerfile: ./api/Dockerfile + args: + - ENVIRONMENT=production + env_file: + - .env.prod + environment: + ENVIRONMENT: "production" + + client: + container_name: seismic_client-prod + extends: + file: dk-cp.base.yml + service: client_base + build: + context: ./client + dockerfile: ./Dockerfile + args: + - ENVIRONMENT=production + env_file: + - .env.prod + environment: + ENVIRONMENT: "production" + ports: + - "${CLIENT_PORT:-3030}:80" + +volumes: + pgdata: + name: seismic_pgdata-prod diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..63c4d408 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,41 @@ +version: "3.8" +name: seismic-test + +services: + + db: + container_name: seismic_db-test + extends: + file: docker-compose.dev.yml + service: db_base + + api: + container_name: seismic_api-test + extends: + file: docker-compose.dev.yml + service: api_base + build: + context: . + dockerfile: ./api/Dockerfile.dev + args: + - ENVIRONMENT=test + env_file: + - .env + - .env.test + environment: + ENVIRONMENT: "test" + + client: + container_name: seismic_client-test + extends: + file: docker-compose.dev.yml + service: client_base + env_file: + - .env + - .env.test + environment: + ENVIRONMENT: "test" + +volumes: + pgdata: + name: seismic_pgdata-test diff --git a/docker-compose.yml b/docker-compose.yml index a148ef67..3fc46141 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,54 @@ version: "3.8" +name: seismic-prod services: + db: - container_name: seismic-db - image: postgres:15.4-bookworm - env_file: - - .env + container_name: seismic_db-dev + extends: + file: dk-cp.base.yml + service: db_base ports: - - "5444:5432" - volumes: - - pgdata:/var/lib/postgresql/data + - "${DB_PORT:-5444}:5432" api: - container_name: seismic-api + container_name: seismic_api-dev + extends: + file: dk-cp.base.yml + service: api_base build: context: . - dockerfile: ./api/Dockerfile + dockerfile: ./api/Dockerfile.dev + args: + - ENVIRONMENT=development env_file: - .env + - .env.dev + environment: + ENVIRONMENT: "development" volumes: - - ./api:/code - ports: - - "8030:80" - depends_on: - - db + - ./api:/var/www/seismic/api/ client: - container_name: seismic-client + container_name: seismic_client-dev + extends: + file: dk-cp.base.yml + service: client_base build: context: ./client + dockerfile: ./Dockerfile.dev + args: + - ENVIRONMENT=development env_file: - .env + - .env.dev + environment: + ENVIRONMENT: "development" volumes: - ./client:/code ports: - - "3030:3000" - depends_on: - - api + - "${CLIENT_PORT:-3030}:3000" volumes: - pgdata: - name: seismic-risc_pgdata + pgdata: + name: seismic_pgdata-dev diff --git a/docker/s6-rc.d/api/run b/docker/s6-rc.d/api/run index 8a3e3f2a..ce34adf5 100644 --- a/docker/s6-rc.d/api/run +++ b/docker/s6-rc.d/api/run @@ -1,12 +1,18 @@ #!/command/with-contenv sh -cd /var/www/seismic/api/ +cd /var/www/seismic/api/ || exit 1 -# https://docs.gunicorn.org/en/latest/design.html#how-many-workers -WORKERS=$(((2 * $(nproc)) + 1)) +# if ENVIRONMENT="production", then run with gunicorn +if [ "${ENVIRONMENT}" = "production" ]; then + # https://docs.gunicorn.org/en/latest/design.html#how-many-workers + WORKERS=${GUNICORN_WORKESRS:-$(((2 * $(nproc)) + 1))} -python3 -m gunicorn seismic_site.wsgi \ + python3 -m gunicorn seismic_site.wsgi \ --bind "unix:///run/gunicorn.sock" \ --log-level info \ --worker-class gevent \ - --workers $WORKERS \ + --workers "${WORKERS}" \ + --bind 0.0.0.0:8000 \ --timeout 60 +else + python3 manage.py runserver_plus 0.0.0.0:8000 +fi diff --git a/docker/s6-rc.d/init/init.sh b/docker/s6-rc.d/init/init.sh index cef74313..67e10624 100755 --- a/docker/s6-rc.d/init/init.sh +++ b/docker/s6-rc.d/init/init.sh @@ -1,26 +1,23 @@ #!/command/with-contenv bash - # Convert one parameter to uppercase to_uppercase() { - echo $(echo $1 | tr '[:lower:]' '[:upper:]') + echo "${1}" | tr '[:lower:]' '[:upper:]' } # Check if the parameter is string True/False and return it as success/failure is_enabled() { - if test "$(to_uppercase $1)" = "TRUE"; then + _UPPER_VALUE=$(to_uppercase "$1") + if [ "${_UPPER_VALUE}" = "TRUE" ]; then return 0 + elif [ "${_UPPER_VALUE}" = "FALSE" ]; then + return 1 else - if test "$(to_uppercase $1)" = "FALSE"; then - return 1 - else - echo "WARNING: init.sh interpreting \"$1\" as False" >/dev/stderr - return 1 - fi + echo "WARNING: init.sh interpreting \"$1\" as False" >/dev/stderr + return 1 fi } - cd "${BACKEND_ROOT:-/var/www/seismic/api}" || exit 1 echo "Running Django self-checks" @@ -34,38 +31,32 @@ if is_enabled "${RUN_MIGRATION:-False}"; then fi # Compile the translation messages -if is_enabled ${RUN_COMPILE_MESSAGES:-False}; then +if is_enabled "${RUN_COMPILE_MESSAGES:-False}"; then echo "Compiling translation messages" python3 manage.py compilemessages fi # Collect the static files -if is_enabled ${RUN_COLLECT_STATIC:-False}; then +if is_enabled "${RUN_COLLECT_STATIC:-False}"; then echo "Collecting static files" mkdir static python3 manage.py collectstatic --noinput fi # Create the Django Admin super user -if is_enabled ${RUN_CREATE_SUPER_USER:-False}; then - echo "Checking superuser presence" - SUPERUSERS=$(python3 manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); print(User.objects.filter(username=\"${DJANGO_ADMIN_EMAIL}\").count())") +if is_enabled "${RUN_CREATE_SUPER_USER:-False}"; then + echo "Running the superuser seed script" - if test "${SUPERUSERS}" = "0"; then - echo "Creating first superuser" - python3 manage.py createsuperuser --noinput --username "${DJANGO_ADMIN_EMAIL}" - - echo "Setting superuser password" - python3 manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); u = User.objects.get(username=\"${DJANGO_ADMIN_EMAIL}\"); u.set_password(\"${DJANGO_ADMIN_PASSWORD}\"); u.save()" - else - echo "A superuser already exists; nothing created" - fi + python3 manage.py seed_superuser \ + --email "${DJANGO_ADMIN_EMAIL}" \ + --first_name "${DJANGO_ADMIN_FIRST_NAME}" \ + --last_name "${DJANGO_ADMIN_LAST_NAME}" fi # Load the dummy data -if is_enabled ${RUN_LOAD_DUMMY_DATA:-False}; then +if is_enabled "${RUN_LOAD_DUMMY_DATA:-False}"; then echo "Loading dummy data into the database" + ./manage.py loaddata proximal_utilities ./manage.py loaddata work_performed fi - diff --git a/docker/s6-rc.d/qcluster/run b/docker/s6-rc.d/qcluster/run index 74a86668..11fd6116 100644 --- a/docker/s6-rc.d/qcluster/run +++ b/docker/s6-rc.d/qcluster/run @@ -1,4 +1,4 @@ #!/command/with-contenv sh -cd /var/www/seismic/api/ +cd /var/www/seismic/api/ || exit 1 python3 manage.py qcluster