config.yml

# Python CircleCI 2.1 configuration file
# Circle docs https://circleci.com/docs/2.0/language-python/
# Set deployment variables in CircleCI as environment variables

version: 2.1

# Snippet for installing CloudFoundry CLI version 8
install-cf8: &install-cf8
  run:
    name: Install CF8
    command: |
      curl -A "" -L -o cf8.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
      sudo dpkg -i cf8.deb
      rm cf8.deb
      cf8 api https://api.fr.cloud.gov

install-python-dev-dependencies: &install-python-dev-dependencies
  run:
    name: Install python dev dependencies
    command: |
      pip install -U pip
      pip install pipenv
      pipenv install --dev
      pipenv run python -m playwright install

wait-for-tasks: &wait-for-tasks
  # Each task starts a container, which counts towards our RAM limit
  # This makes sure we wait for them to complete before running new ones.
  # At the time of writing, our tasks don't take more than 60 seconds to complete on production.
  run:
    name: Wait for maintenance tasks to complete
    command: |
      sleep 120  # (Two minutes)

jobs:
  build_and_test: # runs not using Workflows must have a `build` job as entry point
    # directory where steps are run
    working_directory: ~/code
    docker:
      # CircleCI Python images available at: https://hub.docker.com/r/cimg/python/
      - image: cimg/python:3.12.5-browsers
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
          DATABASE_URL: postgresql://root@localhost/circle_test?sslmode=disable
          POSTGRES_ANALYTICS_USER: analytics
          POSTGRES_ANALYTICS_PASSWORD: analytics_password
      # CircleCI PostgreSQL images available at: https://hub.docker.com/_/postgres
      - image: postgres:12.15-alpine
        environment:
          POSTGRES_USER: root
          POSTGRES_DB: circle_test
          POSTGRES_PASSWORD: password
    steps:
      - checkout
      # Removing chache for now, we either need to build our image or move the cache location
      # Read about caching dependencies: https://circleci.com/docs/2.0/caching/
      - restore_cache:
          keys:
            - node-v4-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "gulpfile.js" }}
      #      - node-v4-{{ .Branch }}-
      #      - node-v4-
      - run:
          name: Install gettext
          command: sudo apt-get update && sudo apt-get install -yqq gettext
      - run:
          name: Node install, generate CSS
          command: |
            npm ci
            node node_modules/gulp/bin/gulp build
      - save_cache:
          paths:
            - /home/circleci/code/node_modules/
          key: node-v4-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "gulpfile.js" }}
      - restore_cache:
          keys:
            - pip-packages-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
      #      - pip-packages-v4-{{ .Branch }}-
      #      - pip-packages-v4-
      - *install-python-dev-dependencies
      - run:
          name: Migrate the database
          command: pipenv run crt_portal/manage.py migrate
      - save_cache:
          paths:
            - /home/circleci/code/.venv # this path depends on where pipenv creates a virtualenv
          key: pip-packages-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
      - run:
          name: Run prettier code formatting check
          command: npm run lint:check
      - run:
          name: Run unit tests & check coverage
          command: |
            pipenv run coverage run --source='crt_portal' crt_portal/manage.py test --settings=crt_portal.test_settings shortener tms features cts_forms
            pipenv run coverage report --fail-under=80 -m
      - run:
          name: Run bandit (Python security) tests
          command: |
            pipenv run bandit -r crt_portal/ --exclude cts_forms/tests
      - run:
          name: Run flake8 test for Python code style
          command: |
            pipenv run flake8
      - run:
          name: add or update response templates
          command: |
            pipenv run crt_portal/manage.py update_response_templates
      - run:
          name: add or update analytics notebooks examples
          command: |
            pipenv run crt_portal/manage.py update_ipynb_examples
      - run:
          name: compile messages for internationalization
          command: |
            pipenv run crt_portal/manage.py compilemessages
      - run:
          name: create mock reports
          command: |
            pipenv run crt_portal/manage.py create_mock_reports 100
      - run:
          name: save a sample of reports as an artifact
          command: |
            PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -h localhost -p 5432 -d circle_test -U root --no-owner --no-privileges -t '(geocoding_|cts_forms_|tms_|shortener_)*' -t django_content_type -t actstream_action > example.sql

      - store_artifacts:
          path: example.sql
          destination: example.sql
      - run:
          name: run server for tests
          environment:
            DEBUG: true
          command: |
            # create pa11y_tester
            echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_user('pa11y_tester', 'admin@myproject.com', '$PA11Y_PASSWORD')" | pipenv run crt_portal/manage.py shell
            # runserver
            pipenv run crt_portal/manage.py runserver 0.0.0.0:8000
          background: true
      - run:
          name: Run pa11y-ci accessibility check
          environment:
            PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome
          command: |
            curl --retry-delay 5 --retry 10 --retry-all-errors http://localhost:8000
            npm run test:a11y-without-report
      - store_artifacts:
          path: ./pa11y-screenshots
      - run:
          name: E2E tests
          environment:
            PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome
          command: pipenv run pytest crt_portal/cts_forms/tests/integration/*.py --base-url=http://localhost:8000
      - run:
          name: Create a test user for e2e tests that require authentication.
          command: pipenv run crt_portal/manage.py create_test_user
      - run:
          name: Authed E2E tests
          environment:
            PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome
          command: pipenv run pytest crt_portal/cts_forms/tests/integration_authed/*.py --base-url=http://localhost:8000
      - store_artifacts:
          path: ./e2e-screenshots
  e2e-test-dev:
    description: Runs E2E tests against dev
    working_directory: ~/code
    docker:
      # CircleCI Python images available at: https://hub.docker.com/r/cimg/python/
      - image: cimg/python:3.12.5-browsers
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
    steps:
      - checkout
      - *install-python-dev-dependencies
      - run:
          name: E2E tests
          command: pipenv run pytest crt_portal/cts_forms/tests/integration/*.py --base-url=https://crt-portal-django-dev.app.cloud.gov

  e2e-test-stage:
    description: Runs E2E tests against stage
    working_directory: ~/code
    docker:
      # CircleCI Python images available at: https://hub.docker.com/r/cimg/python/
      - image: cimg/python:3.12.5-browsers
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
    steps:
      - checkout
      - *install-python-dev-dependencies
      - run:
          name: E2E tests
          command: pipenv run pytest crt_portal/cts_forms/tests/integration/*.py --base-url=https://crt-portal-django-stage.app.cloud.gov

  # owasp scans on live sites
  # NOTE: These don't use docker or the circleci base image because, as of writing, circleci docker-in-docker doesn't support mounting volumes.
  owasp-scan-dev:
    machine:
      image: ubuntu-2204:2024.01.1
    working_directory: ~/code
    steps:
      - checkout
      - run:
          name: Pull owasp zap docker image and run scan
          command: |
            docker run -v $(pwd):/zap/wrk/:rw -t softwaresecurityproject/zap-stable zap-baseline.py -t https://crt-portal-django-dev.app.cloud.gov/report/ -c .circleci/zap.conf --autooff -z "-config rules.cookie.ignorelist=django_language"
  owasp-scan-stage:
    machine:
      image: ubuntu-2204:2024.01.1
    working_directory: ~/code
    steps:
      - checkout
      - run:
          name: Pull owasp zap docker image and run scan
          command: |
            docker run -v $(pwd):/zap/wrk/:rw -t softwaresecurityproject/zap-stable zap-baseline.py -t https://crt-portal-django-stage.app.cloud.gov/report/ -c .circleci/zap.conf --autooff -z "-config rules.cookie.ignorelist=django_language"
  owasp-scan-prod:
    machine:
      image: ubuntu-2204:2024.01.1
    working_directory: ~/code
    steps:
      - checkout
      - run:
          name: Pull owasp zap docker image and run scan
          command: |
            docker run -v $(pwd):/zap/wrk/:rw -t softwaresecurityproject/zap-stable zap-baseline.py -t https://civilrights.justice.gov/report -c .circleci/zap.conf --autooff -z "-config rules.cookie.ignorelist=django_language"
  # deployments
  deploy-dev:
    working_directory: ~/code
    docker:
      # standard docker python image
      - image: python:3.12.5
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
          DATABASE_URL: postgresql://root@localhost/circle_test?sslmode=disable
          CF_MANIFEST: manifest_dev.yaml
          CF_STAGING_TIMEOUT: 40
          CF_STARTUP_TIMEOUT: 10
          CF_SPACE: dev
      # CircleCI PostgreSQL images available at: https://hub.docker.com/_/postgres
      - image: postgres:12.15-alpine
        environment:
          POSTGRES_USER: root
          POSTGRES_DB: circle_test
          POSTGRES_PASSWORD: password
    steps:
      - checkout
      # removing as we deal with caching problem
      # - restore_cache:
      # # Read about caching dependencies: https://circleci.com/docs/2.0/caching/
      #     key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
      - run:
          name: Node install
          command: |
            echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
            curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
            apt-get update && \
            apt-get install -yqq nodejs && \
            rm -rf /var/lib/apt/lists/*
            npm ci --production
            node node_modules/gulp/bin/gulp build-css
            node node_modules/gulp/bin/gulp build-js
      - run:
          name: install cf8
          command: |
            curl -A "" -L -o cf8.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
            dpkg -i cf8.deb
            rm cf8.deb
            cf8 api https://api.fr.cloud.gov
      # - save_cache: # cache Python and Node dependencies using checksum of Pipfile and package.json as the cache-key
      #     key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
      #     paths:
      #       - ".venv"
      #       - "/usr/local/bin"
      #       - "/usr/local/lib/python3.8/site-packages"
      - run:
          name: Login to cloud.gov
          command: cf8 login -u ${CRT_USERNAME_DEV} -p ${CRT_PASSWORD_DEV} -o doj-crtportal -s ${CF_SPACE}
      - acquire-lock:
          app: crt-portal-django
      - run:
          name: Deploy clamav-rest to dev
          # Hide possibly sensitive build information, while still producing output to avoid triggering a no_output_timeout
          command: cf8 push clamav-rest --strategy rolling -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-django to dev
          no_output_timeout: 20m
          command: cf8 push crt-portal-django --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-jupyter to dev
          no_output_timeout: 20m
          command: cf8 push crt-portal-jupyter --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - release-lock:
          app: crt-portal-django

  deploy-stage:
    working_directory: ~/code
    docker:
      - image: python:3.12.5
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
          DATABASE_URL: postgresql://root@localhost/circle_test?sslmode=disable
          CF_MANIFEST: manifest_staging.yaml
          CF_STAGING_TIMEOUT: 40
          CF_STARTUP_TIMEOUT: 10
          CF_SPACE: staging
      # CircleCI PostgreSQL images available at: https://hub.docker.com/_/postgres
      - image: postgres:12.15-alpine
        environment:
          POSTGRES_USER: root
          POSTGRES_DB: circle_test
          POSTGRES_PASSWORD: password
    steps:
      - checkout
      - restore_cache:
          # Read about caching dependencies: https://circleci.com/docs/2.0/caching/
          key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
      - run:
          name: Node install
          command: |
            echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
            curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
            apt-get update && \
            apt-get install -yqq nodejs && \
            rm -rf /var/lib/apt/lists/*
            npm ci --production
            node node_modules/gulp/bin/gulp build-css
            node node_modules/gulp/bin/gulp build-js
      # database will be migrated in cloud.gov
      - run:
          name: install cf8
          command: |
            curl -A "" -L -o cf8.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
            dpkg -i cf8.deb
            rm cf8.deb
            cf8 api https://api.fr.cloud.gov
      - save_cache: # cache Python and Node dependencies using checksum of Pipfile and package.json as the cache-key
          key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
          paths:
            - '.venv'
            - '/usr/local/bin'
            - '/usr/local/lib/python3.12/site-packages'
      - run:
          name: Login to cloud.gov
          command: cf8 login -u ${CRT_USERNAME_STAGE} -p ${CRT_PASSWORD_STAGE} -o doj-crtportal -s ${CF_SPACE}
      - run:
          name: Deploy clamav-rest to staging
          command: cf8 push clamav-rest --strategy rolling -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-django to staging
          no_output_timeout: 20m
          command: cf8 push crt-portal-django --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-jupyter to staging
          no_output_timeout: 20m
          command: cf8 push crt-portal-jupyter --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'

  deploy-prod:
    working_directory: ~/code
    docker:
      # CircleCI Python images available at: https://hub.docker.com/r/cimg/python/
      - image: python:3.12.5
        environment: # environment variables for primary container
          PIPENV_VENV_IN_PROJECT: true
          DATABASE_URL: postgresql://root@localhost/circle_test?sslmode=disable
          CF_MANIFEST: manifest_prod.yaml
          CF_STAGING_TIMEOUT: 40
          CF_STARTUP_TIMEOUT: 10
          CF_SPACE: prod
      # CircleCI PostgreSQL images available at: https://hub.docker.com/_/postgres
      - image: postgres:12.15-alpine
        environment:
          POSTGRES_USER: root
          POSTGRES_DB: circle_test
          POSTGRES_PASSWORD: password
    steps:
      - checkout
      - restore_cache:
          # Read about caching dependencies: https://circleci.com/docs/2.0/caching/
          key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
      - run:
          name: Node install
          command: |
            echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
            curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
            apt-get update && \
            apt-get install -yqq nodejs && \
            rm -rf /var/lib/apt/lists/*
            npm ci --production
            node node_modules/gulp/bin/gulp build-css
            node node_modules/gulp/bin/gulp build-js
      # database will be migrated in cloud.gov
      - run:
          name: install cf8
          command: |
            curl -A "" -L -o cf8.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
            dpkg -i cf8.deb
            rm cf8.deb
            cf8 api https://api.fr.cloud.gov
      - save_cache: # cache Python and Node dependencies using checksum of Pipfile and package.json as the cache-key
          key: deps-v4-{{ .Branch }}-{{ checksum "Pipfile.lock" }}-{{ checksum "package.json" }}
          paths:
            - '.venv'
            - '/usr/local/bin'
            - '/usr/local/lib/python3.12/site-packages'
      - run:
          name: Login to cloud.gov
          command: cf8 login -u ${CRT_USERNAME_PROD} -p ${CRT_PASSWORD_PROD} -o doj-crtportal -s ${CF_SPACE}
      - run:
          name: Deploy clamav-rest to production
          command: cf8 push clamav-rest --strategy rolling -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-django to production
          no_output_timeout: 20m
          command: cf8 push crt-portal-django --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'
      - run:
          name: Deploy crt-portal-jupyter to production
          no_output_timeout: 20m
          command: cf8 push crt-portal-jupyter --strategy rolling --app-start-timeout 180 -f ${CF_MANIFEST} | sed 's/.*/Use cf logs to view build information/'

  prod-maintenance-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: prod
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login into cloud.gov prod space
      - run:
          name: Login to cloud.gov Production
          command: cf login -u ${CRT_USERNAME_PROD} -p ${CRT_PASSWORD_PROD} -o doj-crtportal -s ${CF_SPACE}
      # Refresh the trend view task in prod deactivated
      # - run:
      #    name: Refresh trends view
      #    command: cf run-task crt-portal-django  -c "python crt_portal/manage.py refresh_trends" --name refresh-trends -m 512M -k 7G
      # Mark by_repeat_writer true on reports that were submitted by repeat writers or which have violation summaries identical to other reports
      - run:
          name: Flag reports submitted by repeat writers or that have violation summaries identical to other reports
          command: cf run-task crt-portal-django -c "python crt_portal/manage.py flag_repeat_writers" --name flag-repeat-writers -m 512M -k 7G
      - *wait-for-tasks
      # Clear expired sessions in staging session table
      - run:
          name: clearsessions management command
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py clearsessions" --name clear-sessions -m 512M -k 7G
      - *wait-for-tasks
      #
      # - run:
      #     name: generate yearly reports command
      #     command: cf run-task crt-portal-django  -c "python crt_portal/manage.py generate_yearly_reports" --name generate-yearly-reports -m 512M -k 7G
      - run:
          name: generate repeat_writer_info
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py generate_repeat_writer_info" --name generate-repeat-writer-info -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: check saved searches
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py check_saved_searches" --name process-reports -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: send scheduled notifications
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py send_scheduled_notifications" --name send-scheduled-notifications -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: dispose of reports
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py dispose_reports" --name dispose-reports -m 512M -k 7G

  staging-maintenance-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: staging
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login to Staging Cloud.gov space
      - run:
          name: Login to cloud.gov Staging
          command: cf login -u ${CRT_USERNAME_STAGE} -p ${CRT_PASSWORD_STAGE} -o doj-crtportal -s ${CF_SPACE}
      # Refresh the form letters sent view in staging
      - run:
          name: Refresh form letters sent view
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py refresh_form_letters_sent_view" --name refresh-form-letters-sent-view -m 512M -k 5G
      - *wait-for-tasks
      # Mark by_repeat_writer true on reports that were submitted by repeat writers or which have violation summaries identical to other reports
      - run:
          name: Flag reports submitted by repeat writers or that have violation summaries identical to other reports
          command: cf run-task crt-portal-django -c "python crt_portal/manage.py flag_repeat_writers" --name flag-repeat-writers -m 512M -k 5G
      - *wait-for-tasks
      # Refresh the trend view in staging deactivated
      # - run:
      #     name: Refresh trends view
      #     command: cf run-task crt-portal-django  -c "python crt_portal/manage.py refresh_trends" --name refresh-trends -m 512M -k 5G
      # Clear expired sessions in staging session table
      - run:
          name: clearsessions management command
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py clearsessions" --name clear-sessions -m 512M -k 5G
      - *wait-for-tasks
      # - run:
      #     name: generate yearly reports command
      #     command: cf run-task crt-portal-django  -c "python crt_portal/manage.py generate_yearly_reports" --name generate-yearly-reports -m 512M -k 5G
      - run:
          name: generate repeat_writer_info
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py generate_repeat_writer_info" --name generate-repeat-writer-info -m 512M -k 5G
      - *wait-for-tasks
      - run:
          name: check saved searches
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py check_saved_searches" --name process-reports -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: send scheduled notifications
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py send_scheduled_notifications" --name send-scheduled-notifications -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: dispose of reports
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py dispose_reports" --name dispose-reports -m 512M -k 7G

  dev-maintenance-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: dev
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login to Cloud.gov dev space
      - run:
          name: Login to cloud.gov Dev
          command: cf login -u ${CRT_USERNAME_DEV} -p ${CRT_PASSWORD_DEV} -o doj-crtportal -s ${CF_SPACE}
      # Refresh the form letters sent view in dev
      - run:
          name: Refresh form letters sent view
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py refresh_form_letters_sent_view" --name refresh-form-letters-sent-view -m 512M -k 5G
      - *wait-for-tasks
      # Mark by_repeat_writer true on reports that were submitted by repeat writers or which have violation summaries identical to other reports
      - run:
          name: Flag reports submitted by repeat writers or that have violation summaries identical to other reports
          command: cf run-task crt-portal-django -c "python crt_portal/manage.py flag_repeat_writers" --name flag-repeat-writers -m 512M -k 5G
      - *wait-for-tasks
      # Refresh the trend view in dev deactivated
      # - run:
      #    name: Refresh trends view
      #    command: cf run-task crt-portal-django  -c "python crt_portal/manage.py refresh_trends" --name refresh-trends -m 512M -k 5G
      # Clear expired sessions in staging session table
      - run:
          name: clearsessions management command
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py clearsessions" --name clear-sessions -m 512M -k 5G
      - *wait-for-tasks
      # - run:
      #     name: generate yearly reports command
      #     command: cf run-task crt-portal-django  -c "python crt_portal/manage.py generate_yearly_reports" --name generate-yearly-reports -m 512M -k 5G
      - run:
          name: check saved searches
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py check_saved_searches" --name process-reports -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: send scheduled notifications
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py send_scheduled_notifications" --name send-scheduled-notifications -m 512M -k 7G
      - *wait-for-tasks
      - run:
          name: dispose of reports
          command: cf run-task crt-portal-django  -c "python crt_portal/manage.py dispose_reports" --name dispose-reports -m 512M -k 7G

  dev-jupyter-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: dev
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login to Cloud.gov dev space
      - run:
          name: Login to cloud.gov Dev
          command: cf login -u ${CRT_USERNAME_DEV} -p ${CRT_PASSWORD_DEV} -o doj-crtportal -s ${CF_SPACE}
      - run:
          name: run scheduled notebooks
          command: cf ssh crt-portal-jupyter  -c 'export LD_LIBRARY_PATH="$HOME/deps/0/lib:$HOME/deps/1/lib:$HOME/deps/2/lib" && export PATH="$PATH:$HOME/deps/2/python/bin/" && cd /home/vcap/app && python run_scheduled_refresh.py'

  staging-jupyter-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: staging
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login to Cloud.gov stage space
      - run:
          name: Login to cloud.gov Staging
          command: cf login -u ${CRT_USERNAME_STAGE} -p ${CRT_PASSWORD_STAGE} -o doj-crtportal -s ${CF_SPACE}
      - run:
          name: run scheduled notebooks
          command: cf ssh crt-portal-jupyter  -c 'export LD_LIBRARY_PATH="$HOME/deps/0/lib:$HOME/deps/1/lib:$HOME/deps/2/lib" && export PATH="$PATH:$HOME/deps/2/python/bin/" && cd /home/vcap/app && python run_scheduled_refresh.py'

  prod-jupyter-tasks:
    docker:
      # Use CircleCI convenience base image since we only need
      # CloudFoundry CLI
      - image: cimg/base:2024.02
        environment:
          - CF_SPACE: prod
    steps:
      # Install cloud foundry tools
      - *install-cf8
      # login to Cloud.gov prod space
      - run:
          name: Login to cloud.gov Prod
          command: cf login -u ${CRT_USERNAME_PROD} -p ${CRT_PASSWORD_PROD} -o doj-crtportal -s ${CF_SPACE}
      - run:
          name: run scheduled notebooks
          command: cf ssh crt-portal-jupyter  -c 'export LD_LIBRARY_PATH="$HOME/deps/0/lib:$HOME/deps/1/lib:$HOME/deps/2/lib" && export PATH="$PATH:$HOME/deps/2/python/bin/" && cd /home/vcap/app && python run_scheduled_refresh.py'

commands:
  acquire-lock:
    description: 'Make sure nothing else is deploying.'
    parameters:
      app:
        type: enum
        enum: ['clamav-rest', 'crt-portal-django']
    steps:
      - run:
          name: 'Wait for other deployments to finish'
          no_output_timeout: 2h
          command: |
            . env-helpers.sh << parameters.app >>
            cf_wait_for_lock CCI_NEXT_TICKET CCI_SERVING_TICKET

  release-lock:
    description: "Indicate that we're done using cloudfoundry, allowing the next process in line to use it."
    parameters:
      app:
        type: enum
        enum: ['clamav-rest', 'crt-portal-django']
    steps:
      - run:
          name: 'Let other deployments start'
          command: |
            . env-helpers.sh << parameters.app >>
            serving="$(cf_increment_env CCI_SERVING_TICKET)"
            printf "Lock released - now serving ticket $((serving + 1))\n"

workflows:
  build-test-deploy:
    jobs:
      - build_and_test

      - owasp-scan-dev:
          filters:
            branches:
              only: /^release.*/

      - owasp-scan-stage:
          filters:
            branches:
              only: master

      - owasp-scan-prod:
          requires:
            - deploy-prod
          filters:
            branches:
              only: master

      - deploy-dev:
          requires:
            - build_and_test
          filters:
            branches:
              only: develop

      - deploy-stage:
          requires:
            - owasp-scan-dev
            - build_and_test
          filters:
            branches:
              only: /^release.*/

      - deploy-prod:
          requires:
            - owasp-scan-stage
            - build_and_test
          filters:
            branches:
              only: master

      - e2e-test-dev:
          requires:
            - deploy-dev
          filters:
            branches:
              only: develop

      - e2e-test-stage:
          requires:
            - deploy-stage
          filters:
            branches:
              only: /^release.*/

  # Scheduled maintenance jobs for portal
  # Dev scheduled jobs
  periodic-tasks-dev:
    jobs:
      - dev-maintenance-tasks #run on dev
    triggers:
      - schedule:
          cron: '0 0 * * *' # run past 8pm every night EST
          filters:
            branches:
              only: develop

  periodic-tasks-jupyter-dev:
    jobs:
      - dev-jupyter-tasks
    triggers:
      - schedule:
          cron: '0 0,6,12,18 * * *' # run four times per day
          filters:
            branches:
              only: develop

  # Staging scheduled jobs
  periodic-tasks-staging:
    jobs:
      - staging-maintenance-tasks #run on stage
    triggers:
      - schedule:
          cron: '0 4 * * *' # run past midnight every night EST
          filters:
            branches:
              only: /^release.*/

  periodic-tasks-jupyter-staging:
    jobs:
      - staging-jupyter-tasks
    triggers:
      - schedule:
          cron: '0 1,7,13,19 * * *' # run four times per day
          filters:
            branches:
              only: /^release.*/

  # prod scheduled jobs
  periodic-tasks-prod:
    jobs:
      - prod-maintenance-tasks #run on prod
    triggers:
      - schedule:
          cron: '0 5 * * *' # run at 1am every night EST
          filters:
            branches:
              only: master

  periodic-tasks-jupyter-prod:
    jobs:
      - prod-jupyter-tasks
    triggers:
      - schedule:
          cron: '0 * * * *' # run every hour
          filters:
            branches:
              only: master