Merge branch 'develop' into gh-85-add-user-api

.github/workflows/nextjs-test-ci.yaml

@@ -20,15 +20,17 @@ jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Node
-      uses: actions/setup-node@v4
-      with:
-        node-version: '18'
     - name: Check out the repository
       uses: actions/checkout@v4
       with:
         token: ${{ github.token }}
+    - name: Set up Node
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
     - name: Install dependencies
-      run: cd app/web && npm ci
+      uses: bahmutov/npm-install@v1
+      with:
+        working-directory: app/web
     - name: Run tests
       run: cd app/web && npm run test

.github/workflows/vidserver-lint.yaml

@@ -0,0 +1,37 @@
+name: Video Processing Server Lint
+
+concurrency:
+  group: ${{ github.run_id }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - "develop"
+      - "master"
+      - "main"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repository
+      uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+        cache-dependency-path: "**/requirements*.txt"
+    - name: Install dependencies
+      run: |
+        pip install -r app/video-processing/requirements.txt -r app/video-processing/requirements_dev.txt
+    - name: Run tests
+      run: cd app/video-processing/ && make lint

.github/workflows/vidserver-test-ci.yaml

@@ -20,15 +20,17 @@ jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.8' 
     - name: Check out the repository
       uses: actions/checkout@v4
       with:
         token: ${{ github.token }}
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+        cache-dependency-path: "**/requirements.txt"
     - name: Install dependencies
-      run: pip install -r app/video-processing/requirements.txt
+      run: pip install -r app/video-processing/requirements.txt -r app/video-processing/requirements_dev.txt
     - name: Run tests
-      run: cd app/video-processing/ && python3 -m unittest discover -s .
+      run: cd app/video-processing/ && make test

app/smoketest/compose/db.yaml

@@ -0,0 +1,75 @@
+version: "3"
+
+services:
+  db:
+    image: docker.io/library/postgres:16-bullseye
+    networks:
+      - backend
+      - db-admin
+    hostname: db
+    expose:
+      - 5432
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_DB: privacypal
+      POSTGRES_USER: privacypal
+      POSTGRES_PASSWORD: password
+    volumes:
+      - postgres:/var/lib/pgsql/data
+    restart: unless-stopped
+    healthcheck:
+      test: pg_isready -U privacypal -d privacypal || exit 1
+      interval: 10s
+      retries: 3
+      start_period: 10s
+      timeout: 5s
+
+  db-init:
+    image: ghcr.io/cosc-499-w2023/privacypal-init-db:0.1.0-dev
+    depends_on:
+      db:
+        condition: service_healthy
+    build: ../../web/db
+    networks:
+      - db-admin
+    environment:
+      DATABASE_URL: postgresql://privacypal:password@db:5432/privacypal
+      PRIVACYPAL_AUTH_MANAGER: ${PRIVACYPAL_AUTH_MANAGER:-basic}
+      PRIVACYPAL_USER_PROPERTY_PATH: /opt/privacypal/user.properties.csv
+    volumes:
+      - ../../web/db/sample/user.properties.csv:/opt/privacypal/user.properties.csv:z
+
+  db-viewer:
+    image: docker.io/dpage/pgadmin4:7
+    depends_on:
+      db:
+        condition: service_healthy
+    hostname: db-viewer
+    ports:
+      - "8989:8989"
+    networks:
+      - db-admin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@privacypal.io
+      PGADMIN_DEFAULT_PASSWORD: admin
+      PGADMIN_LISTEN_PORT: 8989
+    volumes:
+      - pgadmin:/var/lib/pgadmin
+      - ./include/servers.json:/pgadmin4/servers.json:z
+    restart: always
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8989 || exit 1
+      interval: 10s
+      retries: 3
+      start_period: 10s
+      timeout: 5s
+
+networks:
+  db-admin:
+
+volumes:
+  postgres:
+    driver: local
+  pgadmin:
+    driver: local

app/smoketest/compose/privacypal.yaml

@@ -26,7 +26,7 @@ services:
       PRIVACYPAL_OUTPUT_VIDEO_DIR: /opt/privacypal/output-videos
       NEXTAUTH_SECRET: a-very-secure-one
       NEXTAUTH_URL: http://localhost:8080
-      DATABASE_URL: postgresql://privacypal:a_secure_password@db:5432/privacypal
+      DATABASE_URL: postgresql://privacypal:password@db:5432/privacypal
     volumes:
       - output_videos:/opt/privacypal/output-videos
       - input_videos:/opt/privacypal/input-videos
@@ -58,76 +58,11 @@ services:
       - input_videos:/opt/privacypal/input-videos
     restart: unless-stopped
 
-  db:
-    image: docker.io/library/postgres:16-bullseye
-    networks:
-      - backend
-      - db-admin
-    hostname: db
-    expose:
-      - 5432
-    ports:
-      - "5432:5432"
-    environment:
-      POSTGRES_DB: privacypal
-      POSTGRES_USER: privacypal
-      POSTGRES_PASSWORD: a_secure_password
-    volumes:
-      - postgres:/var/lib/pgsql/data
-    restart: unless-stopped
-    healthcheck:
-      test: pg_isready -U privacypal -d privacypal || exit 1
-      interval: 10s
-      retries: 3
-      start_period: 10s
-      timeout: 5s
-
-  db-init:
-    image: ghcr.io/cosc-499-w2023/privacypal-init-db:0.1.0-dev
-    depends_on:
-      db:
-        condition: service_healthy
-    build: ../../web/db
-    networks:
-      - db-admin
-    environment:
-      DATABASE_URL: postgresql://privacypal:a_secure_password@db:5432/privacypal
-
-  db-viewer:
-    image: docker.io/dpage/pgadmin4:7
-    depends_on:
-      db:
-        condition: service_healthy
-    hostname: db-viewer
-    ports:
-      - "8989:8989"
-    networks:
-      - db-admin
-    environment:
-      PGADMIN_DEFAULT_EMAIL: admin@privacypal.io
-      PGADMIN_DEFAULT_PASSWORD: admin
-      PGADMIN_LISTEN_PORT: 8989
-    volumes:
-      - pgadmin:/var/lib/pgadmin
-      - ./include/servers.json:/pgadmin4/servers.json:z
-    restart: always
-    healthcheck:
-      test: wget --no-verbose --tries=1 --spider http://localhost:8989 || exit 1
-      interval: 10s
-      retries: 3
-      start_period: 10s
-      timeout: 5s
-
 networks:
   backend:
-  db-admin:
 
 volumes:
   output_videos:
     driver: local
   input_videos:
     driver: local
-  postgres:
-    driver: local
-  pgadmin:
-    driver: local

app/smoketest/compose/smoketest.sh

@@ -1,14 +1,27 @@
 #!/bin/bash
 
+set -ex
+
 COMPOSE_TOOL=${COMPOSE_TOOL:-docker-compose}
 
+FILES=(
+    privacypal.yaml
+    db.yaml
+)
+
+CMDS=()
+
+for file in "${FILES[@]}"; do
+    CMDS+=( -f "${file}" )
+done
+
 cleanup() {
-    DOWN_FLAGS=('--remove-orphans')
+    local DOWN_FLAGS=('--remove-orphans')
     if [ "${KEEP_VOLUMES}" != "true" ]; then
         DOWN_FLAGS+=('--volumes')
     fi
     ${COMPOSE_TOOL} \
-        -f privacypal.yaml \
+        "${CMDS[@]}" \
         down "${DOWN_FLAGS[@]}"
 }
 
@@ -17,5 +30,5 @@ trap cleanup EXIT
 cleanup
 
 ${COMPOSE_TOOL} \
-    -f privacypal.yaml \
+    "${CMDS[@]}" \
     up

app/video-processing/.dockerignore

@@ -89,3 +89,19 @@ venv.bak/
 
 # VS Code
 .vscode/
+
+# Makefile
+Makefile
+
+# Docs
+**/*.md
+
+# Test resources
+**/*.mp4
+**/*.jpg
+**/*.png
+test-samples/
+*-videos/
+
+# Test
+test_*

app/video-processing/.gitignore

@@ -87,5 +87,6 @@ venv.bak/
 # Logs
 **/*.log
 
-# we need these files for tests
-!/resources/*.mp4
+# Exlude samples
+!/samples/*.mp4
+!/samples/*.jpg

app/video-processing/Dockerfile

@@ -1,42 +1,41 @@
-FROM docker.io/library/python:3.8-slim-bullseye AS builder
+FROM docker.io/library/python:3.9-slim-bullseye AS base
 
 USER root
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
+RUN apt-get update
+
+FROM base AS builder
+
 WORKDIR /build
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential gcc ffmpeg libsm6 libxext6
+RUN apt-get install -y --no-install-recommends build-essential gcc ffmpeg libsm6 libxext6
 
 COPY requirements.txt .
 
 RUN pip wheel --no-cache-dir --no-deps --wheel-dir /build/wheels -r requirements.txt
 
-FROM docker.io/library/python:3.8-slim-bullseye AS runner
+FROM base AS runner
 
-USER root
+RUN apt-get install -y --no-install-recommends ffmpeg
 
 ENV CONFIG_DIR=/opt/privacypal
 
 RUN mkdir -p $CONFIG_DIR && chmod -R g=u $CONFIG_DIR
 
-RUN adduser --system --ingroup root gunicorn
+RUN adduser --system --ingroup root uvicorn
 
 WORKDIR /app
 
 COPY --from=builder /build/wheels /wheels
 
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends ffmpeg
-
 RUN pip install --no-cache /wheels/*
 
-COPY --chown=gunicorn . .
+COPY --chown=uvicorn . .
 
-USER gunicorn
+USER uvicorn
 
 EXPOSE 3000