Merge branch 'develop' into gh-349-video-delete-api

COSC-499-W2023 · Feb 19, 2024 · 00eb775 · 00eb775
2 parents d830189 + 6c6db9f
commit 00eb775
Show file tree

Hide file tree

Showing 107 changed files with 2,526 additions and 2,336 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -9,7 +9,7 @@ updates:
       - "dependencies"
       - "chore"
   - package-ecosystem: "pip"
-    directory: "/app/video-processing"
+    directory: "/app/video-processing/lambda"
     target-branch: "develop"
     schedule:
       interval: "daily"

diff --git a/.github/workflows/build-ci.yaml b/.github/workflows/build-ci.yaml
@@ -63,46 +63,6 @@ jobs:
         tags: ${{ steps.get-image-tag.outputs.tags }}
         registry: ${{ secrets.AWS_ECR_REPO_BASE }}
 
-  build-video-processor:
-    name: Build video processing server image
-    runs-on: ubuntu-latest
-    steps:
-    - name: Install podman v4
-      run: |
-        echo "deb $OPENSUSE_UNOFFICIAL_LIBCONTAINERS_SOURCE_URL/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list
-        curl -fsSL $OPENSUSE_UNOFFICIAL_LIBCONTAINERS_KEY_URL | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/devel_kubic_libcontainers_unstable.gpg > /dev/null
-        sudo apt -y purge podman
-        sudo apt update && sudo apt -y install podman
-    - uses: actions/checkout@v4
-      with:
-        token: ${{ github.token }}
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v3
-      with:
-        role-to-assume: ${{ secrets.AWS_GITHUB_ROLE_ARN }}
-        aws-region: ${{ env.AWS_REGION }}
-    - name: Build video processor image
-      working-directory: app/video-processing
-      run: |
-        IMAGE_NAMESPACE="${{ secrets.AWS_ECR_REPO_BASE }}" make oci-build
-    - name: Get image tag
-      id: get-image-tag
-      working-directory: app/video-processing
-      run: |
-        IMG_TAG="$(make --eval='print-img-ver: ; @echo $(IMAGE_VERSION)' print-img-ver)"
-        echo "tags=$IMG_TAG" >> $GITHUB_OUTPUT
-    - name: Login to Amazon ECR
-      uses: aws-actions/amazon-ecr-login@v2
-      with:
-        mask-password: true
-    - name: Push to AWS ECR
-      id: push-to-ecr
-      uses: redhat-actions/push-to-registry@v2
-      with:
-        image: privacypal-vidprocess
-        tags: ${{ steps.get-image-tag.outputs.tags }}
-        registry: ${{ secrets.AWS_ECR_REPO_BASE }}
-
   build-video-processor-lambda:
     name: Build video processing server Lambda image
     runs-on: ubuntu-latest

diff --git a/.github/workflows/vidserver-test-ci.yaml → .github/workflows/db-test-ci.yaml b/.github/workflows/vidserver-test-ci.yaml → .github/workflows/db-test-ci.yaml
@@ -1,4 +1,4 @@
-name: Video Processing Server Unit Tests
+name: Database Initializer Unit Tests
 
 concurrency:
   group: ${{ github.run_id }}
@@ -24,15 +24,6 @@ jobs:
       uses: actions/checkout@v4
       with:
         token: ${{ github.token }}
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.9'
-        cache: 'pip'
-        cache-dependency-path: '**/requirements*.txt'
-    - name: Install dependencies
-      working-directory: app/video-processing
-      run: pip install -r requirements.txt -r requirements_dev.txt
     - name: Run tests
-      working-directory: app/video-processing
+      working-directory: app/web/db
       run: make test
diff --git a/.github/workflows/vidserver-lint.yaml → .github/workflows/lambda-lint.yaml b/.github/workflows/vidserver-lint.yaml → .github/workflows/lambda-lint.yaml
@@ -1,4 +1,4 @@
-name: Video Processing Server Format Check
+name: Video Processing Lambda Format Check
 
 concurrency:
   group: ${{ github.run_id }}
@@ -27,11 +27,12 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.9'
+        python-version: '3.12'
         cache: 'pip'
         cache-dependency-path: "**/requirements*.txt"
     - name: Install dependencies
-      run: |
-        pip install -r app/video-processing/requirements.txt -r app/video-processing/requirements_dev.txt
+      working-directory: app/video-processing/lambda
+      run: pip install -r requirements_dev.txt
     - name: Run flake8 check
-      run: cd app/video-processing/ && make lint
+      working-directory: app/video-processing
+      run: make lint
diff --git a/.github/workflows/license-check.yaml b/.github/workflows/license-check.yaml
@@ -0,0 +1,50 @@
+name: License check
+
+concurrency:
+  group: ${{ github.run_id }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - "develop"
+      - "master"
+      - "main"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  web-license:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repository
+      uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+    - name: Set up Node
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+        cache: 'npm'
+        cache-dependency-path: app/web/package-lock.json
+    - name: Install dependencies
+      uses: bahmutov/npm-install@v1
+      with:
+        working-directory: app/web
+    - name: Check license headers
+      working-directory: app/web
+      run: npm run license:check
+
+  lambda-license:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out the repository
+      uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+    - name: Check license headers
+      working-directory: app/video-processing
+      run: make check-license
diff --git a/.github/workflows/nextjs-lint.yaml b/.github/workflows/nextjs-lint.yaml
@@ -28,6 +28,8 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: '18'
+        cache: 'npm'
+        cache-dependency-path: app/web/package-lock.json
     - name: Install dependencies
       uses: bahmutov/npm-install@v1
       with:

diff --git a/.github/workflows/nextjs-test-ci.yaml b/.github/workflows/nextjs-test-ci.yaml
@@ -28,6 +28,8 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: '18'
+        cache: 'npm'
+        cache-dependency-path: app/web/package-lock.json
     - name: Install dependencies
       uses: bahmutov/npm-install@v1
       with:

diff --git a/.gitignore b/.gitignore
@@ -18,3 +18,6 @@
 
 # don't commit system files
 .DS_Store
+
+# makefile for windows
+**/Makefile-win
diff --git a/app/smoketest/compose/README.md b/app/smoketest/compose/README.md
@@ -7,23 +7,6 @@
 
 **Notes**: Alternatively, `docker` and `docker compose` plugin can be used.
 
-
-## PULL IMAGES
-
-To configure the container tool (e.g. `podman`) to pull images from Github Container Registry (`ghcr.io`):
-
-- Generate a Github Personal Access Token (PAT) with `read:packages` scope.
-- Login with container tool with
-
-    ```bash
-    $ podman login ghcr.io
-    Username: <github_username>
-    Password: <PAT>
-    ```
-
-
-For more details, follow this [guide](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-with-a-personal-access-token-classic). **Note**: With `podman`, re-authentication across reboots might be necessary.
-
 ## RUN
 
 ### Launch Podman API
@@ -51,6 +34,19 @@ To export the credentials for the processing server, run:
 eval $(aws configure export-credentials --profile <your-profile> --format env)
 ```
 
+
+### Configure AWS Cognito
+
+The application requires Cognito connection information in order to authenticate users, set the following environment variables:
+
+```bash
+export AWS_CLIENT=<client>
+export AWS_CLIENT_SECRET=<secret>
+export AWS_POOL_ID=<id>
+```
+
+Note: You can find these values in AWS Cognito console.
+
 ### Launch services
 
 To launch composed services, run:

diff --git a/app/smoketest/compose/db.yaml b/app/smoketest/compose/db.yaml
@@ -31,14 +31,16 @@ services:
             db:
                 condition: service_healthy
         build: ../../web/db
+        pull_policy: build
         networks:
             - db-admin
         environment:
             DATABASE_URL: postgresql://privacypal:password@db:5432/privacypal
-            PRIVACYPAL_AUTH_MANAGER: ${PRIVACYPAL_AUTH_MANAGER:-basic}
-            PRIVACYPAL_USER_PROPERTY_PATH: /opt/privacypal/user.properties.csv
-        volumes:
-            - ../../web/db/sample/user.properties.csv:/opt/privacypal/user.properties.csv:z
+            PRIVACYPAL_POSTGRES_USERNAME: privacypal
+            PRIVACYPAL_POSTGRES_PASSWORD: password
+            PRIVACYPAL_POSTGRES_HOST: db
+            PRIVACYPAL_POSTGRES_PORT: 5432
+            PRIVACYPAL_POSTGRES_DATABASE: privacypal
 
     db-viewer:
         image: docker.io/dpage/pgadmin4:8
@@ -56,7 +58,7 @@ services:
             PGADMIN_LISTEN_PORT: 8989
         volumes:
             - pgadmin:/var/lib/pgadmin
-            - ./include/servers.json:/pgadmin4/servers.json:z
+            - ./include/servers.json:/pgadmin4/servers.json
         restart: always
         healthcheck:
             test: wget --no-verbose --tries=1 --spider http://localhost:8989 || exit 1

diff --git a/app/smoketest/compose/privacypal.yaml b/app/smoketest/compose/privacypal.yaml
@@ -4,7 +4,7 @@ services:
     privacypal:
         image: ghcr.io/cosc-499-w2023/privacypal:0.1.0-dev
         build: ../../web/
-        user: "0"
+        pull_policy: build
         depends_on:
             db-init:
                 condition: service_completed_successfully
@@ -20,51 +20,26 @@ services:
             AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
             AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
             AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
+            COGNITO_CLIENT: ${COGNITO_CLIENT}
+            COGNITO_CLIENT_SECRET: ${COGNITO_CLIENT_SECRET}
+            COGNITO_POOL_ID: ${COGNITO_POOL_ID}
             AWS_REGION: ca-central-1
             PRIVACYPAL_CONFIG_DIR: /opt/privacypal/
-            PRIVACYPAL_INPUT_VIDEO_DIR: /opt/privacypal/input-videos
-            PRIVACYPAL_OUTPUT_VIDEO_DIR: /opt/privacypal/output-videos
-            PRIVACYPAL_AUTH_MANAGER: basic
-            PRIVACYPAL_DEBUG: true
+            PRIVACYPAL_AUTH_MANAGER: cognito
+            PRIVACYPAL_DEBUG: "true"
             PRIVACYPAL_AUTH_SECRET: NZFbGxlrFl5Ae2kh0pcKbsiNVL37eEvtSg4zunBpfmw=
             PRIVACYPAL_COOKIE_NAME: privacypal
-            DATABASE_URL: postgresql://privacypal:password@db:5432/privacypal
-            PRIVACYPAL_PROCESSOR_URL: http://video-processor:3000
-        volumes:
-            - output_videos:/opt/privacypal/output-videos
-            - input_videos:/opt/privacypal/input-videos
-        restart: unless-stopped
-
-    video-processor:
-        image: ghcr.io/cosc-499-w2023/privacypal-vidprocess:0.1.0-dev
-        build: ../../video-processing
-        user: "0"
-        networks:
-            - backend
-        hostname: video-processor
-        expose:
-            - 3000
-        ports:
-            - "3000:3000"
-        environment:
-            PORT: 3000
-            AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-            AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-            AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
-            AWS_DEFAULT_REGION: ca-central-1
-            PRIVACYPAL_CONFIG_DIR: /opt/privacypal/
-            PRIVACYPAL_INPUT_VIDEO_DIR: /opt/privacypal/input-videos
-            PRIVACYPAL_OUTPUT_VIDEO_DIR: /opt/privacypal/output-videos
-        volumes:
-            - output_videos:/opt/privacypal/output-videos
-            - input_videos:/opt/privacypal/input-videos
+            PRIVACYPAL_LAMBDA_NAME: processVideoContainer
+            PRIVACYPAL_OUTPUT_BUCKET: privacypal-output
+            PRIVACYPAL_TMP_BUCKET: privacypal-input
+            NEXTAUTH_URL: http://localhost:8080
+            NEXTAUTH_SECRET: wp9CNmEFfrfAAE46zJzWTUGzpzOvIbwj/P7Hs9E9i9s=
+            PRIVACYPAL_POSTGRES_USERNAME: privacypal
+            PRIVACYPAL_POSTGRES_PASSWORD: password
+            PRIVACYPAL_POSTGRES_HOST: db
+            PRIVACYPAL_POSTGRES_PORT: 5432
+            PRIVACYPAL_POSTGRES_DATABASE: privacypal
         restart: unless-stopped
 
 networks:
     backend:
-
-volumes:
-    output_videos:
-        driver: local
-    input_videos:
-        driver: local