3042: Refactor client applications into a monorepo with turborepo, extract fides-js into a standalone package, and improve privacy-center to load configuration at runtime

.dockerignore

@@ -8,7 +8,14 @@ src/ui-build/
 # Frontend
 **/node_modules
 **/.next
-
+**/.eslintrc.json
+**/.jest.config.js
+**/jest.config.jest
+**/.prettierignore
+**/.prettierrc.json
+**/__tests__
+**/npm-debug.log
+**/*__mocks__
 
 # Ignore Python-Specific Files
 **/.mypy_cache/
@@ -38,6 +45,7 @@ src/ui-build/
 # docs/
 
 # Ignore dev files
+.gitignore
 .github/
 .devcontainer/
 

.github/workflows/frontend_checks.yml

@@ -13,14 +13,18 @@ env:
   CI: true
 
 jobs:
-  Admin-UI-Unit:
+  Clients-Unit:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16.x]
+        clients:
+          - "admin-ui"
+          - "privacy-center"
+          - "fides-js"
     defaults:
       run:
-        working-directory: clients/admin-ui
+        working-directory: clients
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -39,17 +43,24 @@ jobs:
       - name: Format
         run: npm run format:ci
 
-      - name: Jest test
-        run: npm run test:ci
-
       - name: Build
         run: npm run build
 
-  Admin-UI-Cypress:
+      - name: Unit test (${{ matrix.clients }})
+        working-directory: clients/${{ matrix.clients }}
+        run: npm run test:ci
+
+  Clients-Cypress:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16.x]
+        clients:
+          - "admin-ui"
+          - "privacy-center"
+    defaults:
+      run:
+        working-directory: clients
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -60,14 +71,15 @@ jobs:
           node-version: ${{ matrix.node-version }}
 
       - name: Install dependencies
-        run: |
-          cd clients/admin-ui
-          npm install
+        run: npm install
 
-      - name: Cypress E2E tests
+      - name: Build
+        run: npm run build
+
+      - name: Cypress E2E Tests (${{ matrix.clients }})
         uses: cypress-io/github-action@v4
         with:
-          working-directory: clients/admin-ui
+          working-directory: clients/${{ matrix.clients }}
           install: false
           start: npm run cy:start
           wait-on: "http://localhost:3000"
@@ -76,24 +88,19 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: cypress-videos
-          path: /home/runner/work/fides/fides/clients/admin-ui/cypress/videos/*.mp4
+          name: cypress-videos-${{ matrix.clients }}
+          path: /home/runner/work/fides/fides/clients/${{ matrix.clients }}/cypress/videos/*.mp4
 
-      - name: Cypress component tests
-        uses: cypress-io/github-action@v4
-        with:
-          working-directory: clients/admin-ui
-          install: false
-          component: true
-
-  Privacy-Center-Unit:
+  Clients-Cypress-Component:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16.x]
+        clients:
+          - "admin-ui"
     defaults:
       run:
-        working-directory: clients/privacy-center
+        working-directory: clients
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -106,42 +113,13 @@ jobs:
       - name: Install dependencies
         run: npm install
 
-      - name: Lint
-        run: npm run lint
-
-      - name: Format
-        run: npm run format:ci
-
       - name: Build
         run: npm run build
 
-      - name: Jest test
-        run: npm run test:ci
-
-  Privacy-Center-Cypress:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: Install dependencies
-        run: |
-          cd clients/privacy-center 
-          npm install
-
-      - name: Cypress E2E tests
+      - name: Cypress Component Tests (${{ matrix.clients }})
         uses: cypress-io/github-action@v4
         with:
-          working-directory: clients/privacy-center
+          working-directory: clients/${{ matrix.clients }}
           install: false
-          start: npm run cy:start
-          wait-on: "http://localhost:3000"
-          wait-on-timeout: 120
+          component: true
+

.github/workflows/publish_package.yaml

@@ -27,13 +27,13 @@ jobs:
 
       - name: Install node modules
         run: |
-          cd clients/admin-ui
+          cd clients
           npm install
 
       - name: Build and export frontend files
         run: |
-          cd clients/admin-ui
-          npm run prod-export
+          cd clients
+          npm run prod-export-admin-ui
 
       - name: Install Twine and wheel
         run: pip install twine wheel

Dockerfile

@@ -93,23 +93,54 @@ RUN pip install -e . --no-deps
 ###################
 ## Frontend Base ##
 ###################
-FROM node:16-slim as frontend
+FROM node:16-alpine as frontend
+
+RUN apk add --no-cache libc6-compat
+# Build the frontend clients
+WORKDIR /fides/clients
+COPY clients/package.json clients/package-lock.json ./
+COPY clients/fides-js/package.json ./fides-js/package.json
+COPY clients/admin-ui/package.json ./admin-ui/package.json
+COPY clients/privacy-center/package.json ./privacy-center/package.json
 
-# Build the admin-ui frontend
-WORKDIR /fides/clients/admin-ui
-COPY clients/admin-ui/package.json clients/admin-ui/package-lock.json ./
 RUN npm install
-COPY clients/admin-ui/ .
+
+COPY clients/ .
 
 ####################
 ## Built frontend ##
 ####################
 FROM frontend as built_frontend
-RUN npm run export
 
-#############################
+# Builds and exports admin-ui
+RUN npm run export-admin-ui
+# Builds privacy-center
+RUN npm run build-privacy-center
+
+###############################
+## Production Privacy Center ##
+###############################
+FROM node:16-alpine as prod_pc
+
+WORKDIR /fides/clients
+
+ENV NODE_ENV production
+ENV NEXT_TELEMETRY_DISABLED 1
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+USER nextjs
+
+COPY --from=built_frontend --chown=nextjs:nodejs /fides/clients .
+WORKDIR /fides/clients/privacy-center
+
+EXPOSE 3000
+
+CMD ["npm", "run", "start"]
+
+############################
 ## Production Application ##
-#############################
+############################
 FROM backend as prod
 
 # Copy frontend build over

clients/.gitignore

@@ -0,0 +1 @@
+.turbo

clients/README.md

@@ -0,0 +1,51 @@
+# Clients
+
+The clients directory houses all front-end packages and shared code amongst clients, and also includes e2e tests.
+
+## Prerequisites
+
+Dependencies within this directory are managed by Turborepo. Our root `package.json` file defines 3 workspaces (or packages) that are part of the Turbo ecosystem:
+1. admin-ui
+2. privacy-center
+3. fides-js
+
+To run any or all workspaces, you'll have to complete the following setup: 
+
+1. Install Turborepo globally so that you can use it on the command line `npm install turbo --global`
+2. In `clients` folder, run `npm install`. This will install the dev dependency `turbo`. This will also install the appropriate dependencies in `clients` and in each client within `clients` folder. You may need to first remove all `node_modules` in each of the clients (admin-ui, privacy-center, etc)
+
+## Running Locally
+
+To run the project use `turbo` instead of `npm`. 
+
+Incorrect:
+```
+npm run dev
+```
+
+Correct:
+```
+turbo run dev
+```
+
+Running this in the root `clients` folder will result in every workspace being run.
+
+Running this command within `admin-ui` will result in only admin-ui being run.
+
+Available commands that exist for every workspace are defined in the root `turbo.json` file, while commands unique to a specific workspace are defined in the `turbo.json` file within the workspace.
+
+It's important to use the turbo command because, as you see in the `turbo.json` files, we've defined some dependencies and caching details on some turbo commands. 
+
+
+
+## Adding packages
+
+To install packages in any package in `clients`, run the following from `clients`:
+```
+npm install <package> --workspace=<workspace>
+```
+Example:
+```
+npm install react --workspace=admin-ui
+```
+See https://turbo.build/repo/docs/handbook/package-installation#addingremovingupgrading-packages for more details

clients/admin-ui/README.md

@@ -4,7 +4,7 @@ Admin UI for managing Fides privacy requests. A web application built in Next.js
 
 ## Running Locally
 
-1. In a new shell, `cd` into `clients/admin-ui`, then run `npm run dev`.
+1. In a new shell, `cd` into `clients/admin-ui`, then run `turbo run dev`.
 1. Nav to `http://localhost:3000/` and log in using the created user. The `email` field is simply the `user` that was created, not a valid email address.
 
 ## Unit test locations
@@ -25,16 +25,16 @@ of the page header. These changes will be saved per-browser, per-environment, un
 
 Feature flags can be configured independently for the development, test, and production environments:
 
-- **development** - Running `npm run dev` uses the development environment. This will include bleeding-edge features.
-- **test** - Cypress runs in the tet environment via `npm run cy:start`. This will usually match production, but
+- **development** - Running `turbo run dev` uses the development environment. This will include bleeding-edge features.
+- **test** - Cypress runs in the tet environment via `turbo run cy:start`. This will usually match production, but
   new features may be enabled in test to verify them in CI before release.
-- **production** - Only features ready for release will be enabled for production. `npm run build` always produces a
+- **production** - Only features ready for release will be enabled for production. `turbo run build` always produces a
   build configured for production, which is then bundled into the Fides server.
 
 You can switch between these environments manually by overriding the `NEXT_PUBLIC_APP_ENV` environment variable when
 running the app, for example:
 
-`NEXT_PUBLIC_APP_ENV=production npm run dev`
+`NEXT_PUBLIC_APP_ENV=production turbo run dev`
 
 Or you can configure the environment using `env.local` as described by the [Next.js docs](https://nextjs.org/docs/basic-features/environment-variables#loading-environment-variables).
 
@@ -44,7 +44,7 @@ In addition, you can mark a flag as `userCannotModify: true` in `flags.json`. Th
 
 To view a production version of this site, including the backend:
 
-1. Run `npm prod-export`. This will
+1. Run `turbo prod-export`. This will
    1. Export the static site to `out/`
    1. Copy the build from `out/` to the folder in the backend which will serve static assets at `/`
 1. Run `nox -s api` in the top-level `fides` directory.

clients/admin-ui/cypress.config.ts

@@ -2,21 +2,21 @@
 import { defineConfig } from "cypress";
 
 export default defineConfig({
-  e2e: {
-    baseUrl: "http://localhost:3000",
-  },
+    e2e: {
+        baseUrl: "http://localhost:3000",
+    },
 
-  defaultCommandTimeout: 5000,
+    defaultCommandTimeout: 5000,
 
-  retries: {
-    runMode: 3,
-    openMode: 0,
-  },
+    retries: {
+        runMode: 3,
+        openMode: 0,
+    },
 
-  component: {
-    devServer: {
-      framework: "next",
-      bundler: "webpack",
+    component: {
+        devServer: {
+            framework: "next",
+            bundler: "webpack",
+        },
     },
-  },
 });

clients/admin-ui/cypress/.eslintrc.json

@@ -1,11 +1,12 @@
 {
   "extends": ["plugin:cypress/recommended"],
-  "plugins": ["no-only-tests"],
+  "plugins": ["no-only-tests", "cypress"],
   "parserOptions": {
     "project": "./tsconfig.json",
     "tsconfigRootDir": "cypress"
   },
   "rules": {
-    "no-only-tests/no-only-tests": "error"
+    "no-only-tests/no-only-tests": "error",
+    "cypress/unsafe-to-chain-command": "off"
   }
 }