Chore: Docker compose plugins dev (ToolJet#1953)

* add plugins to docker compose

* build common

* change cmd

* build:pkgs

* fix'

* temp: add named vol

* update docker-compose for plugins dev

* update seed script to transpile only

* fix ci

* fix ci

Co-authored-by: gandharv <gandharvkumargarg@gmail.com>

.github/workflows/ci.yml

@@ -123,7 +123,9 @@ jobs:
             ${{ runner.os }}-
       - run: apt update && apt install -y postgresql
       - run: npm i -g npm@7.20.0
-      - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server
+      - run: npm --prefix plugins ci
+      - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server
+      - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server
       - run: npm --prefix server ci
       - run: npm --prefix server run db:create
       - run: npm --prefix server run db:migrate
@@ -160,7 +162,9 @@ jobs:
             ${{ runner.os }}-
       - run: apt update && apt install -y postgresql
       - run: npm i -g npm@7.20.0
-      - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server
+      - run: npm --prefix plugins ci
+      - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server
+      - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server
       - run: npm --prefix server ci
       - run: npm --prefix server run db:create
       - run: npm --prefix server run db:migrate

docker-compose.yaml

@@ -1,6 +1,16 @@
 version: "3"
 
 services:
+  plugins:
+    build:
+      context: ./
+      dockerfile: ./docker/plugins.Dockerfile.dev
+    image: tooljet-plugins:development
+    platform: linux/x86_64
+    volumes:
+      - plugins_vol:/app/plugins
+    command: npm run --prefix plugins start
+
   client:
     build:
       context: ./
@@ -9,9 +19,7 @@ services:
     platform: linux/x86_64
     volumes:
       - ./frontend:/app/frontend:delegated
-      - ./plugins/:/app/plugins
-      - /app/plugins/dist/
-      - /app/plugins/node_modules/
+      - plugins_vol:/app/plugins
       - /app/frontend/node_modules/
     ports:
       - 8082:8082
@@ -30,12 +38,10 @@ services:
       - postgres
     volumes:
       - ./server:/app/server:delegated
-      - ./plugins/:/app/plugins
-      - /app/plugins/dist/
-      - /app/plugins/node_modules/
+      - plugins_vol:/app/plugins
       - /app/server/node_modules/
-      - ".env:/app/.env"
-      - ".env.test:/app/.env.test"
+      - ./.env:/app/.env
+      - ./.env.test:/app/.env.test
     ports:
       - 3000:3000
     environment:
@@ -55,3 +61,8 @@ services:
 
 volumes:
   postgres:
+  plugins_vol:
+    driver_opts:
+      type: none
+      device: ${PWD}/plugins
+      o: bind

docker/client.Dockerfile.dev

@@ -16,12 +16,6 @@ ENV PATH /app/node_modules/.bin:$PATH
 # Fix for heap limit allocation issue
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-# Build plugins
-COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
-RUN npm --prefix plugins install
-COPY ./plugins/ ./plugins/
-RUN npm run build:plugins
-
 # install app dependencies
 COPY ./frontend/package.json ./frontend/package-lock.json ./frontend/
 RUN npm --prefix frontend install

docker/plugins.Dockerfile.dev

@@ -0,0 +1,17 @@
+# pull official base image
+FROM node:14.17.3-alpine AS builder
+
+RUN npm i -g npm@7.20.0
+
+# set working directory
+WORKDIR /app
+
+COPY ./package.json ./package.json
+
+# Fix for heap limit allocation issue
+ENV NODE_OPTIONS="--max-old-space-size=4096"
+
+# Build plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/

docker/server.Dockerfile.dev

@@ -11,12 +11,6 @@ WORKDIR /app
 
 COPY ./package.json ./package.json
 
-# Build plugins
-COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
-RUN npm --prefix plugins install
-COPY ./plugins/ ./plugins/
-RUN npm run build:plugins
-
 # install app dependencies
 COPY ./server/package.json ./server/package-lock.json ./server/
 RUN npm --prefix server install

docs/docs/contributing-guide/setup/docker.md

@@ -80,6 +80,7 @@ docker-compose version 1.26.2, build eefe0d31
 
    ```bash
    docker-compose build
+   docker-compose run --rm  plugins npm run build:plugins
    ```
 
 5. Run ToolJet
@@ -91,7 +92,7 @@ docker-compose version 1.26.2, build eefe0d31
 6. ToolJet server is built using NestJS and the data such as application definitions are persisted on a postgres database. You can run the below command to seed the database.
 
    ```bash
-   docker-compose run --rm server npm run db:seed
+   docker-compose exec server npm run db:seed
    ```
 
 7. ToolJet should now be served locally at `http://localhost:8082`. You can login using the default user created.

package.json

@@ -32,7 +32,7 @@
   },
   "scripts": {
     "prebuild:plugins": "npm --prefix plugins install",
-    "build:plugins": "npm --prefix plugins run build:packages && npm --prefix plugins run build:server",
+    "build:plugins": "npm --prefix plugins run build",
     "prebuild:frontend": "npm --prefix frontend install",
     "build:frontend": "NODE_ENV=production npm --prefix frontend run build",
     "prebuild:server": "npm --prefix server install",

plugins/package.json

@@ -5,12 +5,12 @@
   "types": "dist/server.d.ts",
   "main": "dist/server.js",
   "scripts": {
-    "clean": "rimraf dist/",
     "create:client": "node create-client-entry",
     "create:server": "node create-server-entry",
-    "prebuild:server": "npm run clean",
-    "build:server": "tsc -b",
-    "prebuild:packages": "npm run clean:packages && npm run create:client && npm run create:server",
+    "build:server": "rimraf dist/ && tsc -b",
+    "start": "tsc -b --watch",
+    "build": "npm run create:client && npm run create:server && npm run build:packages && npm run build:server",
+    "prebuild:packages": "npm run clean:packages",
     "build:packages": "npx lerna run build --stream",
     "clean:packages": "npx lerna run clean --parallel"
   },

plugins/tsconfig.json

@@ -18,5 +18,5 @@
     "composite": true,
     "resolveJsonModule": true
   },
-  "exclude": ["packages/*/lib/*.json", "packages/*/__tests__/*"]
+  "exclude": ["packages/*/lib/*.json", "packages/*/__tests__/*", "dist"]
 }

server/package.json

@@ -21,8 +21,8 @@
     "test:e2e": "NODE_ENV=test jest --runInBand --config ./test/jest-e2e.json --detectOpenHandles",
     "db:create": "ts-node ./scripts/create-database.ts",
     "db:drop": "ts-node ./scripts/drop-database.ts",
-    "db:migrate": "npm run typeorm migration:run",
-    "db:seed": "ts-node -r tsconfig-paths/register ./scripts/seeds.ts",
+    "db:migrate": "ts-node -r tsconfig-paths/register --transpile-only ./node_modules/typeorm/cli.js migration:run",
+    "db:seed": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/seeds.ts",
     "db:reset": "npm run db:drop && npm run db:create && npm run db:migrate",
     "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js --config ormconfig.ts"
   },