Move plugins to root (ToolJet#1728)

* feat: move plugins to root * modify tsconfig * add .gitignore * delete old plugins file * add parcel * docker compose volume mount * add gcs * add typescript to plugins folder * gcs to ts * add dynamodb * add elastic search * add firestore * add gsheets * add graphql * add mongodb * mssql * add mysql * add postgresql * add redis * add s3 * add slack * add stripe * remove plugin related packages from pkgjson * add lib folder * add gitignore * remove typescript generated files * remove generated file * remove generated files * add twilio * add dist to docker compose cache binding * add dist prefix * cleanup - 1 * delete dist * rename to index.ts + add jest config * add it.todo in tests * test fixes * test file changes * fix type checks * add @tooljet/plugins to server package json * esm vs commonjs bug, reduce got to 11.8.2 from 12.0.0 * docker file npm package version fix * add typesense * cleaup - 2 * add sendgrid * add lerna build and clean script for all packages + tsconfig * cleanup -3 * add plugins build step * add missing plugins build step in npm run build * add mssql, mysql & postgres as singleton classes * add db connection to cache only if datasourceId is available * client: add data source schema/manifest files * add query operations files * logic for wrapping form with schema * add script to create index file * add @tooljet/plugins to frontend folder * cleanup 1 -frontend * cleanup - 2 // frontend // data queries * add client and index to gitignore * update gitignore * fix lint & test * update ci * fix unit, e2e * cleanup -3 * fix test * fix tests * fix indent * try npm ci * fix tests * fix typo * fix * rename file for server entry * heroku fix * add main and types entry points in pkg json * move common to root * cleanup - 4: remove redundant $ sign prefix * cleanup - 4: remove redundant $ sign prefix * update options in-sync before DOM is painted * change type cloud to cloud storage * update readme * update ci.yml * update ci yml * add pkg-lock.json * rename index.ts to server.ts * update lock files * add server package.lock * remove unused import * revert commit: add minio * add root dep * import server.ts * remove plugins build step * add npm shrinkwrap * update version - plugins * add new version - 0.0.8 * upgrade version * move to symlinked package * add lock file * feat: add icon inside package * add plugin creation docs * Remove seed * move icons to plugins folder * install pg dep * add react to packages * add seed cmd * revert change * add plugins build in lint, e2e, unit * e2e, lint use npm ci * update dockerfile for plugins * try combining release with web * limit memory on release * try executing seed script post transpile * try executing seed from server directory * update seed execution * add minio * add correct type * add minio to pkg json * remove old file * fix provider key * add python installable + npm ^7.2.0 (ToolJet#1752) * add python installable + npm ^7.2.0 * add py to prod file * pin npm version to 7.20.0 * pin npm version to 7.20.0 * split into multi stage build and remove python for buildx * copy plugins from buider stage * update dependencies * add freetds dependency * update server dockerfile * update client dockerfile * update dev dockerfile and compose file * fix entrypoint * fix server dev dockerfile * update docker-compose * remove npm install on root dir on docker build * fix heroku script * make lerna prod dependency to enable prod builds * remove redundant env setup Co-authored-by: Akshay Sasidharan <akshaysasidharan93@gmail.com> Co-authored-by: navaneeth <navaneethpk@outlook.com>
krishnam1998gkp · Jan 17, 2022 · 2f3b441 · 2f3b441
1 parent 077c297
commit 2f3b441
Show file tree

Hide file tree

Showing 276 changed files with 46,956 additions and 19,264 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -83,6 +83,7 @@ jobs:
             ${{ runner.os }}-
 
       - run: npm i -g npm@7.20.0
+      - run: npm run build:plugins
       - run: npm --prefix frontend ci && npm --prefix server ci
       - run: npm --prefix server run lint && npm --prefix frontend run lint
 
@@ -117,6 +118,7 @@ jobs:
             ${{ runner.os }}-
       - run: apt update && apt install -y postgresql
       - run: npm i -g npm@7.20.0
+      - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server
       - run: npm --prefix server ci
       - run: npm --prefix server run db:create
       - run: npm --prefix server run db:migrate
@@ -153,6 +155,7 @@ jobs:
             ${{ runner.os }}-
       - run: apt update && apt install -y postgresql
       - run: npm i -g npm@7.20.0
+      - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server
       - run: npm --prefix server ci
       - run: npm --prefix server run db:create
       - run: npm --prefix server run db:migrate

diff --git a/Procfile b/Procfile
@@ -1,2 +1,2 @@
 web: npm run start:prod --prefix server
-release: npm run db:migrate --prefix server && npm run db:seed --prefix server
+release: (export NODE_OPTIONS="--max_old_space_size=1024"; cd server && npm run db:migrate && node dist/scripts/seeds.js)
diff --git a/deploy/ec2/setup_machine.sh b/deploy/ec2/setup_machine.sh
@@ -51,6 +51,8 @@ mv /tmp/.env ~/app/.env
 mv /tmp/setup_app ~/app/setup_app
 sudo chmod +x ~/app/setup_app
 
+sudo npm install -g npm@7.20.0
+
 # Building ToolJet app
 sudo npm install -g @nestjs/cli
 sudo npm run build
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,37 +1,43 @@
-version: '3'
+version: "3"
 
 services:
   client:
     build:
-      context: ./frontend
-      dockerfile: ../docker/client.Dockerfile.dev
+      context: ./
+      dockerfile: ./docker/client.Dockerfile.dev
     image: tooljet-client:development
+    platform: linux/x86_64
     volumes:
-      - ./frontend:/app:delegated
-      - /app/node_modules/
+      - ./frontend:/app/frontend:delegated
+      - ./plugins/:/app/plugins:cached
+      - /app/frontend/node_modules/
     ports:
       - 8082:8082
     environment:
       - WEBPACKER_DEV_SERVER_HOST=0.0.0.0
       - NODE_ENV=development
-    command: ['npm', 'start']
+    command: npm run --prefix frontend start
 
   server:
     build:
-      context: .
+      context: ./
       dockerfile: ./docker/server.Dockerfile.dev
     image: tooljet-server:development
+    platform: linux/x86_64
     depends_on:
       - postgres
     volumes:
-      - ./server:/app:delegated
-      - /app/node_modules/
+      - ./server:/app/server:delegated
+      - ./plugins/:/app/plugins:cached
+      - /app/server/node_modules/
+      - '.env:/app/.env'
+      - '.env.test:/app/.env.test'
     ports:
       - 3000:3000
     environment:
       - SERVE_CLIENT=false
       - FORCE_COLOR=1
-    command: npm run start:dev
+    command: npm run --prefix server start:dev
 
   postgres:
     image: postgres:13

diff --git a/docker/client.Dockerfile b/docker/client.Dockerfile
@@ -1,22 +1,28 @@
 # pull official base image
 FROM node:14.17.3-alpine AS builder
-ENV NODE_ENV=production
+
+RUN npm i -g npm@7.20.0
 
 # set working directory
 WORKDIR /app
 
-# add `/app/node_modules/.bin` to $PATH
-ENV PATH /app/node_modules/.bin:$PATH
+COPY ./package.json ./package.json
 
 # Fix for heap limit allocation issue
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-# install app dependencies
-COPY ./frontend/package.json ./frontend/package-lock.json  ./
-RUN npm install --only=production
-COPY ./frontend .
-RUN NODE_ENV=production npm run-script build
+# Build plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/
+RUN npm run build:plugins
 
+# Build frontend
+ENV NODE_ENV=production
+COPY ./frontend/package.json ./frontend/package-lock.json  ./frontend/
+RUN npm --prefix frontend install --only=production
+COPY ./frontend ./frontend
+RUN npm --prefix frontend run build
 
 FROM openresty/openresty:1.19.9.1rc1-buster-fat
 
@@ -28,7 +34,7 @@ RUN luarocks install lua-resty-auto-ssl
 
 RUN mkdir /etc/resty-auto-ssl /var/log/openresty /var/www /etc/fallback-certs
 
-COPY --from=builder /app/build /var/www
+COPY --from=builder /app/frontend/build /var/www
 
 COPY ./frontend/config/nginx.conf.template /etc/openresty/nginx.conf.template
 COPY ./frontend/config/entrypoint.sh /entrypoint.sh

diff --git a/docker/client.Dockerfile.dev b/docker/client.Dockerfile.dev
@@ -3,25 +3,33 @@ FROM node:14.17.3-alpine
 
 ENV NODE_ENV=development
 
+RUN npm i -g npm@7.20.0
+
 # set working directory
 WORKDIR /app
 
+COPY ./package.json ./package.json
+
 # add `/app/node_modules/.bin` to $PATH
 ENV PATH /app/node_modules/.bin:$PATH
 
 # Fix for heap limit allocation issue
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
+# Build plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/
+RUN npm run build:plugins
 
 # install app dependencies
-COPY package.json package-lock.json  ./
-RUN npm install
-RUN npm install react-scripts@3.4.1 -g --silent
+COPY ./frontend/package.json ./frontend/package-lock.json ./frontend/
+RUN npm --prefix frontend install
+COPY ./frontend/ ./frontend/
 
-# add app
-COPY . ./
+RUN npm install react-scripts@3.4.1 -g --silent
 
 # start app
-CMD ["npm", "start"]
+CMD ["npm", "--prefix", "frontend", "start"]
 
 EXPOSE 8082
diff --git a/docker/production.Dockerfile b/docker/production.Dockerfile
@@ -1,29 +1,59 @@
-FROM node:14.17.3-buster
+FROM node:14.17.3-alpine AS builder
 
 # Fix for JS heap limit allocation issue
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-RUN apt update && apt install -y \
-  build-essential  \
-  postgresql \
-  freetds-dev
-
+RUN npm i -g npm@7.20.0
 RUN mkdir -p /app
+
 WORKDIR /app
+
+# Scripts for building
+COPY ./package.json ./package.json
+
+# Build plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/
+RUN npm run build:plugins
+
 ENV NODE_ENV=production
-COPY ./package.json ./package-lock.json ./
 
-# Building ToolJet client
+# Build frontend
 COPY ./frontend/package.json ./frontend/package-lock.json ./frontend/
 RUN npm --prefix frontend install
 COPY ./frontend/ ./frontend/
-RUN NODE_ENV=production npm --prefix frontend run build
+RUN npm --prefix frontend run build
 
-# Building ToolJet server
+# Build server
 COPY ./server/package.json ./server/package-lock.json ./server/
 RUN npm --prefix server install
 COPY ./server/ ./server/
 RUN npm install -g @nestjs/cli
 RUN npm --prefix server run build
 
+FROM node:14.17.3-alpine
+
+ENV NODE_ENV=production
+ENV NODE_OPTIONS="--max-old-space-size=4096"
+RUN apk add postgresql-client freetds
+RUN mkdir -p /app
+
+# copy npm scripts
+COPY --from=builder /app/package.json ./app/package.json
+# copy plugins dependencies
+COPY --from=builder /app/plugins/dist ./app/plugins/dist
+COPY --from=builder /app/plugins/client.js ./app/plugins/client.js
+COPY --from=builder /app/plugins/node_modules ./app/plugins/node_modules
+COPY --from=builder /app/plugins/packages/common ./app/plugins/packages/common
+COPY --from=builder /app/plugins/package.json ./app/plugins/package.json
+# copy frontend build
+COPY --from=builder /app/frontend/build ./app/frontend/build
+# copy server build
+# NOTE: typescript dependency on /server/scripts and typeorm for db creation and migration.
+# Need to check if we can optimize such that only executable dist from prev stage can be copied
+COPY --from=builder /app/server ./app/server
+
+WORKDIR /app
+
 ENTRYPOINT ["./server/entrypoint.sh"]
diff --git a/docker/server.Dockerfile b/docker/server.Dockerfile
@@ -1,24 +1,48 @@
-FROM node:14.17.3-buster
-
-ENV NODE_ENV=production
+FROM node:14.17.3-alpine as builder
 
 # Fix for JS heap limit allocation issue
 ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-RUN apt update && apt install -y \
-  build-essential  \
-  postgresql \
-  freetds-dev
-
+RUN npm i -g npm@7.20.0
 RUN npm install -g @nestjs/cli
 
 RUN mkdir -p /app
 WORKDIR /app
 
+COPY ./package.json ./package.json
+
+# Building ToolJet plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/
+RUN npm run build:plugins
+
+ENV NODE_ENV=production
 # Building ToolJet server
-COPY ./server/package.json ./server/package-lock.json ./
-RUN npm install --only=production
-COPY ./server/ ./
-RUN npm run build
+COPY ./server/package.json ./server/package-lock.json ./server/
+RUN npm --prefix server install --only=production
+COPY ./server/ ./server/
+RUN npm --prefix server run build
+
+FROM node:14.17.3-alpine
+
+ENV NODE_ENV=production
+ENV NODE_OPTIONS="--max-old-space-size=4096"
+RUN apk add postgresql-client freetds
+
+RUN mkdir -p /app
+
+# copy plugins dependencies
+COPY --from=builder /app/plugins/dist ./app/plugins/dist
+COPY --from=builder /app/plugins/client.js ./app/plugins/client.js
+COPY --from=builder /app/plugins/node_modules ./app/plugins/node_modules
+COPY --from=builder /app/plugins/packages/common ./app/plugins/packages/common
+COPY --from=builder /app/plugins/package.json ./app/plugins/package.json
+# copy server build
+# NOTE: typescript dependency on /server/scripts and typeorm for db creation and migration.
+# Need to check if we can optimize such that only executable dist from prev stage can be copied
+COPY --from=builder /app/server ./app/server
+
+WORKDIR /app
 
-ENTRYPOINT ["./entrypoint.sh"]
+ENTRYPOINT ["./server/entrypoint.sh"]
diff --git a/docker/server.Dockerfile.dev b/docker/server.Dockerfile.dev
@@ -1,21 +1,25 @@
 # pull official base image
-FROM node:14.17.3-buster
+FROM node:14.17.3-alpine
+RUN apk add postgresql-client freetds
 
 ENV NODE_ENV=development
+ENV NODE_OPTIONS="--max-old-space-size=4096"
 
-RUN apt update && apt install -y \
-  build-essential  \
-  postgresql \
-  freetds-dev
-
+RUN npm i -g npm@7.20.0
 RUN mkdir -p /app
 WORKDIR /app
 
-COPY ./server/package.json ./server/package-lock.json ./
-RUN npm install
-COPY ./server/ ./
+COPY ./package.json ./package.json
+
+# Build plugins
+COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/
+RUN npm --prefix plugins install
+COPY ./plugins/ ./plugins/
+RUN npm run build:plugins
 
-COPY ./.env ../.env
-COPY ./.env.test ../.env.test
+# install app dependencies
+COPY ./server/package.json ./server/package-lock.json ./server/
+RUN npm --prefix server install
+COPY ./server/ ./server/
 
-ENTRYPOINT ["./entrypoint.sh"]
+ENTRYPOINT ["./server/entrypoint.sh"]