Chore: ToolJet setup to try locally (ToolJet#3158)

* add dockerfile for try tooljet setup

* update npm scripts and logged messages

* refactor npm scripts for dist folder scripts

* update usages of db setup and seed

* setup script to init db and boot app

* fix dockerfile

* update dockerfile

* user production.Dockerfile as base image

* fix script

* fix script exit

* fix lint

deploy/ec2/setup_app

@@ -24,9 +24,8 @@ else
   exit 1
 fi
 
-sudo npm --prefix server run db:create
-sudo npm --prefix server run db:migrate
-sudo npm --prefix server run db:seed
+sudo npm --prefix server run db:setup:prod
+sudo npm --prefix server run db:seed:prod
 
 if sudo systemctl start nest
 then

docker/try-tooljet.Dockerfile

@@ -0,0 +1,24 @@
+FROM tooljet/tooljet-ce:latest
+
+# Install Postgres
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN apt update && apt -y install postgresql-13 postgresql-client-13
+USER postgres
+RUN service postgresql start && \
+    psql -c "create role tooljet with login superuser password 'postgres';"
+
+# ENV defaults
+ENV TOOLJET_HOST=http://localhost:3000 \
+    LOCKBOX_MASTER_KEY=replace_with_lockbox_master_key \
+    SECRET_KEY_BASE=replace_with_secret_key_base \
+    PG_DB=tooljet_production \
+    PG_USER=tooljet \
+    PG_PASS=postgres \
+    PG_HOST=localhost \
+    ORM_LOGGING=true \
+    DEPLOYMENT_PLATFORM=docker:local \
+    TERM=xterm
+
+# Prepare DB and start application
+ENTRYPOINT service postgresql start 1> /dev/null && bash /app/server/scripts/init-db-boot.sh

package.json

@@ -52,8 +52,13 @@
     "build": "npm run build:plugins:prod && npm run build:frontend && npm run build:server",
     "start:prod": "npm --prefix server run start:prod",
     "db:create": "npm --prefix server run db:create",
+    "db:create:prod": "npm --prefix server run db:create:prod",
     "db:migrate": "npm --prefix server run db:migrate",
+    "db:migrate:prod": "npm --prefix server run db:migrate:prod",
     "db:seed": "npm --prefix server run db:seed",
+    "db:seed:prod": "npm --prefix server run db:seed:prod",
+    "db:setup": "npm --prefix server run db:setup",
+    "db:setup:prod": "npm --prefix server run db:setup:prod",
     "db:reset": "npm --prefix server run db:reset",
     "db:drop": "npm --prefix server run db:drop",
     "deploy": "cp -a frontend/build/. public/",
@@ -62,4 +67,4 @@
     "cy:open": "cypress open --env db.name=$TEST_PG_DB,db.user=$TEST_PG_USERNAME,db.password=$TEST_PG_PASSWORD",
     "prepare": "husky install"
   }
-}
+}

server/entrypoint.sh

@@ -1,7 +1,11 @@
 #!/bin/sh
 set -e
 
-npm run db:create
-npm run db:migrate
+if [ -d "./server/dist" ]
+then
+    npm run db:setup:prod
+else
+    npm run db:setup
+fi
 
 exec "$@"

server/ormconfig.ts

@@ -57,19 +57,9 @@ function determineFilePathForEnv(env: string | undefined): string {
   }
 }
 
-function throwErrorIfFileNotPresent(filePath: string, env: string): void {
-  if (!fs.existsSync(filePath)) {
-    console.log(
-      `Unable to fetch database config from env file for environment: ${env}\n` +
-        'Picking up config from the environment'
-    );
-  }
-}
-
 function fetchConnectionOptions(): TypeOrmModuleOptions {
   const env: string | undefined = process.env.NODE_ENV;
   const filePath: string = determineFilePathForEnv(env);
-  throwErrorIfFileNotPresent(filePath, env);
 
   return buildConnectionOptions(filePath, env);
 }

server/package.json

@@ -20,10 +20,15 @@
     "test:debug": "NODE_ENV=test node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "NODE_ENV=test jest --runInBand --config ./test/jest-e2e.json --detectOpenHandles",
     "db:create": "ts-node ./scripts/create-database.ts",
+    "db:create:prod": "node dist/scripts/create-database.js ",
     "db:drop": "ts-node ./scripts/drop-database.ts",
     "db:migrate": "ts-node -r tsconfig-paths/register --transpile-only ./node_modules/typeorm/cli.js migration:run",
+    "db:migrate:prod": "node ./node_modules/typeorm/cli.js migration:run --config dist/ormconfig.js 1> /dev/null ",
     "db:seed": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/seeds.ts",
-    "db:reset": "npm run db:drop && npm run db:create && npm run db:migrate",
+    "db:seed:prod": "node dist/scripts/seeds.js",
+    "db:setup": "npm run db:create && npm run db:migrate",
+    "db:setup:prod": "(export ORM_LOGGING=false; npm run db:create:prod && npm run db:migrate:prod)",
+    "db:reset": "npm run db:drop && npm run db:setup",
     "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js --config ormconfig.ts"
   },
   "dependencies": {
@@ -109,4 +114,4 @@
     "node": ">=14.17.3",
     "npm": "<=7.20.0"
   }
-}
+}

server/scripts/create-database.ts

@@ -21,33 +21,36 @@ function createDatabase(): void {
     throw new Error(`Config validation error: ${error.message}`);
   }
 
-  exec('command -v createdb', (err, _stdout, _stderr) => {
-    if (err) {
-      console.error(err);
-      return;
+  const connectivityCheck = exec('command -v createdb');
+
+  connectivityCheck.on('exit', function (signal) {
+    if (signal === 1) {
+      console.error('Unable to connect to database');
+      process.exit(1);
     }
+  });
 
-    const createdb =
-      `PGPASSWORD=${envVars.PG_PASS} createdb ` +
-      `-h ${envVars.PG_HOST} ` +
-      `-p ${envVars.PG_PORT} ` +
-      `-U ${envVars.PG_USER} ` +
-      process.env.PG_DB;
+  const createdb =
+    `PGPASSWORD=${envVars.PG_PASS} createdb ` +
+    `-h ${envVars.PG_HOST} ` +
+    `-p ${envVars.PG_PORT} ` +
+    `-U ${envVars.PG_USER} ` +
+    process.env.PG_DB;
 
-    exec(createdb, (err, _stdout, _stderr) => {
-      if (!err) {
-        console.log(`Created database ${envVars.PG_DB}`);
-        return;
-      }
+  exec(createdb, (err, _stdout, _stderr) => {
+    if (!err) {
+      console.log(`Created database ${envVars.PG_DB}`);
+      return;
+    }
 
-      const errorMessage = `database "${envVars.PG_DB}" already exists`;
+    const errorMessage = `database "${envVars.PG_DB}" already exists`;
 
-      if (err.message.includes(errorMessage)) {
-        console.log(errorMessage);
-      } else {
-        console.error(err);
-      }
-    });
+    if (err.message.includes(errorMessage)) {
+      console.log(`Using database: ${envVars.PG_DB}`);
+    } else {
+      console.error(err);
+      process.exit(1);
+    }
   });
 }
 
@@ -60,7 +63,7 @@ if (fs.existsSync(nodeEnvPath)) {
 } else if (fs.existsSync(fallbackPath)) {
   createDatabaseFromFile(fallbackPath);
 } else {
-  console.log(`${nodeEnvPath} file not found to create database\n` + 'Picking up config from the environment');
+  console.log('Picking up config from the environment');
 
   createDatabase();
 }

server/scripts/drop-database.ts

@@ -21,33 +21,35 @@ function dropDatabase(): void {
     throw new Error(`Config validation error: ${error.message}`);
   }
 
-  exec('command -v dropdb', (err, _stdout, _stderr) => {
-    if (err) {
-      console.error(err);
-      return;
+  const connectivityCheck = exec('command -v createdb');
+  connectivityCheck.on('exit', function (signal) {
+    if (signal === 1) {
+      console.error('Unable to connect to database');
+      process.exit(1);
     }
+  });
 
-    const dropdb =
-      `PGPASSWORD=${envVars.PG_PASS} dropdb ` +
-      `-h ${envVars.PG_HOST} ` +
-      `-p ${envVars.PG_PORT} ` +
-      `-U ${envVars.PG_USER} ` +
-      process.env.PG_DB;
+  const dropdb =
+    `PGPASSWORD=${envVars.PG_PASS} dropdb ` +
+    `-h ${envVars.PG_HOST} ` +
+    `-p ${envVars.PG_PORT} ` +
+    `-U ${envVars.PG_USER} ` +
+    process.env.PG_DB;
 
-    exec(dropdb, (err, _stdout, _stderr) => {
-      if (!err) {
-        console.log(`Dropped database ${envVars.PG_DB}`);
-        return;
-      }
+  exec(dropdb, (err, _stdout, _stderr) => {
+    if (!err) {
+      console.log(`Dropped database ${envVars.PG_DB}`);
+      return;
+    }
 
-      const errorMessage = `database "${envVars.PG_DB}" does not exist`;
+    const errorMessage = `database "${envVars.PG_DB}" does not exist`;
 
-      if (err.message.includes(errorMessage)) {
-        console.log(errorMessage);
-      } else {
-        console.error(err);
-      }
-    });
+    if (err.message.includes(errorMessage)) {
+      console.log(errorMessage);
+    } else {
+      console.error(err);
+      process.exit(1);
+    }
   });
 }
 

server/scripts/init-db-boot.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+echo "Initializing database.."
+echo "This may take a couple of minutes"
+echo -ne "                          (0%)\r"
+npm run db:create:prod --silent 1> /dev/null
+echo -ne "#####                     (33%)\r"
+npm run db:migrate:prod --silent 1> /dev/null
+echo -ne "#############             (66%)\r"
+npm run db:seed:prod --silent
+echo -ne "#######################   (100%)\r"
+echo -ne "\n\n"
+
+
+echo "
+   _____           _   ___      _
+  |_   _|         | | |_  |    | |
+    | | ___   ___ | |   | | ___| |_
+    | |/ _ \ / _ \| |   | |/ _ \ __|
+    | | (_) | (_) | /\__/ /  __/ |_
+    \_/\___/ \___/|_\____/ \___|\__|
+
+Everything you need to build internal tools!
+GitHub: https://github.com/ToolJet/ToolJet
+"
+
+npm run start:prod --silent

server/src/app.module.ts

@@ -1,4 +1,4 @@
-import { Module, OnApplicationBootstrap, OnModuleInit, RequestMethod, MiddlewareConsumer } from '@nestjs/common';
+import { Module, OnModuleInit, RequestMethod, MiddlewareConsumer } from '@nestjs/common';
 
 import { Connection } from 'typeorm';
 import { TypeOrmModule } from '@nestjs/typeorm';
@@ -107,7 +107,7 @@ if (process.env.COMMENT_FEATURE_ENABLE !== 'false') {
   controllers: [AppController],
   providers: [EmailService, SeedsService],
 })
-export class AppModule implements OnModuleInit, OnApplicationBootstrap {
+export class AppModule implements OnModuleInit {
   constructor(private connection: Connection) {}
 
   configure(consumer: MiddlewareConsumer): void {
@@ -118,10 +118,7 @@ export class AppModule implements OnModuleInit, OnApplicationBootstrap {
   }
 
   onModuleInit(): void {
-    console.log(`Initializing ToolJet server modules 📡 `);
-  }
-
-  onApplicationBootstrap(): void {
-    console.log(`Initialized ToolJet server, waiting for requests 🚀`);
+    console.log(`Version: ${globalThis.TOOLJET_VERSION}`);
+    console.log(`Initializing server modules 📡 `);
   }
 }

server/src/main.ts

@@ -6,6 +6,7 @@ import { Logger } from 'nestjs-pino';
 import { urlencoded, json } from 'express';
 import { AllExceptionsFilter } from './all-exceptions-filter';
 import { ValidationPipe } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
 
 const fs = require('fs');
 
@@ -16,6 +17,7 @@ async function bootstrap() {
     bufferLogs: true,
     abortOnError: false,
   });
+  const configService = app.get<ConfigService>(ConfigService);
   const host = new URL(process.env.TOOLJET_HOST);
   const domain = host.hostname;
 
@@ -66,7 +68,8 @@ async function bootstrap() {
   const port = parseInt(process.env.PORT) || 3000;
 
   await app.listen(port, '0.0.0.0', function () {
-    console.log('Listening on port %d', port);
+    const tooljetHost = configService.get<string>('TOOLJET_HOST');
+    console.log(`Ready to use at ${tooljetHost} 🚀`);
   });
 }
 

server/src/services/seeds.service.ts

@@ -18,10 +18,7 @@ export class SeedsService {
         },
       });
 
-      if (defaultUser) {
-        console.log('Default user already present. Skipping seed.');
-        return;
-      }
+      if (defaultUser) return;
 
       const organization = manager.create(Organization, {
         ssoConfigs: [
@@ -58,7 +55,9 @@ export class SeedsService {
 
       await this.createDefaultUserGroups(manager, user);
 
-      console.log('Seeding complete. Use default credentials to login.');
+      console.log(
+        'Seeding complete. Use default credentials to login.\n' + 'email: dev@tooljet.io\n' + 'password: password'
+      );
     });
   }