[td] Refactor Tornado server (mage-ai#449)

* [td] Refactor Tornado server

* add to docker compose

* update readme

* use forked jupyter client

* update readme

* made docker ready

Dockerfile

@@ -1,38 +1,28 @@
-FROM jupyter/minimal-notebook
+FROM python
 
 LABEL description="Mage data management platform"
 
 ARG PIP=pip3
 
 USER root
 
-# Install some handful libraries like curl, wget, git, build-essential, zlib
-RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y && \
-    apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        ca-certificates \
-        curl \
-        wget \
-        git \
-        libopencv-dev \
-        openssh-client \
-        openssh-server \
-        vim \
-        zlib1g-dev \
-        graphviz \
-        yarn
+RUN apt update && apt install curl
+RUN curl -fsSL https://deb.nodesource.com/setup_17.x | bash -
+RUN apt install nodejs
 
 # Install Python dependencies
 COPY requirements.txt requirements.txt
 RUN ${PIP} install -r requirements.txt
 
-COPY . /home/jovyan/src
-
-# Install Node version 17
-RUN echo -e 'y' | conda install nodejs==17.9.0
+COPY ./mage_ai /home/src/mage_ai
 
 # Install node modules used in front-end
-RUN cd /home/jovyan/src/mage_ai/frontend && npm install -g npm@8.12.2
+RUN npm install --global yarn
+RUN yarn global add next
+RUN cd /home/src/mage_ai/frontend && yarn install
+
+ENV PYTHONPATH="${PYTHONPATH}:/home/src"
+
+RUN ${PIP} install jupyterlab
 
-ENV PYTHONPATH="${PYTHONPATH}:/home/jovyan/src"
+WORKDIR /home/src

docker-compose.yml

@@ -1,39 +1,69 @@
 version: '3'
 services:
-  lab:
-    image: mage/lab
+  server_kernel:
+    image: mage/data
     build:
       context: .
       dockerfile: ./Dockerfile
+    command: "python mage_ai/server/server.py"
+    ports:
+      - 6789:6789
+    volumes:
+      - .:/home/src
+    stdin_open: true # used for interactive debugging
+    tty: true # used for interactive debugging
+  lab:
+    image: mage/data
+    depends_on:
+      - server_kernel
     environment:
-      - HOME_DIR=/home/jovyan
-    command: "jupyter lab --allow-root"
+      - HOME_DIR=/home/src
+    command: "jupyter lab --allow-root --ip=0.0.0.0"
     ports:
       - 8888:8888
     volumes:
-      - .:/home/jovyan
+      - .:/home/src
     mem_limit: 99999m
     oom_kill_disable: true
   server:
-    image: mage/lab
+    image: mage/data
     depends_on:
-      - lab
+      - server_kernel
     environment:
       - FLASK_ENV=development
-    command: "python3 mage_ai/server/app.py --host 0.0.0.0 --port 5789"
+    command: "python mage_ai/server/app.py --host 0.0.0.0 --port 5789"
     ports:
       - 5789:5789
     volumes:
-      - .:/home/jovyan
+      - .:/home/src
     stdin_open: true # used for interactive debugging
     tty: true # used for interactive debugging
   app:
-    image: mage/lab
+    image: mage/data
     depends_on:
       - server
-    command: "npm run dev"
+    command: "yarn run dev"
+    environment:
+      - NODE_PATH=/home/src/node_modules
     ports:
       - 3000:3000
     volumes:
-      - .:/home/jovyan
-    working_dir: /home/jovyan/mage_ai/frontend
+      - ./mage_ai/frontend/.babelrc:/home/src/mage_ai/frontend/.babelrc
+      - ./mage_ai/frontend/.eslintrc.js:/home/src/mage_ai/frontend/.eslintrc.js
+      - ./mage_ai/frontend/api:/home/src/mage_ai/frontend/api
+      - ./mage_ai/frontend/components:/home/src/mage_ai/frontend/components
+      - ./mage_ai/frontend/hocs:/home/src/mage_ai/frontend/hocs
+      - ./mage_ai/frontend/interfaces:/home/src/mage_ai/frontend/interfaces
+      - ./mage_ai/frontend/next-env.d.ts:/home/src/mage_ai/frontend/next-env.d.ts
+      - ./mage_ai/frontend/next.config.js:/home/src/mage_ai/frontend/next.config.js
+      - ./mage_ai/frontend/oracle:/home/src/mage_ai/frontend/oracle
+      - ./mage_ai/frontend/package.json:/home/src/mage_ai/frontend/package.json
+      - ./mage_ai/frontend/pages:/home/src/mage_ai/frontend/pages
+      - ./mage_ai/frontend/public:/home/src/mage_ai/frontend/public
+      - ./mage_ai/frontend/storage:/home/src/mage_ai/frontend/storage
+      - ./mage_ai/frontend/stories:/home/src/mage_ai/frontend/stories
+      - ./mage_ai/frontend/styles:/home/src/mage_ai/frontend/styles
+      - ./mage_ai/frontend/tsconfig.json:/home/src/mage_ai/frontend/tsconfig.json
+      - ./mage_ai/frontend/utils:/home/src/mage_ai/frontend/utils
+      - ./mage_ai/frontend/yarn.lock:/home/src/mage_ai/frontend/yarn.lock
+    working_dir: /home/src/mage_ai/frontend

docs/contributing/README.md

@@ -28,6 +28,9 @@ Jupyter Lab is running on [http://127.0.0.1:8888/lab](http://127.0.0.1:8888/lab)
 Look at your terminal and find a corresponding URL with a token in the URL parameter;
 e.g. `http://127.0.0.1:8888/lab?token=`.
 
+The backend server running Tornado and an iPython kernel is running
+at [http://localhost:6789/api/?value=test](http://localhost:6789/api/?value=test).
+
 #### Debugging
 
 Instead of using `breakpoint()`, add the following line to your code where you
@@ -139,6 +142,13 @@ Install Python packages
 $ pip3 install -r requirements.txt
 ```
 
+#### Run backend server with iPython kernel
+This server is used in the data preparation tool. It leverages ZMQ and WebSocket.
+
+```bash
+$ python3 mage_ai/server/server.py
+```
+
 #### Run backend server locally
 
 You can optionally set the host or port environment variables:

mage_ai/frontend/pages/test.tsx

@@ -47,8 +47,8 @@ function Test() {
   //   console.log(data.toString('utf-8'));
   // });
 
-  const socketUrlPublish = 'ws://localhost:8888/websocket/';
-  // const socketUrlSubscribe = 'ws://localhost:8888/websocket/subscribe/';
+  const socketUrlPublish = 'ws://localhost:6789/websocket/';
+  // const socketUrlSubscribe = 'ws://localhost:6789/websocket/subscribe/';
 
   const {
     sendMessage,