fix(studio): add db logger

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botpress/studio",
-  "version": "0.0.18",
+  "version": "0.0.19",
   "description": "Create your virtual assistants",
   "main": "index.js",
   "bin": "./packages/studio-be/out/index.js",

packages/studio-be/src/core/app/botpress.ts

@@ -3,7 +3,9 @@ import { BotService } from 'core/bots'
 import { GhostService } from 'core/bpfs'
 import { CMSService } from 'core/cms'
 import { BotpressConfig, ConfigProvider } from 'core/config'
+import Database from 'core/database'
 import { LoggerFilePersister, LoggerProvider } from 'core/logger'
+import { LoggerDbPersister } from 'core/logger/persister/db-persister'
 import { copyDir } from 'core/misc/pkg-fs'
 import { ModuleLoader } from 'core/modules'
 import { HintsService } from 'core/user-code'
@@ -36,6 +38,7 @@ export class Botpress {
 
   constructor(
     @inject(TYPES.ConfigProvider) private configProvider: ConfigProvider,
+    @inject(TYPES.Database) private database: Database,
     @inject(TYPES.Logger)
     @tagged('name', 'Server')
     private logger: sdk.Logger,
@@ -45,6 +48,7 @@ export class Botpress {
     @inject(TYPES.HintsService) private hintsService: HintsService,
     @inject(TYPES.CMSService) private cmsService: CMSService,
     @inject(TYPES.LoggerProvider) private loggerProvider: LoggerProvider,
+    @inject(TYPES.LoggerDbPersister) private loggerDbPersister: LoggerDbPersister,
     @inject(TYPES.LoggerFilePersister) private loggerFilePersister: LoggerFilePersister,
     @inject(TYPES.BotService) private botService: BotService,
     @inject(TYPES.WorkspaceService) private workspaceService: WorkspaceService
@@ -135,6 +139,9 @@ export class Botpress {
   }
 
   private async initializeServices() {
+    await this.loggerDbPersister.initialize(this.database, await this.loggerProvider('LogDbPersister'))
+    this.loggerDbPersister.start()
+
     await this.loggerFilePersister.initialize(this.config!, await this.loggerProvider('LogFilePersister'))
     await this.cmsService.initialize()
 

packages/studio-be/src/core/app/inversify/app.inversify.ts

@@ -3,6 +3,7 @@ import { Botpress } from 'core/app/botpress'
 import { HTTPServer } from 'core/app/server'
 import { ConfigProvider } from 'core/config'
 import { LoggerFilePersister, LoggerProvider, PersistedConsoleLogger } from 'core/logger'
+import { LoggerDbPersister } from 'core/logger/persister/db-persister'
 import { ModuleLoader } from 'core/modules'
 import { WorkspaceService } from 'core/users'
 import { Container } from 'inversify'
@@ -43,6 +44,11 @@ container.bind<LoggerProvider>(TYPES.LoggerProvider).toProvider<Logger>(context
   }
 })
 
+container
+  .bind<LoggerDbPersister>(TYPES.LoggerDbPersister)
+  .to(LoggerDbPersister)
+  .inSingletonScope()
+
 container
   .bind<LoggerFilePersister>(TYPES.LoggerFilePersister)
   .to(LoggerFilePersister)

packages/studio-be/src/core/logger/persister/console-logger.ts

@@ -9,6 +9,7 @@ import os from 'os'
 import stripAnsi from 'strip-ansi'
 import util from 'util'
 
+import { LoggerDbPersister } from './db-persister'
 import { LoggerFilePersister } from './file-persister'
 
 export type LoggerProvider = (module: string) => Promise<Logger>
@@ -41,6 +42,7 @@ export class PersistedConsoleLogger implements Logger {
 
   constructor(
     @inject(TYPES.Logger_Name) private name: string,
+    @inject(TYPES.LoggerDbPersister) private loggerDbPersister: LoggerDbPersister,
     @inject(TYPES.LoggerFilePersister) private loggerFilePersister: LoggerFilePersister
   ) {
     this.displayLevel = process.VERBOSITY_LEVEL
@@ -161,8 +163,12 @@ export class PersistedConsoleLogger implements Logger {
       timestamp: new Date()
     }
 
-    // We reset it right away to prevent race conditions (since the persister might log a new message asynchronously)
-    this.willPersistMessage = true
+    if (this.willPersistMessage && level !== LoggerLevel.Debug) {
+      this.loggerDbPersister.appendLog(entry)
+    } else {
+      // We reset it right away to prevent race conditions (since the persister might log a new message asynchronously)
+      this.willPersistMessage = true
+    }
 
     if (this.displayLevel >= this.currentMessageLevel!) {
       // eslint-disable-next-line no-console

packages/studio-be/src/core/logger/persister/db-persister.ts

@@ -0,0 +1,84 @@
+import { KnexExtended, Logger, LoggerEntry } from 'botpress/sdk'
+import Database from 'core/database'
+import { injectable } from 'inversify'
+import _ from 'lodash'
+import ms from 'ms'
+
+@injectable()
+export class LoggerDbPersister {
+  private readonly BATCH_SIZE = 100
+  private readonly TABLE_NAME = 'srv_logs'
+  private readonly INTERVAL = ms('2s')
+
+  private knex!: KnexExtended
+  private batch: LoggerEntry[] = []
+  private intervalRef
+  private currentPromise
+  private logger!: Logger
+
+  constructor() {}
+
+  async initialize(database: Database, logger: Logger) {
+    this.knex = database.knex
+    this.logger = logger
+  }
+
+  appendLog(log: LoggerEntry) {
+    this.batch.push(log)
+  }
+
+  start() {
+    this.validateInit()
+
+    if (this.intervalRef) {
+      return
+    }
+    this.intervalRef = setInterval(this.runTask, this.INTERVAL)
+  }
+
+  stop() {
+    clearInterval(this.intervalRef)
+    this.intervalRef = undefined
+    this.logger.info('Stopped')
+  }
+
+  private validateInit() {
+    if (!this.knex) {
+      throw new Error('The database is not initialized. You have to call initialize() first.')
+    }
+
+    if (!this.logger) {
+      throw new Error('Required a logger instance')
+    }
+  }
+
+  private runTask = async () => {
+    if (process.env.DEBUG_LOGGER) {
+      this.logger.debug(`Saving ${this.batch.length} log${this.batch.length === 1 ? '' : 's'}`)
+    }
+
+    if (this.currentPromise || this.batch.length === 0) {
+      return
+    }
+
+    const batchCount = this.batch.length >= this.BATCH_SIZE ? this.BATCH_SIZE : this.batch.length
+    const elements = this.batch.splice(0, batchCount)
+    const formatedRows = elements.map(log => ({
+      ...log,
+      timestamp: this.knex.date.format(log.timestamp)
+    }))
+
+    this.currentPromise = this.knex
+      .batchInsert(this.TABLE_NAME, formatedRows, this.BATCH_SIZE)
+      .catch(err => {
+        this.logger
+          .attachError(err)
+          .persist(false)
+          .error('Error persisting messages')
+        this.batch.push(...elements)
+      })
+      .finally(() => {
+        this.currentPromise = undefined
+      })
+  }
+}