Skip to content

Commit

Permalink
feat: replace current logging solution with log server (#1381)
Browse files Browse the repository at this point in the history
* feat: add log server

* feat(runtimes&server): adapt to log server

* chore(build): add helm template for log-server

* feat(log-server): health check

* refactor(server): fetch function logs from log server

* chore(docs): tidy docs for function log

* chore: clean code

* chore(ci): dockerize log server

* fix(server): replace db in FunctionConsole.write

* feat: verify appid for log server

* chore(runtimes): restart app when file changes

* chore: adjust env and region conf for supporting log server

* refactor(server): get logs from log server
  • Loading branch information
0fatal authored Jul 14, 2023
1 parent 1c4a760 commit 83a9960
Show file tree
Hide file tree
Showing 40 changed files with 3,204 additions and 137 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/dockerize-log-server.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: dockerize-log-server

on:
workflow_dispatch:
push:
branches:
- main
paths:
- "services/log-server/**"
- ".github/workflows/dockerize-log-server.yml"
- "!**/*.md"
- "!services/log-server/package-lock.json"

concurrency:
group: dockerize-log-server-${{ github.ref }}
cancel-in-progress: true

jobs:
dockerize-log-server:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
ghcr.io/${{ github.repository_owner }}/log-server
docker.io/${{ secrets.DOCKER_USERNAME }}/log-server
# https://github.com/docker/metadata-action#typesemver
tags: |
type=raw,value=latest,enable=true
type=sha,enable=true,format=short
- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Login to Github Container Hub
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push
uses: docker/build-push-action@v3
with:
context: ./services/log-server
file: ./services/log-server/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64, linux/arm64

trigger-workflow-build-cluster-image:
needs: [dockerize-log-server]
runs-on: ubuntu-latest
steps:
- name: trigger cluster image workflow
uses: peter-evans/repository-dispatch@v2
with:
event-type: docker_build_success
client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "version": "latest"}'
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
charts
20 changes: 20 additions & 0 deletions build/charts/laf-server/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Common labels
*/}}
{{- define "log-server.labels" -}}
helm.sh/chart: {{ include "laf-server.chart" . }}
{{ include "log-server.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Selector labels
*/}}
Expand All @@ -50,6 +62,14 @@ app.kubernetes.io/name: {{ include "laf-server.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Selector labels
*/}}
{{- define "log-server.selectorLabels" -}}
app.kubernetes.io/name: log-server
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Create the name of the service account to use
*/}}
Expand Down
6 changes: 6 additions & 0 deletions build/charts/laf-server/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@ spec:
value: {{ .Values.default_region.apisix_api_url }}
- name: DEFAULT_REGION_APISIX_API_KEY
value: {{ .Values.default_region.apisix_api_key }}
- name: DEFAULT_REGION_LOG_SERVER_URL
value: {{ .Values.default_region.log_server_url }}
- name: DEFAULT_REGION_LOG_SERVER_SECRET
value: {{ .Values.default_region.log_server_secret }}
- name: DEFAULT_REGION_LOG_SERVER_DATABASE_URL
value: {{ .Values.default_region.log_server_database_url }}
- name: SITE_NAME
value: {{ .Values.siteName | quote}}
{{- with .Values.nodeSelector }}
Expand Down
52 changes: 52 additions & 0 deletions build/charts/laf-server/templates/log-server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
{{- include "log-server.labels" . | nindent 4 }}
name: log-server
spec:
replicas: 1
selector:
matchLabels:
{{- include "log-server.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "log-server.selectorLabels" . | nindent 8 }}
spec:
containers:
- image: docker.io/lafyun/log-server:latest
imagePullPolicy: Always
name: log-server
ports:
- name: http
containerPort: 5060
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: http
readinessProbe:
httpGet:
path: /healthz
port: http
env:
- name: DB_URI
value: {{ .Values.default_region.log_server_database_url | quote }}
- name: JWT_SECRET
value: {{ .Values.default_region.log_server_secret | quote }}
---
apiVersion: v1
kind: Service
metadata:
labels:
{{- include "log-server.labels" . | nindent 4 }}
name: log-server
spec:
ports:
- name: http
port: 5060
protocol: TCP
targetPort: http
selector:
{{- include "log-server.selectorLabels" . | nindent 4 }}
4 changes: 4 additions & 0 deletions build/charts/laf-server/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ default_region:
tls: false
runtime_domain: ""
website_domain: ""
# log-server
log_server_url: ""
log_server_secret: ""
log_server_database_url: ""
jwt:
secret: laf_server_abc123
expires_in: 7d
Expand Down
6 changes: 6 additions & 0 deletions build/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ helm install minio -n ${NAMESPACE} \

## 4. install laf-server
SERVER_JWT_SECRET=$PASSWD_OR_SECRET
LOG_SERVER_URL="http://log-server.${NAMESPACE}.svc.cluster.local:5060"
LOG_SERVER_DATABASE_URL="mongodb://${DB_USERNAME:-admin}:${PASSWD_OR_SECRET}@mongodb-0.mongo.${NAMESPACE}.svc.cluster.local:27017/function-logs?authSource=admin&replicaSet=rs0&w=majority"
LOG_SERVER_SECRET=$PASSWD_OR_SECRET
helm install server -n ${NAMESPACE} \
--set databaseUrl=${DATABASE_URL} \
--set meteringDatabaseUrl=${METERING_DATABASE_URL} \
Expand All @@ -90,6 +93,9 @@ helm install server -n ${NAMESPACE} \
--set default_region.apisix_api_url=${APISIX_API_URL} \
--set default_region.apisix_api_key=${APISIX_API_KEY} \
--set default_region.apisix_public_port=80 \
--set default_region.log_server_url=${LOG_SERVER_URL} \
--set default_region.log_server_secret=${LOG_SERVER_SECRET} \
--set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \
./charts/laf-server

## 6. install laf-web
Expand Down
22 changes: 0 additions & 22 deletions docs/en/guide/function/logs.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,3 @@ You can filter logs based on the `requestId` and the name of the cloud function.
1. Click on a log to switch to the log section.
2. Use the `requestId` and the name of the cloud function to search for specific logs.
3. Click on an individual log to view detailed content.

## Manual Log Cleaning

The running logs of Laf cloud functions are stored in a hidden collection called `__function_logs__`. Therefore, we can use the method to manipulate the database in cloud functions to clean the logs.

The following is the code to clean all logs in a cloud function:

::: danger
The following operation will delete all historical logs. Please proceed with caution.
:::

```typescript
import cloud from '@lafjs/cloud'

export async function main(ctx: FunctionContext) {
console.log('Hello World')
// Database, remove all logs
const db = cloud.database();
const res = await db.collection('__function_logs__').remove({multi:true})
console.log(res)
}
```
24 changes: 0 additions & 24 deletions docs/guide/function/logs.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,3 @@ title: 云函数历史日志
2、可根据 `requestId` 和云函数名搜索指定日志

3、点击单个日志,可查看详细内容

## 手动清理日志

Laf 云函数的运行日志都在一个隐藏的集合中:`__function_logs__`

所以我们可以通过云函数操作数据库的方法,清理日志

以下是清理全部日志的云函数写法:

::: danger
以下操作会删除全部历史日志,请谨慎操作
:::

```typescript
import cloud from '@lafjs/cloud'

export async function main(ctx: FunctionContext) {
console.log('Hello World')
// 数据库,删除全部日志
const db = cloud.database();
const res = await db.collection('__function_logs__').remove({multi:true})
console.log(res)
}
```
3 changes: 2 additions & 1 deletion lerna.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"./server",
"./web",
"./runtimes/nodejs",
"./cli"
"./cli",
"./services/*"
],
"version": "1.0.0-beta.10",
"command": {
Expand Down
8 changes: 5 additions & 3 deletions runtimes/nodejs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
"typings": "./dist/index.d.ts",
"private": "true",
"scripts": {
"start": "node --expose-internals ./dist/index.js",
"dev": "NODE_ENV=development npm run start",
"start": "node ./dist/index.js",
"dev": "npx concurrently npm:dev:*",
"dev:start": "npx nodemon ./dist/index.js",
"dev:watch": "npm run watch",
"build": "tsc -p tsconfig.json",
"watch": "tsc -p tsconfig.json -w",
"prepublishOnly": "npm run build",
Expand Down Expand Up @@ -73,4 +75,4 @@
],
"delay": 1000
}
}
}
15 changes: 8 additions & 7 deletions runtimes/nodejs/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,6 @@ export default class Config {
return process.env.NODE_ENV === 'production'
}

/**
* Expired time of function logs, in seconds
*/
static get FUNCTION_LOG_EXPIRED_TIME(): number {
return (process.env.FUNCTION_LOG_EXPIRED_TIME ?? 3600 * 24 * 3) as number
}

static get RUNTIME_IMAGE(): string {
return process.env.RUNTIME_IMAGE
}
Expand All @@ -83,4 +76,12 @@ export default class Config {
static get REQUEST_LIMIT_SIZE(): string {
return process.env.REQUEST_LIMIT_SIZE || '10mb'
}

static get LOG_SERVER_URL(): string {
return process.env.LOG_SERVER_URL || ''
}

static get LOG_SERVER_TOKEN(): string {
return process.env.LOG_SERVER_TOKEN || ''
}
}
1 change: 0 additions & 1 deletion runtimes/nodejs/src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
*/
export const CLOUD_FUNCTION_COLLECTION = '__functions__'
export const POLICY_COLLECTION = '__policies__'
export const FUNCTION_LOG_COLLECTION = '__function_logs__'
export const CONFIG_COLLECTION = '__conf__'

export const WEBSOCKET_FUNCTION_NAME = '__websocket__'
Expand Down
2 changes: 0 additions & 2 deletions runtimes/nodejs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,10 @@ import xmlparser from 'express-xml-bodyparser'
import './support/cloud-sdk'
import { FunctionCache } from './support/function-engine/cache'
import { DatabaseChangeStream } from './support/db-change-stream'
import { ensureCollectionIndexes } from './support/function-log'

const app = express()

DatabaseAgent.accessor.ready.then(() => {
ensureCollectionIndexes()
FunctionCache.initialize()
DatabaseChangeStream.initialize()
})
Expand Down
22 changes: 21 additions & 1 deletion runtimes/nodejs/src/support/function-engine/console.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,30 @@
import * as util from 'util'
import { FunctionContext } from './types'
import Config from '../../config'
import axios from 'axios'

export class FunctionConsole {
ctx: FunctionContext

static write: (message: string, ctx: FunctionContext) => void = console.log
static write(message: string, ctx: FunctionContext) {
if (!Config.LOG_SERVER_URL || !Config.LOG_SERVER_TOKEN) return

const doc = {
request_id: ctx.requestId,
func: ctx.__function_name,
data: message,
created_at: new Date(),
}

axios.post(`${Config.LOG_SERVER_URL}/function/log`, {
appid: Config.APPID,
log: doc,
}, {
headers: {
'x-token': Config.LOG_SERVER_TOKEN
}
})
}

constructor(ctx: FunctionContext) {
this.ctx = ctx
Expand Down
Loading

0 comments on commit 83a9960

Please sign in to comment.