From 8751858cb2fd130fc8030fb5e4caaae86b948c54 Mon Sep 17 00:00:00 2001 From: maslow Date: Tue, 10 Oct 2023 11:59:58 +0800 Subject: [PATCH] feat(server): add tls config for ingress gateway (#1569) --- .../laf-server/templates/deployment.yaml | 4 ++++ build/charts/laf-server/values.yaml | 4 +++- build/start.sh | 2 +- server/src/constants.ts | 18 ++++++++++++-- .../ingress/runtime-ingress.service.ts | 21 +++++++++++----- .../ingress/website-ingress.service.ts | 24 ++++++++++++++++--- server/src/gateway/runtime-domain.service.ts | 4 +--- server/src/gateway/website-task.service.ts | 1 + server/src/initializer/initializer.service.ts | 7 +++--- 9 files changed, 66 insertions(+), 19 deletions(-) diff --git a/build/charts/laf-server/templates/deployment.yaml b/build/charts/laf-server/templates/deployment.yaml index 4aa20e2a63..4fefe0fca0 100644 --- a/build/charts/laf-server/templates/deployment.yaml +++ b/build/charts/laf-server/templates/deployment.yaml @@ -77,6 +77,10 @@ spec: value: {{ .Values.default_region.runtime_domain }} - name: DEFAULT_REGION_WEBSITE_DOMAIN value: {{ .Values.default_region.website_domain }} + - name: DEFAULT_REGION_TLS_ENABLED + value: {{ .Values.default_region.tls.enabled }} + - name: DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME + value: {{ .Values.default_region.tls.wildcard_certificate_secret_name }} - name: DEFAULT_REGION_LOG_SERVER_URL value: {{ .Values.default_region.log_server_url }} - name: DEFAULT_REGION_LOG_SERVER_SECRET diff --git a/build/charts/laf-server/values.yaml b/build/charts/laf-server/values.yaml index d8d284d72a..e88d3a8965 100644 --- a/build/charts/laf-server/values.yaml +++ b/build/charts/laf-server/values.yaml @@ -19,7 +19,9 @@ default_region: minio_root_access_key: "" minio_root_secret_key: "" # gateway conf - tls: false + tls: + enabled: false + wildcard_certificate_secret_name: "" runtime_domain: "" website_domain: "" # log-server diff --git a/build/start.sh b/build/start.sh index 4ac8180a5f..5b6218d452 100644 --- a/build/start.sh +++ b/build/start.sh @@ -95,7 +95,7 @@ helm install server -n ${NAMESPACE} \ --set default_region.minio_root_secret_key=${MINIO_ROOT_SECRET_KEY} \ --set default_region.runtime_domain=${DOMAIN} \ --set default_region.website_domain=site.${DOMAIN} \ - --set default_region.tls=false \ + --set default_region.tls.enabled=false \ --set default_region.log_server_url=${LOG_SERVER_URL} \ --set default_region.log_server_secret=${LOG_SERVER_SECRET} \ --set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \ diff --git a/server/src/constants.ts b/server/src/constants.ts index fa14ef90e1..6bc63d01f8 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -113,11 +113,25 @@ export class ServerConfig { } static get DEFAULT_REGION_RUNTIME_DOMAIN() { - return process.env.DEFAULT_REGION_RUNTIME_DOMAIN || 'localhost' + if (!process.env.DEFAULT_REGION_RUNTIME_DOMAIN) { + throw new Error('DEFAULT_REGION_RUNTIME_DOMAIN is not defined') + } + return process.env.DEFAULT_REGION_RUNTIME_DOMAIN } static get DEFAULT_REGION_WEBSITE_DOMAIN() { - return process.env.DEFAULT_REGION_WEBSITE_DOMAIN || 'localhost' + if (!process.env.DEFAULT_REGION_WEBSITE_DOMAIN) { + throw new Error('DEFAULT_REGION_WEBSITE_DOMAIN is not defined') + } + return process.env.DEFAULT_REGION_WEBSITE_DOMAIN + } + + static get DEFAULT_REGION_TLS_ENABLED() { + return process.env.DEFAULT_REGION_TLS_ENABLED === 'true' + } + + static get DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME() { + return process.env.DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME } static get DEFAULT_REGION_MINIO_DOMAIN() { diff --git a/server/src/gateway/ingress/runtime-ingress.service.ts b/server/src/gateway/ingress/runtime-ingress.service.ts index 727c4498a4..7d03de51dc 100644 --- a/server/src/gateway/ingress/runtime-ingress.service.ts +++ b/server/src/gateway/ingress/runtime-ingress.service.ts @@ -1,4 +1,4 @@ -import { V1Ingress } from '@kubernetes/client-node' +import { V1Ingress, V1IngressTLS } from '@kubernetes/client-node' import { Injectable, Logger } from '@nestjs/common' import { LABEL_KEY_APP_ID } from 'src/constants' import { ClusterService } from 'src/region/cluster/cluster.service' @@ -55,11 +55,20 @@ export class RuntimeGatewayService { }) // build tls - const tls = [] - if (runtimeDomain.customDomain) { - const secretName = - this.certificate.getRuntimeCertificateName(runtimeDomain) - tls.push({ secretName, hosts }) + const tls: Array = [] + if (region.gatewayConf.tls.enabled) { + // add wildcardDomain tls + if (region.gatewayConf.tls.wildcardCertificateSecretName) { + const secretName = region.gatewayConf.tls.wildcardCertificateSecretName + tls.push({ secretName, hosts: [runtimeDomain.domain] }) + } + + // add customDomain tls + if (runtimeDomain.customDomain) { + const secretName = + this.certificate.getRuntimeCertificateName(runtimeDomain) + tls.push({ secretName, hosts: [runtimeDomain.customDomain] }) + } } // create ingress diff --git a/server/src/gateway/ingress/website-ingress.service.ts b/server/src/gateway/ingress/website-ingress.service.ts index 130aafa988..9497e1a877 100644 --- a/server/src/gateway/ingress/website-ingress.service.ts +++ b/server/src/gateway/ingress/website-ingress.service.ts @@ -1,16 +1,20 @@ -import { V1Ingress, V1IngressRule } from '@kubernetes/client-node' +import { V1Ingress, V1IngressRule, V1IngressTLS } from '@kubernetes/client-node' import { Injectable, Logger } from '@nestjs/common' import { LABEL_KEY_APP_ID } from 'src/constants' import { ClusterService } from 'src/region/cluster/cluster.service' import { Region } from 'src/region/entities/region' import { GetApplicationNamespace } from 'src/utils/getter' import { WebsiteHosting } from 'src/website/entities/website' +import { CertificateService } from '../certificate.service' @Injectable() export class WebsiteHostingGatewayService { private readonly logger = new Logger(WebsiteHostingGatewayService.name) - constructor(private readonly clusterService: ClusterService) {} + constructor( + private readonly clusterService: ClusterService, + private readonly certificate: CertificateService, + ) {} getIngressName(websiteHosting: WebsiteHosting) { return websiteHosting._id.toString() @@ -44,6 +48,20 @@ export class WebsiteHostingGatewayService { http: { paths: [{ path: '/', pathType: 'Prefix', backend }] }, } + // build tls + const tls: Array = [] + if (region.gatewayConf.tls.enabled) { + if (website.isCustom) { + // add custom domain tls + const secretName = this.certificate.getWebsiteCertificateName(website) + tls.push({ secretName, hosts: [website.domain] }) + } else if (region.gatewayConf.tls.wildcardCertificateSecretName) { + // add wildcardDomain tls + const secretName = region.gatewayConf.tls.wildcardCertificateSecretName + tls.push({ secretName, hosts: [website.domain] }) + } + } + // create ingress const ingressClassName = region.gatewayConf.driver const ingressBody: V1Ingress = { @@ -62,7 +80,7 @@ export class WebsiteHostingGatewayService { 'nginx.ingress.kubernetes.io/enable-cors': 'true', }, }, - spec: { ingressClassName, rules: [rule] }, + spec: { ingressClassName, rules: [rule], tls }, } const res = await this.clusterService.createIngress(region, ingressBody) diff --git a/server/src/gateway/runtime-domain.service.ts b/server/src/gateway/runtime-domain.service.ts index 5d635602ce..1acae69523 100644 --- a/server/src/gateway/runtime-domain.service.ts +++ b/server/src/gateway/runtime-domain.service.ts @@ -42,9 +42,7 @@ export class RuntimeDomainService { async checkResolved(appid: string, customDomain: string) { const runtimeDomain = await this.db .collection('RuntimeDomain') - .findOne({ - appid, - }) + .findOne({ appid }) const cnameTarget = runtimeDomain.domain diff --git a/server/src/gateway/website-task.service.ts b/server/src/gateway/website-task.service.ts index 57168d023e..8d35d6c0a2 100644 --- a/server/src/gateway/website-task.service.ts +++ b/server/src/gateway/website-task.service.ts @@ -99,6 +99,7 @@ export class WebsiteTaskService { assert(bucketDomain, 'bucket domain not found') // create website custom certificate if custom domain is set + // Warning: create certificate before ingress, otherwise apisix ingress will not work if (site.isCustom && region.gatewayConf.tls.enabled) { const waitingTime = Date.now() - site.updatedAt.getTime() diff --git a/server/src/initializer/initializer.service.ts b/server/src/initializer/initializer.service.ts index b72623612e..5811d41898 100644 --- a/server/src/initializer/initializer.service.ts +++ b/server/src/initializer/initializer.service.ts @@ -76,9 +76,10 @@ export class InitializerService { websiteDomain: ServerConfig.DEFAULT_REGION_WEBSITE_DOMAIN, port: 80, tls: { - enabled: false, - issuerRef: { name: 'laf-issuer', kind: 'ClusterIssuer' }, - wildcardCertificateSecretName: null, + enabled: ServerConfig.DEFAULT_REGION_TLS_ENABLED, + issuerRef: { name: 'laf-issuer', kind: 'Issuer' }, + wildcardCertificateSecretName: + ServerConfig.DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME, }, }, logServerConf: {