Explore topologySpreadConstraints for K8s deployments

deployment/index.ts

@@ -235,6 +235,7 @@ const googleConfig = {
 const supertokens = deploySuperTokens(
   { apiKey: supertokensApiKey.result },
   { dependencies: [dbMigrations] },
+  deploymentEnv,
 );
 
 const zendeskConfig = new pulumi.Config('zendesk');

deployment/services/app.ts

@@ -60,6 +60,7 @@ export function deployApp({
       imagePullSecret,
       readinessProbe: '/api/health',
       livenessProbe: '/api/health',
+      availabilityOnEveryNode: true,
       env: [
         { name: 'DEPLOYED_DNS', value: deploymentEnv.DEPLOYED_DNS },
         { name: 'NODE_ENV', value: 'production' },

deployment/services/billing.ts

@@ -2,6 +2,7 @@ import { parse } from 'pg-connection-string';
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { DbMigrations } from './db-migrations';
@@ -39,7 +40,7 @@ export function deployStripeBilling({
     {
       image,
       imagePullSecret,
-      replicas: 1,
+      replicas: isProduction(deploymentEnv) ? 2 : 1,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
       env: {

deployment/services/emails.ts

@@ -1,6 +1,7 @@
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { Redis } from './redis';
@@ -53,7 +54,7 @@ export function deployEmails({
       livenessProbe: '/_health',
       exposesMetrics: true,
       image,
-      replicas: 1,
+      replicas: isProduction(deploymentEnv) ? 2 : 1,
     },
     [redis.deployment, redis.service],
   ).deploy();

deployment/services/graphql.ts

@@ -101,10 +101,11 @@ export function deployGraphQL({
     {
       imagePullSecret,
       image,
-      replicas: isProduction(deploymentEnv) ? 2 : 1,
+      replicas: isProduction(deploymentEnv) ? 3 : 1,
       pdb: true,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
+      availabilityOnEveryNode: true,
       env: {
         ...apiEnv,
         ...deploymentEnv,

deployment/services/policy.ts

@@ -1,6 +1,7 @@
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { ServiceDeployment } from '../utils/service-deployment';
 
 const commonConfig = new pulumi.Config('common');
@@ -31,7 +32,7 @@ export function deploySchemaPolicy({
     readinessProbe: '/_readiness',
     livenessProbe: '/_health',
     exposesMetrics: true,
-    replicas: 1,
+    replicas: isProduction(deploymentEnv) ? 2 : 1,
     pdb: true,
   }).deploy();
 }

deployment/services/proxy.ts

@@ -27,7 +27,7 @@ export function deployProxy({
     address: commonConfig.get('staticIp'),
     aksReservedIpResourceGroup: commonConfig.get('aksReservedIpResourceGroup'),
   })
-    .deployProxy({ replicas: isProduction(deploymentEnv) ? 2 : 1 })
+    .deployProxy({ replicas: isProduction(deploymentEnv) ? 3 : 1 })
     .registerService({ record: appHostname }, [
       {
         name: 'app',

deployment/services/rate-limit.ts

@@ -2,6 +2,7 @@ import { parse } from 'pg-connection-string';
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { DbMigrations } from './db-migrations';
@@ -41,7 +42,7 @@ export function deployRateLimit({
     'rate-limiter',
     {
       imagePullSecret,
-      replicas: 1,
+      replicas: isProduction(deploymentEnv) ? 2 : 1,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
       env: {

deployment/services/schema.ts

@@ -31,6 +31,7 @@ export function deploySchema({
     {
       image,
       imagePullSecret,
+      availabilityOnEveryNode: true,
       env: {
         ...deploymentEnv,
         ...commonEnv,

deployment/services/supertokens.ts

@@ -1,13 +1,16 @@
 import * as kx from '@pulumi/kubernetesx';
 import * as pulumi from '@pulumi/pulumi';
 import { Output } from '@pulumi/pulumi';
+import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 
 export function deploySuperTokens(
   { apiKey }: { apiKey: Output<string> },
   resourceOptions: {
     dependencies: pulumi.Resource[];
   },
+  deploymentEnv: DeploymentEnvironment,
 ) {
   const apiConfig = new pulumi.Config('api');
 
@@ -55,7 +58,7 @@ export function deploySuperTokens(
   const deployment = new kx.Deployment(
     'supertokens',
     {
-      spec: pb.asDeploymentSpec({ replicas: 1 }), // <-- here,
+      spec: pb.asDeploymentSpec({ replicas: isProduction(deploymentEnv) ? 3 : 1 }),
     },
     {
       dependsOn: resourceOptions.dependencies,

deployment/services/tokens.ts

@@ -42,7 +42,8 @@ export function deployTokens({
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
       exposesMetrics: true,
-      replicas: isProduction(deploymentEnv) ? 2 : 1,
+      availabilityOnEveryNode: true,
+      replicas: isProduction(deploymentEnv) ? 3 : 1,
       image,
       env: {
         ...deploymentEnv,

deployment/services/usage-estimation.ts

@@ -1,6 +1,7 @@
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { Clickhouse } from './clickhouse';
 import { DbMigrations } from './db-migrations';
@@ -30,7 +31,7 @@ export function deployUsageEstimation({
     {
       image,
       imagePullSecret,
-      replicas: 1,
+      replicas: isProduction(deploymentEnv) ? 2 : 1,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
       env: {

deployment/services/usage-ingestor.ts

@@ -63,6 +63,7 @@ export function deployUsageIngestor({
       replicas,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
+      availabilityOnEveryNode: true,
       env: {
         ...deploymentEnv,
         ...commonEnv,

deployment/services/usage.ts

@@ -47,6 +47,7 @@ export function deployUsage({
       replicas,
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
+      availabilityOnEveryNode: true,
       env: {
         ...deploymentEnv,
         ...commonEnv,

deployment/services/webhooks.ts

@@ -1,6 +1,7 @@
 import * as k8s from '@pulumi/kubernetes';
 import * as pulumi from '@pulumi/pulumi';
 import { DeploymentEnvironment } from '../types';
+import { isProduction } from '../utils/helpers';
 import { ServiceDeployment } from '../utils/service-deployment';
 import type { Broker } from './cf-broker';
 import { Redis } from './redis';
@@ -47,7 +48,7 @@ export function deployWebhooks({
       readinessProbe: '/_readiness',
       livenessProbe: '/_health',
       exposesMetrics: true,
-      replicas: 1,
+      replicas: isProduction(deploymentEnv) ? 2 : 1,
       image,
     },
     [redis.deployment, redis.service],

deployment/utils/service-deployment.ts

@@ -33,6 +33,7 @@ export class ServiceDeployment {
           cpuAverageToScale: number;
         };
       };
+      availabilityOnEveryNode?: boolean;
       command?: string[];
     },
     protected dependencies?: Array<pulumi.Resource | undefined | null>,
@@ -93,13 +94,31 @@ export class ServiceDeployment {
       );
     }
 
+    const topologySpreadConstraints: k8s.types.input.core.v1.TopologySpreadConstraint[] = [];
+
+    if (this.options.availabilityOnEveryNode) {
+      // This will ensure that services that has >1 replicas will be scheduled on every available node
+      // and ensure that we are not exposed to downtime issues caused by node failures/restarts:
+      topologySpreadConstraints.push({
+        maxSkew: 1,
+        topologyKey: 'topology.kubernetes.io/zone',
+        whenUnsatisfiable: 'DoNotSchedule',
+        labelSelector: {
+          matchLabels: {
+            app: this.name,
+          },
+        },
+      });
+    }
+
     const pb = new PodBuilder({
       restartPolicy: asJob ? 'Never' : 'Always',
       imagePullSecrets: this.options.imagePullSecret
         ? [{ name: this.options.imagePullSecret.metadata.name }]
         : undefined,
       terminationGracePeriodSeconds: 60,
       volumes: this.options.volumes,
+      topologySpreadConstraints,
       containers: [
         {
           livenessProbe,
@@ -150,6 +169,10 @@ export class ServiceDeployment {
       annotations: {},
     };
 
+    metadata.labels = {
+      app: this.name,
+    };
+
     if (this.options.exposesMetrics) {
       metadata.annotations = {
         'prometheus.io/port': '10254',
@@ -174,6 +197,7 @@ export class ServiceDeployment {
           },
           {
             annotations: metadata.annotations,
+            labels: metadata.labels,
           },
         ),
       },