feat: exclude proxy from load balancer when full (#396)

* feat(shulker-proxy-agent): create a readiness lock file when capacity is reached * feat(shulker-operator): take into account the readiness lock in readiness script * feat(shulker-crds): add proxy player delta config * feat(shulker-operator): inject SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION to proxies * feat(shulker-proxy-agent): release agones server if proxy is empty
jeremylvln · Feb 6, 2024 · e53e9f3 · e53e9f3
1 parent 235e6ba
commit e53e9f3
Show file tree

Hide file tree

Showing 16 changed files with 99 additions and 5 deletions.
diff --git a/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml b/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml
@@ -283,6 +283,13 @@ spec:
                             description: Name of an optional ConfigMap already containing the server configuration
                             nullable: true
                             type: string
+                          lifecycleStrategy:
+                            default: AllocateWhenNotEmpty
+                            description: Strategy to apply concerning Agones `GameServer` lifecycle management
+                            enum:
+                            - AllocateWhenNotEmpty
+                            - Manual
+                            type: string
                           maxPlayers:
                             default: 20
                             description: Number of maximum players that can connect to the MinecraftServer Deployment

diff --git a/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml b/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml
@@ -53,6 +53,13 @@ spec:
                     description: Name of an optional ConfigMap already containing the server configuration
                     nullable: true
                     type: string
+                  lifecycleStrategy:
+                    default: AllocateWhenNotEmpty
+                    description: Strategy to apply concerning Agones `GameServer` lifecycle management
+                    enum:
+                    - AllocateWhenNotEmpty
+                    - Manual
+                    type: string
                   maxPlayers:
                     default: 20
                     description: Number of maximum players that can connect to the MinecraftServer Deployment

diff --git a/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml b/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml
@@ -337,6 +337,12 @@ spec:
                               type: object
                             nullable: true
                             type: array
+                          playersDeltaBeforeExclusion:
+                            default: 15
+                            description: Number of player slots to reserve when exclusing a proxy from the load balancer. This will allow load balancer implementations to update itself while still being able to accept some players
+                            format: uint32
+                            minimum: 0.0
+                            type: integer
                           plugins:
                             description: List of references to plugins to download
                             items:

diff --git a/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs b/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs
@@ -130,6 +130,15 @@ pub struct ProxyFleetTemplateConfigurationSpec {
     /// drained automatically
     #[schemars(default = "ProxyFleetTemplateConfigurationSpec::default_ttl_seconds")]
     pub ttl_seconds: u32,
+
+    /// Number of player slots to reserve when exclusing a proxy
+    /// from the load balancer. This will allow load balancer
+    /// implementations to update itself while still being able
+    /// to accept some players
+    #[schemars(
+        default = "ProxyFleetTemplateConfigurationSpec::default_players_delta_before_exclusion"
+    )]
+    pub players_delta_before_exclusion: u32,
 }
 
 #[cfg(not(tarpaulin_include))]
@@ -153,6 +162,10 @@ impl ProxyFleetTemplateConfigurationSpec {
     fn default_ttl_seconds() -> u32 {
         86400
     }
+
+    fn default_players_delta_before_exclusion() -> u32 {
+        15
+    }
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]

diff --git a/packages/shulker-operator/assets/proxy-probe-readiness.sh b/packages/shulker-operator/assets/proxy-probe-readiness.sh
@@ -4,6 +4,8 @@ set -o xtrace
 
 if [ -f "/tmp/drain-lock" ]; then
   echo "Drain lock found" && exit 1
+elif [ -f "/tmp/readiness-lock" ]; then
+  echo "Readiness lock found" && exit 1
 fi
 
 bash /usr/bin/health.sh
diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs
@@ -348,6 +348,7 @@ mod bungeecord {
                 server_icon: "A Server Icon".to_string(),
                 proxy_protocol: true,
                 ttl_seconds: 300,
+                players_delta_before_exclusion: 15,
             };
             let service_spec = Some(ProxyFleetServiceSpec {
                 type_: ProxyFleetServiceType::LoadBalancer,
@@ -552,6 +553,7 @@ mod velocity {
                 server_icon: "A Server Icon".to_string(),
                 proxy_protocol: true,
                 ttl_seconds: 300,
+                players_delta_before_exclusion: 15,
             };
             let service_spec = Some(ProxyFleetServiceSpec {
                 type_: ProxyFleetServiceType::LoadBalancer,

diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs
@@ -64,6 +64,7 @@ lazy_static! {
                         server_icon: "abc==".to_string(),
                         proxy_protocol: true,
                         ttl_seconds: 3600,
+                        players_delta_before_exclusion: 15,
                     },
                     pod_overrides: Some(ProxyFleetTemplatePodOverridesSpec {
                         image: None,

diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs
@@ -447,6 +447,11 @@ impl<'a> FleetBuilder {
                 value: Some(spec.config.ttl_seconds.to_string()),
                 ..EnvVar::default()
             },
+            EnvVar {
+                name: "SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION".to_string(),
+                value: Some(spec.config.players_delta_before_exclusion.to_string()),
+                ..EnvVar::default()
+            },
             EnvVar {
                 name: "SHULKER_NETWORK_ADMINS".to_string(),
                 value: Some(

diff --git a/...pshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap b/...pshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap
@@ -6,7 +6,7 @@ apiVersion: v1
 kind: ConfigMap
 data:
   init-fs.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\ncp \"${SHULKER_CONFIG_DIR}/probe-readiness.sh\" \"${SHULKER_PROXY_DATA_DIR}/probe-readiness.sh\"\ncat \"${SHULKER_CONFIG_DIR}/server-icon.png\" | base64 -d > \"${SHULKER_PROXY_DATA_DIR}/server-icon.png\"\n\nif [ \"${SHULKER_VERSION_CHANNEL}\" == \"Velocity\" ]; then\n  cp \"${SHULKER_CONFIG_DIR}/velocity-config.toml\" \"${SHULKER_PROXY_DATA_DIR}/velocity.toml\"\n  echo \"dummy\" > \"${SHULKER_PROXY_DATA_DIR}/forwarding.secret\"\nelse\n  cp \"${SHULKER_CONFIG_DIR}/bungeecord-config.yml\" \"${SHULKER_PROXY_DATA_DIR}/config.yml\"\nfi\n\nif [ ! -z \"${SHULKER_PROXY_PLUGIN_URLS+x}\" ]; then\n  mkdir -p \"${SHULKER_PROXY_DATA_DIR}/plugins\"\n  for plugin_url in ${SHULKER_PROXY_PLUGIN_URLS//;/ }; do\n    (cd \"${SHULKER_PROXY_DATA_DIR}/plugins\" && wget \"${plugin_url}\")\n  done\nfi\n\nif [ ! -z \"${SHULKER_PROXY_PATCH_URLS+x}\" ]; then\n  for patch_url in ${SHULKER_PROXY_PATCH_URLS//;/ }; do\n    (cd \"${SHULKER_PROXY_DATA_DIR}\" && wget \"${patch_url}\" -O - | tar -xzv)\n  done\nfi\n"
-  probe-readiness.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\nif [ -f \"/tmp/drain-lock\" ]; then\n  echo \"Drain lock found\" && exit 1\nfi\n\nbash /usr/bin/health.sh\n"
+  probe-readiness.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\nif [ -f \"/tmp/drain-lock\" ]; then\n  echo \"Drain lock found\" && exit 1\nelif [ -f \"/tmp/readiness-lock\" ]; then\n  echo \"Readiness lock found\" && exit 1\nfi\n\nbash /usr/bin/health.sh\n"
   server-icon.png: abc==
   velocity-config.toml: "config-version = \"2.6\"\nbind = \"0.0.0.0:25577\"\nmotd = \"A Motd\"\nshow-max-players = 1000\nonline-mode = true\nforce-key-authentication = true\nprevent-client-proxy-connections = true\nforwarding-secret-file = \"/mnt/shulker/forwarding-secret/key\"\nplayer-info-forwarding-mode = \"modern\"\n\n[servers]\nlobby = \"localhost:30000\"\nlimbo = \"localhost:30001\"\ntry = [\"lobby\", \"limbo\"]\n\n[forced-hosts]\n\n[advanced]\nhaproxy-protocol = true\ntcp-fast-open = true\n\n"
 metadata:

diff --git a/...t/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap b/...t/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap
@@ -73,6 +73,8 @@ spec:
                       fieldPath: metadata.namespace
                 - name: SHULKER_PROXY_TTL_SECONDS
                   value: "3600"
+                - name: SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION
+                  value: "15"
                 - name: SHULKER_NETWORK_ADMINS
                   value: ""
                 - name: SHULKER_PROXY_REDIS_HOST

diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt
@@ -2,13 +2,17 @@ package io.shulkermc.proxyagent
 
 import java.util.Optional
 import java.util.UUID
+import kotlin.jvm.optionals.getOrDefault
 
+@SuppressWarnings("detekt:MagicNumber")
 object Configuration {
     val CLUSTER_NAME = getStringEnv("SHULKER_CLUSTER_NAME")
 
     val PROXY_NAMESPACE = getStringEnv("SHULKER_PROXY_NAMESPACE")
     val PROXY_NAME = getStringEnv("SHULKER_PROXY_NAME")
     val PROXY_TTL_SECONDS = getLongEnv("SHULKER_PROXY_TTL_SECONDS")
+    val PROXY_PLAYER_DELTA_BEFORE_EXCLUSION = getOptionalIntEnv("SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION")
+        .getOrDefault(15)
 
     val NETWORK_ADMINS: List<UUID> = getOptionalStringEnv("SHULKER_NETWORK_ADMINS")
         .map {
@@ -26,5 +30,7 @@ object Configuration {
     private fun getStringEnv(name: String): String = requireNotNull(System.getenv(name)) { "Missing $name" }
     private fun getOptionalStringEnv(name: String): Optional<String> = Optional.ofNullable(System.getenv(name))
     private fun getIntEnv(name: String): Int = getStringEnv(name).toInt()
+    private fun getOptionalIntEnv(name: String): Optional<Int> = Optional.ofNullable(System.getenv(name))
+        .map { it.toInt() }
     private fun getLongEnv(name: String): Long = getStringEnv(name).toLong()
 }
diff --git a/.../shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt b/.../shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt
@@ -87,7 +87,7 @@ class ShulkerProxyAgentCommon(val proxyInterface: ProxyInterface, val logger: Lo
             }
 
             this.cache.registerProxy(Configuration.PROXY_NAME, this.proxyInterface.getPlayerCapacity())
-            this.agonesGateway.setAllocated()
+            this.agonesGateway.setReady()
         } catch (@Suppress("TooGenericExceptionCaught") e: Exception) {
             this.logger.log(Level.SEVERE, "Shulker Agent crashed, stopping proxy", e)
             this.shutdown()

diff --git a/...-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt b/...-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt
@@ -1,5 +1,8 @@
 package io.shulkermc.proxyagent.adapters.filesystem
 
 interface FileSystemAdapter {
-    fun createDrainFile()
+    fun createDrainLock()
+
+    fun createReadinessLock()
+    fun deleteReadinessLock()
 }
diff --git a/...t/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt b/...t/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt
@@ -4,11 +4,22 @@ import java.nio.file.Files
 import java.nio.file.Path
 
 private val DRAIN_LOCK_PATH = Path.of("/tmp/drain-lock")
+private val READINESS_LOCK_PATH = Path.of("/tmp/readiness-lock")
 
 class LocalFileSystemAdapter : FileSystemAdapter {
-    override fun createDrainFile() {
+    override fun createDrainLock() {
         if (!Files.exists(DRAIN_LOCK_PATH)) {
             Files.createFile(DRAIN_LOCK_PATH)
         }
     }
+
+    override fun createReadinessLock() {
+        if (!Files.exists(READINESS_LOCK_PATH)) {
+            Files.createFile(READINESS_LOCK_PATH)
+        }
+    }
+
+    override fun deleteReadinessLock() {
+        Files.deleteIfExists(READINESS_LOCK_PATH)
+    }
 }
diff --git a/...r-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt b/...r-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt
@@ -31,6 +31,9 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) {
         )
     }
 
+    private val maxPlayersWithExclusionDelta =
+        this.agent.proxyInterface.getPlayerCapacity() - Configuration.PROXY_PLAYER_DELTA_BEFORE_EXCLUSION
+
     private val onlinePlayerCountSupplier = Suppliers.memoizeWithExpiration(
         { this.agent.cache.countOnlinePlayers() },
         ONLINE_PLAYERS_COUNT_MEMOIZE_SECONDS,
@@ -42,6 +45,7 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) {
         java.util.concurrent.TimeUnit.SECONDS
     )
 
+    private var isAllocatedByAgones = false
     private var acceptingPlayers = true
 
     init {
@@ -57,8 +61,10 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) {
         this.acceptingPlayers = acceptingPlayers
 
         if (acceptingPlayers) {
+            this.agent.fileSystem.deleteReadinessLock()
             this.agent.logger.info("Proxy is now accepting players")
         } else {
+            this.agent.fileSystem.createReadinessLock()
             this.agent.logger.info("Proxy is no longer accepting players")
         }
     }
@@ -71,15 +77,34 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) {
         if (!this.acceptingPlayers) {
             return PlayerPreLoginHookResult.disallow(MSG_NOT_ACCEPTING_PLAYERS)
         }
+
         return PlayerPreLoginHookResult.allow()
     }
 
     private fun onPlayerLogin(player: Player) {
         this.agent.cache.updateCachedPlayerName(player.uniqueId, player.name)
+
+        if (!this.isAllocatedByAgones) {
+            this.isAllocatedByAgones = true
+            this.agent.agonesGateway.setAllocated()
+        }
+
+        if (this.isProxyConsideredFull()) {
+            this.setAcceptingPlayers(false)
+        }
     }
 
     private fun onPlayerDisconnect(player: Player) {
         this.agent.cache.unsetPlayerPosition(player.uniqueId)
+
+        if (this.isAllocatedByAgones && this.agent.proxyInterface.getPlayerCount() == 0) {
+            this.isAllocatedByAgones = false
+            this.agent.agonesGateway.setReady()
+        }
+
+        if (!this.isProxyConsideredFull()) {
+            this.setAcceptingPlayers(true)
+        }
     }
 
     private fun onServerPreConnect(player: Player, originalServerName: String): ServerPreConnectHookResult {
@@ -105,4 +130,8 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) {
     private fun onServerPostConnect(player: Player, serverName: String) {
         this.agent.cache.setPlayerPosition(player.uniqueId, Configuration.PROXY_NAME, serverName)
     }
+
+    private fun isProxyConsideredFull(): Boolean {
+        return this.agent.proxyInterface.getPlayerCount() >= this.maxPlayersWithExclusionDelta
+    }
 }
diff --git a/...r-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt b/...r-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt
@@ -43,7 +43,7 @@ class ProxyLifecycleService(private val agent: ShulkerProxyAgentCommon) {
         }
         this.drained = true
 
-        this.agent.fileSystem.createDrainFile()
+        this.agent.fileSystem.createDrainLock()
         this.agent.playerMovementService.setAcceptingPlayers(false)
 
         this.agent.proxyInterface.scheduleRepeatingTask(