fix: check for pod readiness individually to show more info to the user

Signed-off-by: Lenin Mehedy <lenin.mehedy@swirldslabs.com>
hashgraph · Jan 23, 2024 · 505ab8d · 505ab8d
1 parent 2b3f9a3
commit 505ab8d
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 22 deletions.
diff --git a/fullstack-network-manager/src/commands/chart.mjs b/fullstack-network-manager/src/commands/chart.mjs
@@ -1,9 +1,10 @@
+import chalk from 'chalk'
 import { Listr } from 'listr2'
 import { FullstackTestingError } from '../core/errors.mjs'
 import { BaseCommand } from './base.mjs'
 import * as flags from './flags.mjs'
 import * as paths from 'path'
-import { constants } from '../core/index.mjs'
+import { constants, Templates } from '../core/index.mjs'
 import * as prompts from './prompts.mjs'
 
 export class ChartCommand extends BaseCommand {
@@ -86,7 +87,6 @@ export class ChartCommand extends BaseCommand {
       enableHederaExplorerTls: await prompts.promptEnableHederaExplorerTls(task, enableHederaExplorerTls),
       acmeClusterIssuer: await prompts.promptAcmeClusterIssuer(task, acmeClusterIssuer),
       selfSignedClusterIssuer: await prompts.promptSelfSignedClusterIssuer(task, selfSignedClusterIssuer),
-      timeout: 900,
       version: this.configManager.getVersion()
     }
 
@@ -129,12 +129,29 @@ export class ChartCommand extends BaseCommand {
       },
       {
         title: 'Waiting for network pods to be ready',
-        task: async (ctx, _) => {
-          const timeout = ctx.config.timeout || 900
-          await this.kubectl2.waitForPod(constants.POD_STATUS_RUNNING, [
-            'fullstack.hedera.com/type=network-node'
-          ], ctx.config.nodeIds.length, timeout * 1000, 1000)
-        }
+        task:
+          async (ctx, task) => {
+            const subTasks = []
+            for (const nodeId of ctx.config.nodeIds) {
+              const podName = Templates.renderNetworkPodName(nodeId)
+              subTasks.push({
+                title: `Node: ${chalk.yellow(nodeId)} (Pod: ${podName})`,
+                task: () =>
+                  self.kubectl2.waitForPod(constants.POD_STATUS_RUNNING, [
+                    'fullstack.hedera.com/type=network-node',
+                    `fullstack.hedera.com/node-name=${nodeId}`
+                  ], 1, 60 * 15, 1000) // timeout 15 minutes
+              })
+            }
+
+            // set up the sub-tasks
+            return task.newListr(subTasks, {
+              concurrent: false, // no need to run concurrently since if one node is up, the rest should be up by then
+              rendererOptions: {
+                collapseSubtasks: false
+              }
+            })
+          }
       }
     ], {
       concurrent: false,
@@ -207,10 +224,9 @@ export class ChartCommand extends BaseCommand {
       {
         title: 'Waiting for network pods to be ready',
         task: async (ctx, _) => {
-          const timeout = ctx.config.timeout || 900
           await this.kubectl2.waitForPod(constants.POD_STATUS_RUNNING, [
             'fullstack.hedera.com/type=network-node'
-          ], timeout)
+          ], 1)
         }
       }
     ], {

diff --git a/fullstack-network-manager/src/commands/node.mjs b/fullstack-network-manager/src/commands/node.mjs
@@ -22,7 +22,7 @@ export class NodeCommand extends BaseCommand {
     this.plaformInstaller = opts.platformInstaller
   }
 
-  async checkNetworkNodePod (namespace, nodeId, timeout = '300s') {
+  async checkNetworkNodePod (namespace, nodeId) {
     nodeId = nodeId.trim()
     const podName = Templates.renderNetworkPodName(nodeId)
 
@@ -164,7 +164,7 @@ export class NodeCommand extends BaseCommand {
             }
       },
       {
-        title: 'Extract platform software into network node',
+        title: 'Upload platform software into network nodes',
         task:
             async (ctx, task) => {
               const config = ctx.config
@@ -181,7 +181,7 @@ export class NodeCommand extends BaseCommand {
 
               // set up the sub-tasks
               return task.newListr(subTasks, {
-                concurrent: false,
+                concurrent: false, // parallel uploading of the zip file seems to be unreliable, so we just upload in sequence
                 rendererOptions: {
                   collapseSubtasks: false
                 }

diff --git a/fullstack-network-manager/src/core/kubectl2.mjs b/fullstack-network-manager/src/core/kubectl2.mjs
@@ -508,21 +508,15 @@ export class Kubectl2 {
    * @param status phase of the pod
    * @param labels pod labels
    * @param podCount number of pod expected
-   * @param timeout timeout in milliseconds
+   * @param maxAttempts maximum attempts to check
    * @param delay delay between checks in milliseconds
    * @return {Promise<boolean>}
    */
-  async waitForPod (status = 'Running', labels = [], podCount = 1, timeout = 1000, delay = 200) {
+  async waitForPod (status = 'Running', labels = [], podCount = 1, maxAttempts = 10, delay = 500) {
     const ns = this._getNamespace()
     const fieldSelector = `status.phase=${status}`
     const labelSelector = labels.join(',')
 
-    timeout = Number.parseInt(`${timeout}`)
-    if (timeout <= 0 || timeout < delay) {
-      throw new FullstackTestingError(`invalid timeout '${timeout}' and delay '${delay}'`)
-    }
-
-    const maxAttempts = Math.round(timeout / delay)
     this.logger.debug(`WaitForPod [${fieldSelector}, ${labelSelector}], maxAttempts: ${maxAttempts}`)
 
     // wait for the pod to be available with the given status and labels
@@ -534,7 +528,8 @@ export class Kubectl2 {
         false,
         undefined,
         fieldSelector,
-        labelSelector
+        labelSelector,
+        podCount
       )
 
       if (resp.body && resp.body.items && resp.body.items.length === podCount) {