[CI] [cluster launcher] Fix switched "latest" and "nightly" gcp tests (…

…#40453)

The GCP "latest" and "nightly" tests had their names switched. This PR unswitches them.

The PR also adds more debug logs for the issue #40241.

---------

Signed-off-by: Archit Kulkarni <architkulkarni@users.noreply.github.com>

python/ray/autoscaler/_private/updater.py

@@ -2,6 +2,7 @@
 import os
 import subprocess
 import time
+import traceback
 from threading import Thread
 
 import click
@@ -164,16 +165,19 @@ def run(self):
 
             cli_logger.error("!!!")
             if hasattr(e, "cmd"):
+                stderr_output = getattr(e, "stderr", "No stderr available")
                 cli_logger.error(
-                    "Setup command `{}` failed with exit code {}. stderr:",
+                    "Setup command `{}` failed with exit code {}. stderr: {}",
                     cf.bold(e.cmd),
                     e.returncode,
+                    stderr_output,
                 )
             else:
-                cli_logger.verbose_error("{}", str(vars(e)))
+                cli_logger.verbose_error("Exception details: {}", str(vars(e)))
+                full_traceback = traceback.format_exc()
+                cli_logger.error("Full traceback: {}", full_traceback)
                 # todo: handle this better somehow?
-                cli_logger.error("{}", str(e))
-            # todo: print stderr here
+                cli_logger.error("Error message: {}", str(e))
             cli_logger.error("!!!")
             cli_logger.newline()
 

python/ray/autoscaler/launch_and_verify_cluster.py

@@ -17,6 +17,7 @@
 import sys
 import tempfile
 import time
+import traceback
 from pathlib import Path
 
 import boto3
@@ -187,7 +188,15 @@ def cleanup_cluster(cluster_config):
             return
         except subprocess.CalledProcessError as e:
             print(f"ray down fails[{i+1}/{num_tries}]: ")
-            print(e.output)
+            print(e.output.decode("utf-8"))
+
+            # Print full traceback
+            traceback.print_exc()
+
+            # Print stdout and stderr from ray down
+            print(f"stdout:\n{e.stdout.decode('utf-8')}")
+            print(f"stderr:\n{e.stderr.decode('utf-8')}")
+
             last_error = e
 
     raise last_error
@@ -220,6 +229,9 @@ def run_ray_commands(cluster_config, retries, no_config_cache, num_expected_node
         subprocess.run(cmd, check=True, capture_output=True)
     except subprocess.CalledProcessError as e:
         print(e.output)
+        # print stdout and stderr
+        print(f"stdout:\n{e.stdout.decode('utf-8')}")
+        print(f"stderr:\n{e.stderr.decode('utf-8')}")
         raise e
 
     print("======================================")

release/release_tests.yaml

@@ -6568,7 +6568,7 @@
 
   run:
     timeout: 3600
-    script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20  --docker-override nightly
+    script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20  --docker-override latest
 
 - name: gcp_cluster_launcher_nightly_image
   group: cluster-launcher-test
@@ -6586,7 +6586,7 @@
 
   run:
     timeout: 3600
-    script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20  --docker-override latest
+    script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20  --docker-override nightly
 
 
 - name: gcp_cluster_launcher_release_image