diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d441ae246b..34d33b3120 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -28,6 +28,10 @@ jobs:
       SUDO: ${{ matrix.user.isRoot}}
     steps:
     - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.DEFAULT_RUBY_VERSION }}
+        bundler-cache: true
     - run: ./dev/ci/setup-host cxx
     - run: ./dev/ci/run-tests-with-docker cxx
       if: matrix.os == 'ubuntu-latest'
@@ -63,6 +67,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.DEFAULT_RUBY_VERSION }}
+        bundler-cache: true
     - run: ./dev/ci/setup-host ${{ matrix.integration.label }}
     - run: ./dev/ci/run-tests-with-docker ${{ matrix.integration.label }}
       if: matrix.os == 'ubuntu-latest'
@@ -92,6 +100,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.DEFAULT_RUBY_VERSION }}
+        bundler-cache: true
     - run: ./dev/ci/setup-host ${{matrix.lang.name}}
     - run: ./dev/ci/run-tests-with-docker ${{matrix.lang.name}}
       if: matrix.os == 'ubuntu-latest'
@@ -109,6 +121,10 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.DEFAULT_RUBY_VERSION }}
+        bundler-cache: true
     - run: ./dev/ci/setup-host homebrew-packaging
     - run: ./dev/ci/run-tests-natively homebrew-packaging
     - uses: actions/upload-artifact@v4
@@ -123,6 +139,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.DEFAULT_RUBY_VERSION }}
+        bundler-cache: true
     - run: ./dev/ci/setup-host source-packaging
     - run: ./dev/ci/run-tests-with-docker source-packaging
     - uses: actions/upload-artifact@v4
diff --git a/dev/ci/lib/set-container-envvars.sh b/dev/ci/lib/set-container-envvars.sh
index 0c3b2cc024..face681048 100644
--- a/dev/ci/lib/set-container-envvars.sh
+++ b/dev/ci/lib/set-container-envvars.sh
@@ -36,17 +36,21 @@ else
 fi
 
 if [[ -f ~/.rvm/scripts/rvm ]]; then
-	# shellcheck source=/dev/null
-	source ~/.rvm/scripts/rvm
-else
-	# shellcheck source=/dev/null
-	source /usr/local/rvm/scripts/rvm
-fi
-
-if [[ "$TEST_RUBY_VERSION" != "" ]]; then
+    # shellcheck source=/dev/null
+    source ~/.rvm/scripts/rvm
+    if [[ "$TEST_RUBY_VERSION" != "" ]]; then
+	header2 "Using Ruby version $TEST_RUBY_VERSION"
+	run rvm use "$TEST_RUBY_VERSION"
+	echo
+    fi
+elif [[ -f /usr/local/rvm/scripts/rvm ]]; then
+    # shellcheck source=/dev/null
+    source /usr/local/rvm/scripts/rvm
+    if [[ "$TEST_RUBY_VERSION" != "" ]]; then
 	header2 "Using Ruby version $TEST_RUBY_VERSION"
 	run rvm use "$TEST_RUBY_VERSION"
 	echo
+    fi
 fi
 
 # RVM's cd override causes problems (probably thanks to bash