diff --git a/.azure/gpu-integrations.yml b/.azure/gpu-integrations.yml
index 064f4fd9b57..53b2e25f158 100644
--- a/.azure/gpu-integrations.yml
+++ b/.azure/gpu-integrations.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         "torch | 1.x":
-          docker-image: "pytorchlightning/pytorch:1.13.1-cuda11.6-cudnn8-runtime"
+          docker-image: "pytorchlightning/torchmetrics:ubuntu22.04-cuda11.8.0-py3.9-torch1.13"
           torch-ver: "1.13.1"
           requires: "oldest"
         "torch | 2.x":
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index a3dc4180082..abaaeaa267d 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -94,32 +94,3 @@ jobs:
           push: ${{ env.PUSH_DOCKERHUB }}
           tags: "pytorchlightning/torchmetrics:ubuntu${{ matrix.ubuntu }}-cuda${{ matrix.cuda }}-py${{ matrix.python }}-torch${{ matrix.pytorch }}"
         timeout-minutes: 55
-
-  pytorch-cuda:
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        tag:
-          - "1.13.1-cuda11.6-cudnn8-runtime"
-          - "2.1.1-cuda12.1-cudnn8-runtime"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        if: env.PUSH_DOCKERHUB == 'true' && github.repository_owner == 'Lightning-AI'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build (and Push) runner
-        uses: docker/build-push-action@v5
-        with:
-          build-args: |
-            DOCKER_TAG=${{ matrix.tag }}
-          file: dockers/pytorch-cuda/Dockerfile
-          push: ${{ env.PUSH_DOCKERHUB }}
-          tags: "pytorchlightning/pytorch:${{ matrix.tag }}"
-        timeout-minutes: 35
diff --git a/dockers/pytorch-cuda/Dockerfile b/dockers/pytorch-cuda/Dockerfile
deleted file mode 100644
index ae0bac706b1..00000000000
--- a/dockers/pytorch-cuda/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright The Lightning AI team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG DOCKER_TAG=1.13.1-cuda11.6-cudnn8-runtime
-
-FROM pytorch/pytorch:${DOCKER_TAG}
-
-SHELL ["/bin/bash", "-c"]
-
-RUN \
-    apt-get update -q --fix-missing && \
-    apt list --upgradable && \
-    apt-get upgrade -y && \
-    # Cleaning
-    apt-get clean && \
-    rm -rf /root/.cache && \
-    rm -rf /var/lib/apt/lists/*