diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
index 1c55b7b96df..1ac25022713 100644
--- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
+++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
@@ -20,6 +20,7 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -55,15 +56,17 @@ public class Db2E2eITCase extends FlinkContainerTestEnvironment {
     private static final Path db2CdcJar = TestUtils.getResource("db2-cdc-connector.jar");
     private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
 
+    public static final String DB2_IMAGE = "ibmcom/db2";
+    public static final String DB2_CUSTOM_IMAGE = "custom/db2-cdc:1.4";
     private static final DockerImageName DEBEZIUM_DOCKER_IMAGE_NAME =
             DockerImageName.parse(
-                            new ImageFromDockerfile("custom/db2-cdc:1.4")
+                            new ImageFromDockerfile(DB2_CUSTOM_IMAGE)
                                     .withDockerfile(getFilePath("docker/db2/Dockerfile"))
                                     .get())
-                    .asCompatibleSubstituteFor("ibmcom/db2");
+                    .asCompatibleSubstituteFor(DB2_IMAGE);
     private static boolean db2AsnAgentRunning = false;
 
-    private Db2Container db2Container;
+    private static Db2Container db2Container;
 
     @Before
     public void before() {
@@ -111,6 +114,23 @@ public void after() {
         super.after();
     }
 
+    @AfterClass
+    public static void afterClass() {
+        // Cleanup the db2 image, because it's too large and will cause the next test to fail.
+        db2Container.getDockerClient().removeImageCmd(DB2_CUSTOM_IMAGE).exec();
+        db2Container
+                .getDockerClient()
+                .listImagesCmd()
+                .withImageNameFilter(DB2_IMAGE)
+                .exec()
+                .forEach(
+                        image ->
+                                db2Container
+                                        .getDockerClient()
+                                        .removeImageCmd(image.getId())
+                                        .exec());
+    }
+
     @Test
     public void testDb2CDC() throws Exception {
         List<String> sqlLines =
diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
index 11d185566c2..e7a53396e77 100644
--- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
+++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
@@ -20,6 +20,7 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -53,13 +54,13 @@ public class OracleE2eITCase extends FlinkContainerTestEnvironment {
 
     private static final Path oracleCdcJar = TestUtils.getResource("oracle-cdc-connector.jar");
     private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
-
-    public OracleContainer oracle;
+    private static OracleContainer oracle;
 
     @Before
     public void before() {
         super.before();
         LOG.info("Starting containers...");
+
         oracle =
                 new OracleContainer(ORACLE_IMAGE)
                         .withNetwork(NETWORK)
@@ -77,6 +78,16 @@ public void after() {
         super.after();
     }
 
+    @AfterClass
+    public static void afterClass() {
+        // Cleanup the oracle image, because it's too large and will cause the next test to fail.
+        oracle.getDockerClient()
+                .listImagesCmd()
+                .withImageNameFilter(ORACLE_IMAGE)
+                .exec()
+                .forEach(image -> oracle.getDockerClient().removeImageCmd(image.getId()).exec());
+    }
+
     @Test
     public void testOracleCDC() throws Exception {
         List<String> sqlLines =
diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
index 4dacfaa665d..a31a3c75477 100644
--- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
+++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
@@ -20,12 +20,14 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.MSSQLServerContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.lifecycle.Startables;
@@ -58,6 +60,7 @@ public class SqlServerE2eITCase extends FlinkContainerTestEnvironment {
     private static final Path sqlServerCdcJar =
             TestUtils.getResource("sqlserver-cdc-connector.jar");
     private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
+    public static final String MSSQL_SERVER_IMAGE = "mcr.microsoft.com/mssql/server:2019-latest";
 
     @Parameterized.Parameter(1)
     public boolean parallelismSnapshot;
@@ -75,7 +78,7 @@ public static List<Object[]> parameters() {
 
     @Rule
     public MSSQLServerContainer sqlServer =
-            new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest")
+            new MSSQLServerContainer<>(MSSQL_SERVER_IMAGE)
                     .withPassword("Password!")
                     .withEnv("MSSQL_AGENT_ENABLED", "true")
                     .withEnv("MSSQL_PID", "Standard")
@@ -100,6 +103,12 @@ public void after() {
         super.after();
     }
 
+    @AfterClass
+    public static void afterClass() {
+        // Cleanup the sqlserver image, because it's too large and will cause the next test to fail.
+        DockerClientFactory.instance().client().removeImageCmd(MSSQL_SERVER_IMAGE).exec();
+    }
+
     @Test
     public void testSqlServerCDC() throws Exception {
         List<String> sqlLines =
diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
index 5a117cc1b5e..8a4756786c7 100644
--- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
+++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
@@ -20,6 +20,7 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -141,6 +142,16 @@ public void after() {
         super.after();
     }
 
+    @AfterClass
+    public static void afterClass() {
+        // Cleanup the TIDB image, because it's too large and will cause the next test to fail.
+        TIDB.getDockerClient()
+                .listImagesCmd()
+                .withImageNameFilter("pingcap/tikv")
+                .exec()
+                .forEach(image -> TIDB.getDockerClient().removeImageCmd(image.getId()).exec());
+    }
+
     @Test
     public void testTIDBCDC() throws Exception {
         List<String> sqlLines =
diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
index 34273f61195..ab676e8e68d 100644
--- a/tools/azure-pipelines/jobs-template.yml
+++ b/tools/azure-pipelines/jobs-template.yml
@@ -52,7 +52,7 @@ jobs:
         displayName: "Set JDK"
       # Compile
       - script: |
-          ./tools/ci/compile.sh default || exit $?
+          ./tools/ci/compile.sh || exit $?
           ./tools/azure-pipelines/create_build_artifact.sh
         displayName: Compile