[FLINK-35736][tests] Add migration test scripts & CI workflows

.github/workflows/flink_cdc.yml

@@ -115,19 +115,8 @@ jobs:
       - name: Run license check
         run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
 
-  migration_test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Compile snapshot CDC version
-        run: mvn --no-snapshot-updates -B install -DskipTests
-      - name: Run migration tests
-        run: cd flink-cdc-migration-tests && mvn clean verify
-
   compile_and_test:
+    needs: license_check
     # Only run the CI pipeline for the flink-cdc-connectors repository
 #    if: github.repository == 'apache/flink-cdc-connectors'
     runs-on: ubuntu-latest
@@ -263,3 +252,100 @@ jobs:
             done
           fi
           exit 0
+
+
+  migration_test_ut:
+    needs: license_check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Compile snapshot CDC version
+        run: mvn --no-snapshot-updates -B install -DskipTests
+      - name: Run migration tests
+        run: cd flink-cdc-migration-tests && mvn clean verify
+
+  pipeline_migration_test:
+    needs: migration_test_ut
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ '8', '11' ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: temurin
+          cache: maven
+      - name: Install dependencies
+        run: gem install terminal-table
+      - name: Prepare CDC versions
+        run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
+      - name: Prepare Flink distro
+        run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
+        working-directory: ./tools/mig-test
+      - name: Patch Flink configs
+        run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
+        working-directory: ./tools/mig-test
+      - name: Start containers
+        run: cd conf && docker-compose up -d
+        working-directory: ./tools/mig-test
+      - name: Run migration tests
+        run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
+        working-directory: ./tools/mig-test
+      - name: Stop containers
+        if: always()
+        run: cd conf && docker-compose down
+        working-directory: ./tools/mig-test
+
+  data_stream_migration_test:
+    needs: migration_test_ut
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ '8', '11' ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: temurin
+          cache: maven
+      - name: Install dependencies
+        run: gem install terminal-table
+      - name: Prepare CDC versions
+        run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
+      - name: Prepare Flink distro
+        run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
+        working-directory: ./tools/mig-test
+      - name: Patch Flink configs
+        run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
+        working-directory: ./tools/mig-test
+      - name: Compile Dummy DataStream Jobs
+        run: cd datastream && ruby compile_jobs.rb
+        working-directory: ./tools/mig-test
+      - name: Start containers
+        run: cd conf && docker-compose up -d
+        working-directory: ./tools/mig-test
+      - name: Run migration tests
+        run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
+        working-directory: ./tools/mig-test
+      - name: Stop containers
+        if: always()
+        run: cd conf && docker-compose down
+        working-directory: ./tools/mig-test

tools/mig-test/.gitignore

@@ -0,0 +1,8 @@
+*.sql
+savepoints/**
+cdc-versions/**
+cache/**
+.idea/**
+Gemfile.lock
+/logs/
+conf/temp.yaml

tools/mig-test/README.md

@@ -0,0 +1,38 @@
+# Flink CDC Migration Test Utilities
+
+## Pipeline Jobs
+### Preparation
+
+1. Install Ruby (macOS has embedded it by default)
+2. (Optional) Run `gem install terminal-table` for better display
+
+### Compile snapshot CDC versions
+3. Set `CDC_SOURCE_HOME` to the root directory of the Flink CDC git repository
+4. Go to `tools/mig-test` and run `ruby prepare_libs.rb` to download released / compile snapshot CDC versions
+
+### Run migration tests
+5. Enter `conf/` and run `docker compose up -d` to start up test containers
+6. Set `FLINK_HOME` to the home directory of Flink
+7. Go back to `tools/mig-test` and run `ruby run_migration_test.rb` to start testing
+
+### Result
+The migration result will be displayed in the console like this:
+
+```
++--------------------------------------------------------------------+
+|                       Migration Test Result                        |
++--------------+-------+-------+-------+--------------+--------------+
+|              | 3.0.0 | 3.0.1 | 3.1.0 | 3.1-SNAPSHOT | 3.2-SNAPSHOT |
+| 3.0.0        | ❓    | ❓    | ❌    | ✅           | ✅           |
+| 3.0.1        |       | ❓    | ❌    | ✅           | ✅           |
+| 3.1.0        |       |       | ✅    | ❌           | ❌           |
+| 3.1-SNAPSHOT |       |       |       | ✅           | ✅           |
+| 3.2-SNAPSHOT |       |       |       |              | ✅           |
++--------------+-------+-------+-------+--------------+--------------+
+```
+
+> ✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`
+
+## DataStream Jobs
+
+See `datastream/README.md`.

tools/mig-test/conf/docker-compose.yml

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3'
+services:
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "true"
+      MYSQL_DATABASE: fallen
+    ports:
+      - 3306:3306

tools/mig-test/conf/pipeline-route.yaml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source:
+  type: mysql
+  name: MySQL Source
+  hostname: localhost
+  port: 3306
+  username: root
+  password: ""
+  tables: fallen.\.*
+  server-id: 5400-5500
+  server-time-zone: UTC
+
+sink:
+  type: values
+  name: Values Sink
+
+pipeline:
+  name: ${PIPELINE_NAME}
+  parallelism: 1
+
+transform:
+  - source-table: fallen.\.*
+    projection: \*, 'extras' AS EXTRAS
+
+route:
+  - source-table: fallen.\.*
+    sink-table: fallen.terminus

tools/mig-test/conf/pipeline.yaml

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source:
+  type: mysql
+  name: MySQL Source
+  hostname: localhost
+  port: 3306
+  username: root
+  password: ""
+  tables: fallen.\.*
+  server-id: 5400-5500
+  server-time-zone: UTC
+
+sink:
+  type: values
+  name: Values Sink
+
+pipeline:
+  name: ${PIPELINE_NAME}
+  parallelism: 1

tools/mig-test/datastream/README.md

@@ -0,0 +1,11 @@
+# Flink CDC MigrationTestUtils
+
+## DataStream Jobs
+### Preparation
+
+1. Install Ruby (macOS has embedded it by default)
+2. (Optional) Run `gem install terminal-table` for better display
+
+### Compile DataStream Jobs
+3. Go to `tools/mig-test/datastream` and run `ruby compile_jobs.rb` to compile dummy DataStream jobs with specific version tags
+4. Then, run `ruby run_migration_test.rb` to start testing

tools/mig-test/datastream/compile_jobs.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JOB_VERSIONS = %w[2.4.2 3.0.0 3.0.1 3.1.0 3.1.1 3.2-SNAPSHOT]
+
+JOB_VERSIONS.each do |version|
+  puts "Compiling DataStream job for CDC #{version}"
+  `cd datastream-#{version} && mvn clean package -DskipTests`
+end
+
+puts 'Done'

tools/mig-test/datastream/datastream-2.4.2/.gitignore

@@ -0,0 +1,38 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store