dumpling: better github action (#37090)

pingcap · Aug 15, 2022 · 8cbc4a4 · 8cbc4a4
1 parent 6fb3e06
commit 8cbc4a4
Show file tree

Hide file tree

Showing 6 changed files with 89 additions and 135 deletions.
diff --git a/.github/workflows/integration-test-dumpling-common.yml b/.github/workflows/integration-test-dumpling-common.yml
@@ -0,0 +1,56 @@
+name: DumplingCommon
+on:
+  workflow_call:
+    inputs:
+      debug:
+        type: boolean
+        description: 'set tmate on failure'
+        required: true
+      mysql_version:
+        type: string
+        description: 'specify mysql version'
+        required: true
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+    services:
+      mysql:
+        image: mysql:${{ inputs.mysql_version }}
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+      - name: 'checkout repository'
+        uses: actions/checkout@v3
+      - name: 'set up golang'
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - name: 'try to use build cache'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/Library/Caches/go-build
+            ~\AppData\Local\go-build
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: 'download binary dependencies'
+        run: sh dumpling/install.sh
+      - name: 'build tidb'
+        run: make server
+      - name: 'build lightning'
+        run: make build_lightning
+      - name: 'integration test'
+        run: make dumpling_integration_test VERBOSE="true"
+      - name: 'set up tmate session if necessary'
+        if: ${{ failure() && inputs.debug }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/integration-test-dumpling.yml b/.github/workflows/integration-test-dumpling.yml
@@ -1,5 +1,12 @@
 name: Dumpling
 on:
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled'
+        required: false
+        default: false
   push:
     branches:
       - master
@@ -32,113 +39,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  integration-test-mysql-5735:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:5.7.35
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-5735:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 5.7.35
 
-  integration-test-mysql-8026:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:8.0.26
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-8022:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 8.0.22
 
-  integration-test-mysql-8022:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:8.0.22
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-8026:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 8.0.26
diff --git a/Makefile b/Makefile
@@ -371,7 +371,8 @@ dumpling_unit_test_in_verify_ci: failpoint-enable tools/bin/gotestsum
 	$(RACE_FLAG) -coverprofile="$(TEST_COVERAGE_DIR)/dumpling_cov.unit_test.out" || ( make failpoint-disable && exit 1 )
 	@make failpoint-disable
 
-dumpling_integration_test: dumpling_bins failpoint-enable build_dumpling
+dumpling_integration_test: dumpling_bins failpoint-enable
+	@make build_dumpling
 	@make failpoint-disable
 	./dumpling/tests/run.sh $(CASE)
 

diff --git a/br/pkg/version/version.go b/br/pkg/version/version.go
@@ -387,16 +387,14 @@ func ParseServerInfo(src string) ServerInfo {
 	var err error
 	serverInfo.ServerVersion, err = semver.NewVersion(versionStr)
 	if err != nil {
-		log.L().Warn("fail to parse version",
+		log.L().Warn("fail to parse version, fallback to 0.0.0",
 			zap.String("version", versionStr))
+		serverInfo.ServerVersion = semver.New("0.0.0")
 	}
-	var version string
-	if serverInfo.ServerVersion != nil {
-		version = serverInfo.ServerVersion.String()
-	}
+
 	log.L().Info("detect server version",
 		zap.String("type", serverInfo.ServerType.String()),
-		zap.String("version", version))
+		zap.String("version", serverInfo.ServerVersion.String()))
 
 	return serverInfo
 }
diff --git a/br/pkg/version/version_test.go b/br/pkg/version/version_test.go
@@ -455,7 +455,7 @@ func TestDetectServerInfo(t *testing.T) {
 		{3, "5.7.25-TiDB-v4.0.0-alpha-1263-g635f2e1af", ServerTypeTiDB, mkVer(4, 0, 0, "alpha-1263-g635f2e1af")},
 		{4, "5.7.25-TiDB-v3.0.7-58-g6adce2367", ServerTypeTiDB, mkVer(3, 0, 7, "58-g6adce2367")},
 		{5, "5.7.25-TiDB-3.0.6", ServerTypeTiDB, mkVer(3, 0, 6, "")},
-		{6, "invalid version", ServerTypeUnknown, (*semver.Version)(nil)},
+		{6, "invalid version", ServerTypeUnknown, mkVer(0, 0, 0, "")},
 		{7, "Release Version: v5.2.1\nEdition: Community\nGit Commit Hash: cd8fb24c5f7ebd9d479ed228bb41848bd5e97445", ServerTypeTiDB, mkVer(5, 2, 1, "")},
 		{8, "Release Version: v5.4.0-alpha-21-g86caab907\nEdition: Community\nGit Commit Hash: 86caab907c481bbc4243b5a3346ec13907cc8721\nGit Branch: master", ServerTypeTiDB, mkVer(5, 4, 0, "alpha-21-g86caab907")},
 	}

diff --git a/dumpling/install.sh b/dumpling/install.sh
@@ -1,28 +1,13 @@
 #!/bin/sh
-
 set -e
-TAG="nightly"
-pwd=$(pwd)
+
 mkdir -p bin/
 
 # download lightning and sync_diff_inspector
-wget http://download.pingcap.org/tidb-toolkit-$TAG-linux-amd64.tar.gz -O tools.tar.gz
+TOOLS_TAG="nightly"
+wget http://download.pingcap.org/tidb-toolkit-$TOOLS_TAG-linux-amd64.tar.gz -O tools.tar.gz
 tar -xzvf tools.tar.gz
-mv tidb-toolkit-$TAG-linux-amd64/bin/* bin/
-
-LIGHTNING_TAG="master"
-# download tidb-lightning
-git clone -b $LIGHTNING_TAG https://github.com/pingcap/tidb-lightning
-cd $pwd/tidb-lightning && make
-cd $pwd
-mv tidb-lightning/bin/tidb-lightning bin/
-
-TIDB_TAG="master"
-# download tidb-server
-git clone -b $TIDB_TAG https://github.com/pingcap/tidb
-cd $pwd/tidb && make
-cd $pwd
-mv tidb/bin/tidb-server bin/
+mv tidb-toolkit-$TOOLS_TAG-linux-amd64/bin/* bin/
 
 # download minio
 wget https://dl.min.io/server/minio/release/linux-amd64/minio -O bin/minio