Merge master into feature/fts

.codecov.yml

@@ -10,7 +10,6 @@ coverage:
 ignore:
   - "pkg/cluster/api/dmpb"
   - "pkg/cluster/api/typeutil"
-  - "pkg/telemetry/*.pb.go"
 
 flags:
   tiup:

.github/workflows/integrate-cluster-cmd.yaml

@@ -30,41 +30,44 @@ on:
 
 jobs:
   cluster:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: true
       matrix:
         cases:
           - 'test_cmd'
-          # - 'test_cmd_tls_native_ssh'
+          - 'test_cmd_tls_native_ssh'
           - 'test_upgrade'
           - 'test_upgrade_tls'
           - 'test_tikv_cdc'
-    env:
-      working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.21
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-        id: go
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          path: go/src/github.com/${{ github.repository }}
+      - name: Set up Go 1.24
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.24.x
+        id: go
       - name: Build build_integration_test
-        working-directory: ${{ env.working-directory }}
         run: |
-          export GOPATH=${GITHUB_WORKSPACE}/go
           export PATH=$PATH:$GOPATH/bin
-          make build_integration_test
+          sudo apt-get update
+          sudo apt-get install -y build-essential python-is-python3 python3-pip ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+          echo \
+                "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+                $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+                sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-compose-plugin
+          make build_integration_test tiup
 
       - name: Build the docker-compose stack
-        working-directory: ${{ env.working-directory }}
-        # with --dev the first run will fail for unknow reason, just retry it and will success now..
-        run: TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./docker/up.sh --daemon --dev || TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./docker/up.sh --daemon --dev
+        run: TIUP_CLUSTER_ROOT=$(pwd) ./docker/up.sh --daemon
 
       - name: Check running containers
         run: |
@@ -74,7 +77,6 @@ jobs:
 
       - name: Run test suite
         id: test
-        working-directory: ${{ env.working-directory }}
         run: |
           # ensuere docker ssh is ready
           sleep 3
@@ -83,15 +85,12 @@ jobs:
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/run.sh ${{ matrix.cases }}
 
       - name: Collect component log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
-          ls ${{ env.working-directory }}
 
       - name: Detect error log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
@@ -104,22 +103,27 @@ jobs:
         with:
           overwrite: true
           name: component_logs
-          path: ${{ env.working-directory }}/logs/
+          path: ./logs
 
       - name: Output cluster debug log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
           pwd
           docker ps
           df -h
           free -h
-          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
+          find ./logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
-        working-directory: ${{ env.working-directory }}
         run: |
           curl -Os https://uploader.codecov.io/latest/linux/codecov
           chmod +x codecov
-          ./codecov -F cluster -s ${{ env.working-directory }}/tests/tiup-cluster/cover -f '*.out'
+          ./codecov -F cluster -s ./tests/tiup-cluster/cover -f '*.out'
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ failure() }}
+        with:
+          limit-access-to-actor: true
+

.github/workflows/integrate-cluster-scale.yaml

@@ -30,7 +30,7 @@ on:
 
 jobs:
   cluster:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: true
       matrix:
@@ -40,31 +40,36 @@ jobs:
           - 'test_scale_core_tls'
           - 'test_scale_tools_tls'
           - 'test_scale_tiproxy'
-    env:
-      working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.21
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-        id: go
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          path: go/src/github.com/${{ github.repository }}
+      - name: Set up Go 1.24
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.24.x
+        id: go
+
       - name: Build build_integration_test
-        working-directory: ${{ env.working-directory }}
         run: |
-          export GOPATH=${GITHUB_WORKSPACE}/go
           export PATH=$PATH:$GOPATH/bin
+          sudo apt-get update
+          sudo apt-get install -y build-essential python-is-python3 python3-pip ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+          echo \
+                "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+                $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+                sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-compose-plugin
           make build_integration_test
 
       - name: Build the docker-compose stack
-        working-directory: ${{ env.working-directory }}
         # with --dev the first run will fail for unknow reason, just retry it and will success now..
-        run: TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./docker/up.sh --daemon --dev || TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./docker/up.sh --daemon --dev
+        run: TIUP_CLUSTER_ROOT=$(pwd) ./docker/up.sh --daemon
 
       - name: Check running containers
         run: |
@@ -74,7 +79,6 @@ jobs:
 
       - name: Run test suite
         id: test
-        working-directory: ${{ env.working-directory }}
         run: |
           # ensuere docker ssh is ready
           sleep 5
@@ -83,17 +87,13 @@ jobs:
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/run.sh ${{ matrix.cases }}
 
       - name: Collect component log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
-          ls ${{ env.working-directory }}
 
       - name: Detect error log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
-        # if: always()
         run: |
           bash ./tests/tiup-cluster/script/detect_error.sh ./logs/
 
@@ -104,22 +104,20 @@ jobs:
         with:
           overwrite: true
           name: cluster_logs
-          path: ${{ env.working-directory }}/logs/
+          path: ./logs
 
       - name: Output cluster debug log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
           pwd
           docker ps
           df -h
           free -h
-          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
+          find ./logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
-        working-directory: ${{ env.working-directory }}
         run: |
           curl -Os https://uploader.codecov.io/latest/linux/codecov
           chmod +x codecov
-          ./codecov -F cluster -s ${{ env.working-directory }}/tests/tiup-cluster/cover -f '*.out'
+          ./codecov -F cluster -s ./tests/tiup-cluster/cover -f '*.out'

.github/workflows/integrate-dm.yaml

@@ -30,7 +30,7 @@ on:
 
 jobs:
   dm:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: true
       matrix:
@@ -39,35 +39,39 @@ jobs:
           - '--do-cases test_upgrade'
           - '--native-ssh --do-cases test_cmd'
           - '--native-ssh --do-cases test_upgrade'
-    env:
-      working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.21
+      - name: Set up Go 1.24
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.24.x
         id: go
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          path: go/src/github.com/${{ github.repository }}
 
       - name: Build build_integration_test
-        working-directory: ${{ env.working-directory }}
         run: |
-          export GOPATH=${GITHUB_WORKSPACE}/go
           export PATH=$PATH:$GOPATH/bin
+          sudo apt-get update
+          sudo apt-get install -y build-essential python-is-python3 python3-pip ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+          echo \
+                "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+                $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+                sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-compose-plugin
           make build_integration_test
           make tiup
 
       - name: Build the docker-compose stack
-        working-directory: ${{ env.working-directory }}
         # with --dev the first run will fail for unknow reason, just retry it and will success now..
         run: |
-          cd ${{ env.working-directory }}/docker
-          TIUP_CLUSTER_ROOT=${{ env.working-directory }} ./up.sh --daemon --dev --compose ./docker-compose.dm.yml
+          TIUP_CLUSTER_ROOT=$(pwd) ./docker/up.sh --daemon --compose ./docker-compose.dm.yml
 
       - name: Check running containers
         run: |
@@ -77,7 +81,6 @@ jobs:
 
       - name: Run test suite
         id: test
-        working-directory: ${{ env.working-directory }}
         run: |
           # ensuere docker ssh is ready
           sleep 4
@@ -86,13 +89,10 @@ jobs:
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-dm/run.sh ${{ matrix.cases }}
 
       - name: Collect component log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: ${{ failure() }}
         run: |
           docker exec tiup-cluster-control bash -c 'mkdir -p /tiup-cluster/logs; [[ -d ~/.tiup/logs ]] && find ~/.tiup/logs -type f -name "*.log" -exec cp {} /tiup-cluster/logs \; || true'
-          ls ${{ env.working-directory }}
-          tar czvf ${{ env.working-directory }}/dm.logs.tar.gz ${{ env.working-directory }}/logs/
 
       - name: Upload component log
         if: ${{ failure() }}
@@ -101,22 +101,20 @@ jobs:
         with:
           overwrite: true
           name: dm_logs
-          path: ${{ env.working-directory }}/dm.logs.tar.gz
+          path: ./logs
 
       - name: Output dm debug log
-        working-directory: ${{ env.working-directory }}
         if: ${{ failure() }}
         # if: always()
         run: |
           pwd
           docker ps
           df -h
           free -h
-          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
+          find ./logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
-        working-directory: ${{ env.working-directory }}
         run: |
           curl -Os https://uploader.codecov.io/latest/linux/codecov
           chmod +x codecov
-          ./codecov -F dm -s ${{ env.working-directory }}/tests/tiup-dm/cover -f '*.out'
+          ./codecov -F dm -s ./tests/tiup-dm/cover -f '*.out'