Merge pull request #5649 from lichunzhu/merge-master

feature branch update: merge master into release-multi-branch 220530

.codecov.yml

@@ -51,3 +51,8 @@ flags:
     carryforward: true
     paths:
       - "dm/"
+
+  engine:
+    carryforward: true
+    paths:
+      - "engine/"

.github/ISSUE_TEMPLATE/engine-bug-report.yml

@@ -0,0 +1,39 @@
+name: "\U0001F41B Dataflow Engine Bug Report"
+description: Something isn't working as expected
+labels: ["type/bug","area/engine"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please answer these questions before submitting your issue. Thanks!
+  - type: textarea
+    id: problem
+    attributes:
+      label: What did you do?
+      description: If possible, provide a recipe for reproducing the error.
+    validations:
+      required: true
+  - type: textarea
+    id: expect
+    attributes:
+      label: What did you expect to see?
+    validations:
+      required: false
+  - type: textarea
+    id: see
+    attributes:
+      label: What did you see instead?
+    validations:
+      required: true
+  - type: textarea
+    id: version
+    attributes:
+      label: Versions of the cluster
+      value: |-
+             Dataflow Engine version:
+
+             ```console
+             (paste Dataflow Engine version here)
+             ```
+    validations:
+      required: true

.github/pull_request_template.md

@@ -1,5 +1,6 @@
 <!--
-Thank you for contributing to TiDB-CDC! Please read MD's [CONTRIBUTING](https://github.com/pingcap/tidb-cdc/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+Thank you for contributing to TiFlow! 
+Please read MD's [CONTRIBUTING](https://github.com/pingcap/tiflow/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
 -->
 
 ### What problem does this PR solve?
@@ -19,35 +20,23 @@ Issue Number: close #xxx
 
 ### Check List <!--REMOVE the items that are not applicable-->
 
-Tests <!-- At least one of them must be included. -->
+#### Tests <!-- At least one of them must be included. -->
 
  - Unit test
  - Integration test
  - Manual test (add detailed scripts or steps below)
  - No code
 
-Code changes
+#### Questions <!-- Authors should answer these questions and reviewers should consider these questions. -->
 
- - Has exported function/method change
- - Has exported variable/fields change
- - Has interface methods change
- - Has persistent data change
+##### Will it cause performance regression or break compatibility?
 
-Side effects
-
- - Possible performance regression
- - Increased code complexity
- - Breaking backward compatibility
-
-Related changes
-
- - Need to cherry-pick to the release branch
- - Need to update the documentation
- - Need to update key monitor metrics in both TiCDC document and official document
+##### Do you need to update user documentation, design documentation or monitoring documentation?
 
 ### Release note <!-- bugfixes or new feature need a release note -->
 
 ```release-note
-Please add a release note.
+Please refer to [Release Notes Language Style Guide](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/release-notes-style-guide.html) to write a quality release note.
+
 If you don't think this PR needs a release note then fill it with `None`.
 ```

.github/workflows/check_and_build.yaml

@@ -29,6 +29,9 @@ jobs:
       - name: Build-dm
         run: docker build -f ./dm/Dockerfile .
 
+      - name: Build-engine
+        run: docker build -f ./engine/Dockerfile .
+
   mac_build:
     name: Mac OS Build
     runs-on: macos-latest

.github/workflows/dataflow_engine_e2e.yaml

@@ -0,0 +1,180 @@
+name: Dataflow Engine e2e tests
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'engine/**'
+      - 'dm/**'
+
+# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  Basic-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml -f $GITHUB_WORKSPACE/engine/sample/demo.yaml up -d
+
+      - name: Run tests
+        run: |
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestSubmitTest
+
+      - name: Dump docker container logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          tail: '100'
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: 'logs'
+
+      - name: Upload logs to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@master
+        with:
+          name: basic-workflow-logs
+          path: logs/*
+
+  Node-failure-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml up -d
+
+      - name: Run tests
+        run: |
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestNodeFailure
+
+      - name: Dump docker container logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          tail: '100'
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: 'logs'
+
+      - name: Upload logs to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@master
+        with:
+          name: node-failure-workflow-logs
+          path: logs/*
+
+  Worker-error-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml up -d
+
+      - name: Run tests
+        run: |
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestWorkerExit
+
+      - name: Dump docker container logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          tail: '100'
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: 'logs'
+
+      - name: Upload logs to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@master
+        with:
+          name: worker-errror-workflow-logs
+          path: logs/*
+
+  DM-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml -f $GITHUB_WORKSPACE/engine/sample/dm_databases.yaml up -d
+
+      - name: Run tests
+        run: |
+          $GITHUB_WORKSPACE/engine/test/utils/wait_mysql_online.sh --host 127.0.0.1 --port 3306 --password 123456
+          $GITHUB_WORKSPACE/engine/test/utils/wait_mysql_online.sh --host 127.0.0.1 --port 4000
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestDMJob
+
+      - name: Dump docker container logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          tail: '100'
+
+      - name: Collect docker logs on failure
+        if: ${{ failure() }}
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: 'logs'
+
+      - name: Upload logs to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@master
+        with:
+          name: dm-workflow-logs
+          path: logs/*

.github/workflows/dataflow_engine_ut.yaml

@@ -0,0 +1,35 @@
+name: Dataflow engine unit test
+
+env:
+  PROTOC_VERSION: 3.8.0
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'engine/**'
+
+# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  unit_test:
+    name: Make engine_unit_test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup Go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Test
+        run: make engine_unit_test

.github/workflows/dm_binlog_999999.yaml

@@ -121,9 +121,3 @@ jobs:
         uses: Ilshidur/action-slack@2.1.0
         with:
           args: "binlog 999999 failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"
-
-      # Debug via SSH if previous steps failed
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 30

.github/workflows/dm_chaos.yaml

@@ -305,9 +305,3 @@ jobs:
         uses: Ilshidur/action-slack@2.1.0
         with:
           args: "chaos job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"
-
-      # Debug via SSH if previous steps failed
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 15

.github/workflows/dm_upstream_switch.yaml

@@ -82,9 +82,3 @@ jobs:
         uses: Ilshidur/action-slack@2.1.0
         with:
           args: "upstream-switch job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"
-
-      # Debug via SSH if previous steps failed
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 30

.github/workflows/ticdc_integration.yaml

@@ -8,11 +8,13 @@ on:
       - master
     paths-ignore:
       - 'dm/**'
+      - 'engine/**'
   pull_request:
     branches:
       - master
     paths-ignore:
       - 'dm/**'
+      - 'engine/**'
 
 # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
 concurrency: