diff --git a/.gitignore b/.gitignore index 0deb8a2183978..68145bc905a40 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ bazel-bin bazel-out bazel-testlogs bazel-tidb +MODULE.bazel.lock .ijwb/ /oom_record/ *.log.json diff --git a/DEPS.bzl b/DEPS.bzl index 98aa97dde172c..097a57b33d9a6 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -437,13 +437,13 @@ def go_deps(): name = "com_github_aws_aws_sdk_go", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go", - sha256 = "a34e669cf2f2caa9dab2f4db1c2e4445c3be627f3f3086a82fc678287adb28a8", - strip_prefix = "github.com/aws/aws-sdk-go@v1.48.14", + sha256 = "626ad62e145c8499afb67cd13b438e4a2d5b855ac2dd94c87f5e72e1d0e53365", + strip_prefix = "github.com/aws/aws-sdk-go@v1.50.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.48.14.zip", - "http://ats.apps.svc/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.48.14.zip", - "https://cache.hawkingrei.com/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.48.14.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.48.14.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.50.0.zip", + "http://ats.apps.svc/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.50.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.50.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.50.0.zip", ], ) go_repository( @@ -463,78 +463,65 @@ def go_deps(): name = "com_github_azure_azure_sdk_for_go_sdk_azcore", build_file_proto_mode = "disable_global", importpath = "github.com/Azure/azure-sdk-for-go/sdk/azcore", - sha256 = "bdca5cf77bf71564df8f4cc53b607da0a966c5d8611a2fc0cdfee4acc2bf8cc1", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/azcore@v1.9.0", + sha256 = "bd9b4242622d1d8825cdc2071391a15fb29d4a8844b38bc081410bfec2bca7a3", + strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/azcore@v1.9.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.0.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.1.zip", + "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/azcore/com_github_azure_azure_sdk_for_go_sdk_azcore-v1.9.1.zip", ], ) go_repository( name = "com_github_azure_azure_sdk_for_go_sdk_azidentity", build_file_proto_mode = "disable_global", importpath = "github.com/Azure/azure-sdk-for-go/sdk/azidentity", - sha256 = "39566249254f05e58d8a8a1324cd44c0545ca4091b34d5d86dfb832062b8302c", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/azidentity@v1.4.0", + sha256 = "295c5cbaeb0d7599b6439b5c65cf1bf6d4becc07f9f722f724263887b7de1f00", + strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/azidentity@v1.5.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.4.0.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.4.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.4.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.4.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.5.1.zip", + "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.5.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.5.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/azidentity/com_github_azure_azure_sdk_for_go_sdk_azidentity-v1.5.1.zip", ], ) go_repository( name = "com_github_azure_azure_sdk_for_go_sdk_internal", build_file_proto_mode = "disable_global", importpath = "github.com/Azure/azure-sdk-for-go/sdk/internal", - sha256 = "d7e0270a6da5d9d2e2a6f799d366080f1a6b038ccbe76d036131a0da0835aff8", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/internal@v1.5.0", + sha256 = "67928374bb9f18d720ab15171565c92405c488d1cfbf8dbff25f8b0f0fefe067", + strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/internal@v1.5.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.0.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.1.zip", + "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/internal/com_github_azure_azure_sdk_for_go_sdk_internal-v1.5.1.zip", ], ) go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v4", + name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5", build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4", - sha256 = "b0c3b75b9e8fc156c488016d93e411f3089b5b97cd8250ac30a4746a558d3b62", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4@v4.2.1", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5", + sha256 = "f47869d09394fbe965b013f539bf7c1c65af9833dbfea0c12f7b6a081870b6f6", + strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5@v5.4.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v4-v4.2.1.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v4-v4.2.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v4-v4.2.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v4-v4.2.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5-v5.4.0.zip", + "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5-v5.4.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5-v5.4.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5-v5.4.0.zip", ], ) go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork", + name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4", build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork", - sha256 = "ea444a1c3fcddb0477f7d1df7716c4d9a9edf5d89b12bbd5c92e89c036a1c01b", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork@v1.1.0", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4", + sha256 = "e002f35fb0d7200d8cd31bc6b1e91e56400ddc3f50887e8795285268ac5bdaff", + strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4@v4.3.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork-v1.1.0.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork-v1.1.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork-v1.1.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork-v1.1.0.zip", - ], - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v2", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2", - sha256 = "4e0253514cf7072a29ddb22adf71cea03a44935a05de3897910a3932ae0034e3", - strip_prefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2@v2.2.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v2-v2.2.1.zip", - "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v2-v2.2.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v2-v2.2.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v2-v2.2.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4-v4.3.0.zip", + "http://ats.apps.svc/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4-v4.3.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4-v4.3.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4/com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4-v4.3.0.zip", ], ) go_repository( @@ -567,13 +554,13 @@ def go_deps(): name = "com_github_azuread_microsoft_authentication_library_for_go", build_file_proto_mode = "disable_global", importpath = "github.com/AzureAD/microsoft-authentication-library-for-go", - sha256 = "6f933f00d5310409c8f3fe25917c3c48abb94fa9c582a9ce6ae35eaafe80d06c", - strip_prefix = "github.com/AzureAD/microsoft-authentication-library-for-go@v1.1.1", + sha256 = "6eb2846f7cbcd9091ac954aed668575138ccb064b6c70eff1c64a3524beebc6a", + strip_prefix = "github.com/AzureAD/microsoft-authentication-library-for-go@v1.2.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.1.1.zip", - "http://ats.apps.svc/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.1.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.1.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.1.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.2.1.zip", + "http://ats.apps.svc/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.2.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.2.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/AzureAD/microsoft-authentication-library-for-go/com_github_azuread_microsoft_authentication_library_for_go-v1.2.1.zip", ], ) go_repository( @@ -602,6 +589,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/bazelbuild/rules_go/com_github_bazelbuild_rules_go-v0.42.1-0.20231101215950-df20c987afcb.zip", ], ) + go_repository( + name = "com_github_bboreham_go_loser", + build_file_proto_mode = "disable_global", + importpath = "github.com/bboreham/go-loser", + sha256 = "d39c329a916c6e3af23a77ed3615f49726258cf3fa7b126c5ad06e7a5c3cbb4f", + strip_prefix = "github.com/bboreham/go-loser@v0.0.0-20230920113527-fcc2c21820a3", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/bboreham/go-loser/com_github_bboreham_go_loser-v0.0.0-20230920113527-fcc2c21820a3.zip", + "http://ats.apps.svc/gomod/github.com/bboreham/go-loser/com_github_bboreham_go_loser-v0.0.0-20230920113527-fcc2c21820a3.zip", + "https://cache.hawkingrei.com/gomod/github.com/bboreham/go-loser/com_github_bboreham_go_loser-v0.0.0-20230920113527-fcc2c21820a3.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/bboreham/go-loser/com_github_bboreham_go_loser-v0.0.0-20230920113527-fcc2c21820a3.zip", + ], + ) go_repository( name = "com_github_benbjohnson_clock", build_file_proto_mode = "disable_global", @@ -992,6 +992,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/chzyer/test/com_github_chzyer_test-v0.0.0-20180213035817-a1ea475d72b1.zip", ], ) + go_repository( + name = "com_github_cilium_ebpf", + build_file_proto_mode = "disable_global", + importpath = "github.com/cilium/ebpf", + sha256 = "5130d073e34b07b80ecf6cccb1ff4b20ecee59bf1b5b71272ef9e90949a85952", + strip_prefix = "github.com/cilium/ebpf@v0.11.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/cilium/ebpf/com_github_cilium_ebpf-v0.11.0.zip", + "http://ats.apps.svc/gomod/github.com/cilium/ebpf/com_github_cilium_ebpf-v0.11.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/cilium/ebpf/com_github_cilium_ebpf-v0.11.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cilium/ebpf/com_github_cilium_ebpf-v0.11.0.zip", + ], + ) go_repository( name = "com_github_client9_misspell", build_file_proto_mode = "disable_global", @@ -1200,6 +1213,32 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/colinmarc/hdfs/v2/com_github_colinmarc_hdfs_v2-v2.1.1.zip", ], ) + go_repository( + name = "com_github_containerd_cgroups_v3", + build_file_proto_mode = "disable_global", + importpath = "github.com/containerd/cgroups/v3", + sha256 = "bc22dd9c675f1abd77a9de83506abb15656529848b2274323a88ea9bfce980be", + strip_prefix = "github.com/containerd/cgroups/v3@v3.0.3", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/containerd/cgroups/v3/com_github_containerd_cgroups_v3-v3.0.3.zip", + "http://ats.apps.svc/gomod/github.com/containerd/cgroups/v3/com_github_containerd_cgroups_v3-v3.0.3.zip", + "https://cache.hawkingrei.com/gomod/github.com/containerd/cgroups/v3/com_github_containerd_cgroups_v3-v3.0.3.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/containerd/cgroups/v3/com_github_containerd_cgroups_v3-v3.0.3.zip", + ], + ) + go_repository( + name = "com_github_containerd_log", + build_file_proto_mode = "disable_global", + importpath = "github.com/containerd/log", + sha256 = "2008faf206ec820e7fc3d40baba924936c21347dafad4a7ff122fa90e26e57d7", + strip_prefix = "github.com/containerd/log@v0.1.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/containerd/log/com_github_containerd_log-v0.1.0.zip", + "http://ats.apps.svc/gomod/github.com/containerd/log/com_github_containerd_log-v0.1.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/containerd/log/com_github_containerd_log-v0.1.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/containerd/log/com_github_containerd_log-v0.1.0.zip", + ], + ) go_repository( name = "com_github_coocood_bbloom", build_file_proto_mode = "disable_global", @@ -1555,13 +1594,26 @@ def go_deps(): name = "com_github_digitalocean_godo", build_file_proto_mode = "disable_global", importpath = "github.com/digitalocean/godo", - sha256 = "cb363a73dc6524f61a8c349c9071470d1f709b0911f5faccbf6073da73bb9393", - strip_prefix = "github.com/digitalocean/godo@v1.106.0", + sha256 = "05d6f193e8313915fbea712226612a8d0bdc8e61975564262982b6ca106d38ee", + strip_prefix = "github.com/digitalocean/godo@v1.108.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.106.0.zip", - "http://ats.apps.svc/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.106.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.106.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.106.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.108.0.zip", + "http://ats.apps.svc/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.108.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.108.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/digitalocean/godo/com_github_digitalocean_godo-v1.108.0.zip", + ], + ) + go_repository( + name = "com_github_distribution_reference", + build_file_proto_mode = "disable_global", + importpath = "github.com/distribution/reference", + sha256 = "d812d0281581beb04facbd0ca03bc529ae7de484f959ade09765c1af532e1b7c", + strip_prefix = "github.com/distribution/reference@v0.5.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/distribution/reference/com_github_distribution_reference-v0.5.0.zip", + "http://ats.apps.svc/gomod/github.com/distribution/reference/com_github_distribution_reference-v0.5.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/distribution/reference/com_github_distribution_reference-v0.5.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/distribution/reference/com_github_distribution_reference-v0.5.0.zip", ], ) go_repository( @@ -1590,30 +1642,17 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/dnaeon/go-vcr/com_github_dnaeon_go_vcr-v1.2.0.zip", ], ) - go_repository( - name = "com_github_docker_distribution", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/distribution", - sha256 = "9e0a17bbcaa1419232cd44e3a79209be26d9ccfa079e32e0e9999c81c0991477", - strip_prefix = "github.com/docker/distribution@v2.8.2+incompatible", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/docker/distribution/com_github_docker_distribution-v2.8.2+incompatible.zip", - "http://ats.apps.svc/gomod/github.com/docker/distribution/com_github_docker_distribution-v2.8.2+incompatible.zip", - "https://cache.hawkingrei.com/gomod/github.com/docker/distribution/com_github_docker_distribution-v2.8.2+incompatible.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/docker/distribution/com_github_docker_distribution-v2.8.2+incompatible.zip", - ], - ) go_repository( name = "com_github_docker_docker", build_file_proto_mode = "disable_global", importpath = "github.com/docker/docker", - sha256 = "a9bf4b53188caf41ada5a5490d99c44e808d2f4a725028e2717d9b31c87f5002", - strip_prefix = "github.com/docker/docker@v24.0.7+incompatible", + sha256 = "943382432be5c45b64f779becb4d2c76a67727452b38a17a197a3c5d939f9cdc", + strip_prefix = "github.com/docker/docker@v25.0.0+incompatible", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/docker/docker/com_github_docker_docker-v24.0.7+incompatible.zip", - "http://ats.apps.svc/gomod/github.com/docker/docker/com_github_docker_docker-v24.0.7+incompatible.zip", - "https://cache.hawkingrei.com/gomod/github.com/docker/docker/com_github_docker_docker-v24.0.7+incompatible.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/docker/docker/com_github_docker_docker-v24.0.7+incompatible.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/docker/docker/com_github_docker_docker-v25.0.0+incompatible.zip", + "http://ats.apps.svc/gomod/github.com/docker/docker/com_github_docker_docker-v25.0.0+incompatible.zip", + "https://cache.hawkingrei.com/gomod/github.com/docker/docker/com_github_docker_docker-v25.0.0+incompatible.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/docker/docker/com_github_docker_docker-v25.0.0+incompatible.zip", ], ) go_repository( @@ -1750,13 +1789,13 @@ def go_deps(): name = "com_github_emicklei_go_restful_v3", build_file_proto_mode = "disable_global", importpath = "github.com/emicklei/go-restful/v3", - sha256 = "42f1f1e5d986212ba6c7d96f6e76ba2a28b1d17fad9a40b0c45d1505d39bda26", - strip_prefix = "github.com/emicklei/go-restful/v3@v3.10.2", + sha256 = "fc71c649398fa5d28ac7948d143bc8ac4803c01d24f852b9d50e87724ac8efc8", + strip_prefix = "github.com/emicklei/go-restful/v3@v3.11.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.10.2.zip", - "http://ats.apps.svc/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.10.2.zip", - "https://cache.hawkingrei.com/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.10.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.10.2.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.11.0.zip", + "http://ats.apps.svc/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.11.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.11.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/emicklei/go-restful/v3/com_github_emicklei_go_restful_v3-v3.11.0.zip", ], ) go_repository( @@ -1863,6 +1902,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/evanphx/json-patch/com_github_evanphx_json_patch-v5.6.0+incompatible.zip", ], ) + go_repository( + name = "com_github_facette_natsort", + build_file_proto_mode = "disable_global", + importpath = "github.com/facette/natsort", + sha256 = "08cd11112374bf6bf945345bfdede1141a0bb973706291016facc0508eca3ae7", + strip_prefix = "github.com/facette/natsort@v0.0.0-20181210072756-2cd4dd1e2dcb", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/facette/natsort/com_github_facette_natsort-v0.0.0-20181210072756-2cd4dd1e2dcb.zip", + "http://ats.apps.svc/gomod/github.com/facette/natsort/com_github_facette_natsort-v0.0.0-20181210072756-2cd4dd1e2dcb.zip", + "https://cache.hawkingrei.com/gomod/github.com/facette/natsort/com_github_facette_natsort-v0.0.0-20181210072756-2cd4dd1e2dcb.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/facette/natsort/com_github_facette_natsort-v0.0.0-20181210072756-2cd4dd1e2dcb.zip", + ], + ) go_repository( name = "com_github_fasthttp_contrib_websocket", build_file_proto_mode = "disable_global", @@ -2348,13 +2400,13 @@ def go_deps(): name = "com_github_go_openapi_errors", build_file_proto_mode = "disable_global", importpath = "github.com/go-openapi/errors", - sha256 = "40b1b8d380b340602f760e050ca81fe3abfdd88d4e671ab5b9ca6d0361038eee", - strip_prefix = "github.com/go-openapi/errors@v0.20.4", + sha256 = "fd36596bb434cedffc79748a261193cf1938c19b05afa9e56e65f8b643561fee", + strip_prefix = "github.com/go-openapi/errors@v0.21.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.20.4.zip", - "http://ats.apps.svc/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.20.4.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.20.4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.20.4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.21.0.zip", + "http://ats.apps.svc/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.21.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.21.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/errors/com_github_go_openapi_errors-v0.21.0.zip", ], ) go_repository( @@ -2413,13 +2465,13 @@ def go_deps(): name = "com_github_go_openapi_strfmt", build_file_proto_mode = "disable_global", importpath = "github.com/go-openapi/strfmt", - sha256 = "f2e876720e95509630e706a59d0d2ae735aae1f1afb4b2b8c89ea9231dc71084", - strip_prefix = "github.com/go-openapi/strfmt@v0.21.9", + sha256 = "37f512d6ac447bc026276a87eeb89d3c0ec243740c69e79743f8d9761d29aafe", + strip_prefix = "github.com/go-openapi/strfmt@v0.22.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.21.9.zip", - "http://ats.apps.svc/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.21.9.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.21.9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.21.9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.22.0.zip", + "http://ats.apps.svc/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.22.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.22.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/strfmt/com_github_go_openapi_strfmt-v0.22.0.zip", ], ) go_repository( @@ -2842,13 +2894,13 @@ def go_deps(): name = "com_github_golang_jwt_jwt_v5", build_file_proto_mode = "disable_global", importpath = "github.com/golang-jwt/jwt/v5", - sha256 = "d7d763fe73d36361b7a005a3fa3e7bc908ac395c490e1b5b0fdbc4a65272f0b8", - strip_prefix = "github.com/golang-jwt/jwt/v5@v5.0.0", + sha256 = "017207ac5b0514ea44fb962d1e7df2b636d0bb512e452ae6e8a5b05b33dfe8c6", + strip_prefix = "github.com/golang-jwt/jwt/v5@v5.2.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.0.0.zip", - "http://ats.apps.svc/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.0.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.0.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.0.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.2.0.zip", + "http://ats.apps.svc/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.2.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.2.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang-jwt/jwt/v5/com_github_golang_jwt_jwt_v5-v5.2.0.zip", ], ) go_repository( @@ -2872,13 +2924,13 @@ def go_deps(): patches = [ "//build/patches:com_github_golang_protobuf.patch", ], - sha256 = "93bda6e88d4a0a493a98b481de67a10000a755d15f16a800b49a6b96d1bd6f81", - strip_prefix = "github.com/golang/protobuf@v1.5.3", + sha256 = "9a2f43d3eac8ceda506ebbeb4f229254b87235ce90346692a0e233614182190b", + strip_prefix = "github.com/golang/protobuf@v1.5.4", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.3.zip", - "http://ats.apps.svc/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.3.zip", - "https://cache.hawkingrei.com/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.3.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.3.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.4.zip", + "http://ats.apps.svc/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.4.zip", + "https://cache.hawkingrei.com/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.4.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang/protobuf/com_github_golang_protobuf-v1.5.4.zip", ], ) go_repository( @@ -3540,13 +3592,13 @@ def go_deps(): name = "com_github_hashicorp_consul_api", build_file_proto_mode = "disable_global", importpath = "github.com/hashicorp/consul/api", - sha256 = "0a944611bc32f1206642e7aeb40433cea5e98fd084290e7e44b74973d1c184e7", - strip_prefix = "github.com/hashicorp/consul/api@v1.26.1", + sha256 = "bacf525671822c026b37896d9a7225411bd021e2afd7dd886f846728d51ef62e", + strip_prefix = "github.com/hashicorp/consul/api@v1.27.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.26.1.zip", - "http://ats.apps.svc/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.26.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.26.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.26.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.27.0.zip", + "http://ats.apps.svc/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.27.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.27.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hashicorp/consul/api/com_github_hashicorp_consul_api-v1.27.0.zip", ], ) go_repository( @@ -3748,13 +3800,13 @@ def go_deps(): name = "com_github_hetznercloud_hcloud_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/hetznercloud/hcloud-go/v2", - sha256 = "71e2f7c3acd1b9b8838ce91b16baf302bb39684b03af90f1f710d4917d754ca2", - strip_prefix = "github.com/hetznercloud/hcloud-go/v2@v2.4.0", + sha256 = "751828960948066c67e6682647ac68ed3141865d48a9e0e80c6444e0e00f55a1", + strip_prefix = "github.com/hetznercloud/hcloud-go/v2@v2.6.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.4.0.zip", - "http://ats.apps.svc/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.4.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.4.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.4.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.6.0.zip", + "http://ats.apps.svc/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.6.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.6.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hetznercloud/hcloud-go/v2/com_github_hetznercloud_hcloud_go_v2-v2.6.0.zip", ], ) go_repository( @@ -3878,13 +3930,13 @@ def go_deps(): name = "com_github_ionos_cloud_sdk_go_v6", build_file_proto_mode = "disable_global", importpath = "github.com/ionos-cloud/sdk-go/v6", - sha256 = "6ffa828ff99194d52be83c7b6a62453c9189c0ed8e4543589253d8ccfdc94f1e", - strip_prefix = "github.com/ionos-cloud/sdk-go/v6@v6.1.10", + sha256 = "f9538685d901f7c96779dc5b1341a3a5401ae235dd82ee700ded27c838911e4d", + strip_prefix = "github.com/ionos-cloud/sdk-go/v6@v6.1.11", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.10.zip", - "http://ats.apps.svc/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.10.zip", - "https://cache.hawkingrei.com/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.10.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.10.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.11.zip", + "http://ats.apps.svc/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.11.zip", + "https://cache.hawkingrei.com/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.11.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ionos-cloud/sdk-go/v6/com_github_ionos_cloud_sdk_go_v6-v6.1.11.zip", ], ) go_repository( @@ -4446,6 +4498,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/kataras/pio/com_github_kataras_pio-v0.0.0-20190103105442-ea782b38602d.zip", ], ) + go_repository( + name = "com_github_kimmachinegun_automemlimit", + build_file_proto_mode = "disable_global", + importpath = "github.com/KimMachineGun/automemlimit", + sha256 = "fb03ca71f809b89d513c0cbb5ad7b05c1268e220401298958cb970b76027e4ca", + strip_prefix = "github.com/KimMachineGun/automemlimit@v0.5.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/KimMachineGun/automemlimit/com_github_kimmachinegun_automemlimit-v0.5.0.zip", + "http://ats.apps.svc/gomod/github.com/KimMachineGun/automemlimit/com_github_kimmachinegun_automemlimit-v0.5.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/KimMachineGun/automemlimit/com_github_kimmachinegun_automemlimit-v0.5.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/KimMachineGun/automemlimit/com_github_kimmachinegun_automemlimit-v0.5.0.zip", + ], + ) go_repository( name = "com_github_kisielk_errcheck", build_file_proto_mode = "disable_global", @@ -4792,13 +4857,13 @@ def go_deps(): name = "com_github_linode_linodego", build_file_proto_mode = "disable_global", importpath = "github.com/linode/linodego", - sha256 = "06b5687b4acf4511965b37bf08aec25fb606dc289803e463dfa450ee19518a93", - strip_prefix = "github.com/linode/linodego@v1.25.0", + sha256 = "af21149264f65f5a0383ee0a109e8ef1e4f5db95f951d657cb923b0b4f771d4a", + strip_prefix = "github.com/linode/linodego@v1.27.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.25.0.zip", - "http://ats.apps.svc/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.25.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.25.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.25.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.27.1.zip", + "http://ats.apps.svc/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.27.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.27.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/linode/linodego/com_github_linode_linodego-v1.27.1.zip", ], ) go_repository( @@ -5022,19 +5087,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/matttproud/golang_protobuf_extensions/com_github_matttproud_golang_protobuf_extensions-v1.0.1.zip", ], ) - go_repository( - name = "com_github_matttproud_golang_protobuf_extensions_v2", - build_file_proto_mode = "disable_global", - importpath = "github.com/matttproud/golang_protobuf_extensions/v2", - sha256 = "999b014a892da09d7cdd84e4f7117ff034075d74658b162b35eb61bebf29a14f", - strip_prefix = "github.com/matttproud/golang_protobuf_extensions/v2@v2.0.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/matttproud/golang_protobuf_extensions/v2/com_github_matttproud_golang_protobuf_extensions_v2-v2.0.0.zip", - "http://ats.apps.svc/gomod/github.com/matttproud/golang_protobuf_extensions/v2/com_github_matttproud_golang_protobuf_extensions_v2-v2.0.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/matttproud/golang_protobuf_extensions/v2/com_github_matttproud_golang_protobuf_extensions_v2-v2.0.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/matttproud/golang_protobuf_extensions/v2/com_github_matttproud_golang_protobuf_extensions_v2-v2.0.0.zip", - ], - ) go_repository( name = "com_github_mbilski_exhaustivestruct", build_file_proto_mode = "disable_global", @@ -5130,13 +5182,13 @@ def go_deps(): name = "com_github_miekg_dns", build_file_proto_mode = "disable_global", importpath = "github.com/miekg/dns", - sha256 = "50750ca3ebe181f8c16a3d5ccdbc4f7fe864ba6c731a8013d4df8678f901e1f6", - strip_prefix = "github.com/miekg/dns@v1.1.57", + sha256 = "179bd419d011fd90802355756f59fff70ddf9a5886a4db6336a6d05783552b16", + strip_prefix = "github.com/miekg/dns@v1.1.58", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.57.zip", - "http://ats.apps.svc/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.57.zip", - "https://cache.hawkingrei.com/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.57.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.57.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.58.zip", + "http://ats.apps.svc/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.58.zip", + "https://cache.hawkingrei.com/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.58.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.58.zip", ], ) go_repository( @@ -5490,6 +5542,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/nishanths/predeclared/com_github_nishanths_predeclared-v0.2.2.zip", ], ) + go_repository( + name = "com_github_nsf_jsondiff", + build_file_proto_mode = "disable_global", + importpath = "github.com/nsf/jsondiff", + sha256 = "d468c06359490d96ea701107f848acc7d9497e36eeb9eb81e38c8bef69c42fd2", + strip_prefix = "github.com/nsf/jsondiff@v0.0.0-20230430225905-43f6cf3098c1", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/nsf/jsondiff/com_github_nsf_jsondiff-v0.0.0-20230430225905-43f6cf3098c1.zip", + "http://ats.apps.svc/gomod/github.com/nsf/jsondiff/com_github_nsf_jsondiff-v0.0.0-20230430225905-43f6cf3098c1.zip", + "https://cache.hawkingrei.com/gomod/github.com/nsf/jsondiff/com_github_nsf_jsondiff-v0.0.0-20230430225905-43f6cf3098c1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/nsf/jsondiff/com_github_nsf_jsondiff-v0.0.0-20230430225905-43f6cf3098c1.zip", + ], + ) go_repository( name = "com_github_nunnatsa_ginkgolinter", build_file_proto_mode = "disable_global", @@ -5620,6 +5685,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/opencontainers/image-spec/com_github_opencontainers_image_spec-v1.0.2.zip", ], ) + go_repository( + name = "com_github_opencontainers_runtime_spec", + build_file_proto_mode = "disable_global", + importpath = "github.com/opencontainers/runtime-spec", + sha256 = "bd1531bb27014e2a16ea4bf0b56bff7688555bb859f651c1e4375f4b782269ec", + strip_prefix = "github.com/opencontainers/runtime-spec@v1.0.2", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/opencontainers/runtime-spec/com_github_opencontainers_runtime_spec-v1.0.2.zip", + "http://ats.apps.svc/gomod/github.com/opencontainers/runtime-spec/com_github_opencontainers_runtime_spec-v1.0.2.zip", + "https://cache.hawkingrei.com/gomod/github.com/opencontainers/runtime-spec/com_github_opencontainers_runtime_spec-v1.0.2.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/opencontainers/runtime-spec/com_github_opencontainers_runtime_spec-v1.0.2.zip", + ], + ) go_repository( name = "com_github_openpeedeep_depguard_v2", build_file_proto_mode = "disable_global", @@ -5711,6 +5789,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ovh/go-ovh/com_github_ovh_go_ovh-v1.4.3.zip", ], ) + go_repository( + name = "com_github_pbnjay_memory", + build_file_proto_mode = "disable_global", + importpath = "github.com/pbnjay/memory", + sha256 = "5caf461e903c1060392e03a59eb84eb08c4a0976d0cc2d751fa9715cc6fc03bd", + strip_prefix = "github.com/pbnjay/memory@v0.0.0-20210728143218-7b4eea64cf58", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pbnjay/memory/com_github_pbnjay_memory-v0.0.0-20210728143218-7b4eea64cf58.zip", + "http://ats.apps.svc/gomod/github.com/pbnjay/memory/com_github_pbnjay_memory-v0.0.0-20210728143218-7b4eea64cf58.zip", + "https://cache.hawkingrei.com/gomod/github.com/pbnjay/memory/com_github_pbnjay_memory-v0.0.0-20210728143218-7b4eea64cf58.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pbnjay/memory/com_github_pbnjay_memory-v0.0.0-20210728143218-7b4eea64cf58.zip", + ], + ) go_repository( name = "com_github_pborman_getopt", build_file_proto_mode = "disable_global", @@ -5806,13 +5897,13 @@ def go_deps(): name = "com_github_pingcap_errors", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/errors", - sha256 = "0edb07dbd73a90f97e06e11e54b270d64d5cabe6142025682d840fe302087b23", - strip_prefix = "github.com/pingcap/errors@v0.11.5-0.20240311024730-e056997136bb", + sha256 = "99d36c5e5649442d9923d4b1cd7f1242faaf4071844c2e3c77497230b3f38cd7", + strip_prefix = "github.com/pingcap/errors@v0.11.5-0.20240318064555-6bd07397691f", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240318064555-6bd07397691f.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240318064555-6bd07397691f.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240318064555-6bd07397691f.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240318064555-6bd07397691f.zip", ], ) go_repository( @@ -5858,65 +5949,65 @@ def go_deps(): name = "com_github_pingcap_kvproto", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/kvproto", - sha256 = "6857568a0131c268a444cb5b5717a3ecabf9b9633fd5cf5ca874cfbde0902b19", - strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240208102409-a554af8ee11f", + sha256 = "07dff29e9848e79f36ac8dcd0d5b48bbfbb2796308702451afb862accb79fedb", + strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240227073058-929ab83f9754", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240208102409-a554af8ee11f.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240208102409-a554af8ee11f.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240208102409-a554af8ee11f.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240208102409-a554af8ee11f.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip", ], ) go_repository( name = "com_github_pingcap_log", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/log", - sha256 = "9b0ae182fcc611cc535eb18a6f332846f0744b905e48888c06c1f6aeda8036d5", - strip_prefix = "github.com/pingcap/log@v1.1.1-0.20230317032135-a0d097d16e22", + sha256 = "c393f943bb688e240dcf9a50ac1d915aa27371803524887ab61a235834bb7438", + strip_prefix = "github.com/pingcap/log@v1.1.1-0.20240314023424-862ccc32f18d", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip", ], ) go_repository( name = "com_github_pingcap_sysutil", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/sysutil", - sha256 = "0540f3dfd25706302c0f484baae0b6bb6a6382893bbf8b14720debcb9756f1d0", - strip_prefix = "github.com/pingcap/sysutil@v1.0.1-0.20230407040306-fb007c5aff21", + sha256 = "c2ffa4a6d163ca73e73831de5abaa25d47abba59c41b8a549d70935442921a56", + strip_prefix = "github.com/pingcap/sysutil@v1.0.1-0.20240311050922-ae81ee01f3a5", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip", ], ) go_repository( name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", - sha256 = "5ca34a7440b4291d12c225a5a78944808509b5f6cf91cd32c1c8421d7bc14382", - strip_prefix = "github.com/pingcap/tipb@v0.0.0-20240116032918-9bb28c43bbfc", + sha256 = "42c365f3f99d2577fe29c89d433894b2d0d69b054248bb3bafbced33332933e3", + strip_prefix = "github.com/pingcap/tipb@v0.0.0-20240318032315-55a7867ddd50", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240116032918-9bb28c43bbfc.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240116032918-9bb28c43bbfc.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240116032918-9bb28c43bbfc.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240116032918-9bb28c43bbfc.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", ], ) go_repository( name = "com_github_pkg_browser", build_file_proto_mode = "disable_global", importpath = "github.com/pkg/browser", - sha256 = "415b8b7d7e47074cf3f6c2269d8712efa8a8433cba7bfce7eed22ca7f0b447a4", - strip_prefix = "github.com/pkg/browser@v0.0.0-20210911075715-681adbf594b8", + sha256 = "8524ae36d809564d1f218978593b5c565cf3ee8dccd035d66b336ad0c56e60d1", + strip_prefix = "github.com/pkg/browser@v0.0.0-20240102092130-5ac0b6a4141c", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20210911075715-681adbf594b8.zip", - "http://ats.apps.svc/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20210911075715-681adbf594b8.zip", - "https://cache.hawkingrei.com/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20210911075715-681adbf594b8.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20210911075715-681adbf594b8.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20240102092130-5ac0b6a4141c.zip", + "http://ats.apps.svc/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20240102092130-5ac0b6a4141c.zip", + "https://cache.hawkingrei.com/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20240102092130-5ac0b6a4141c.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pkg/browser/com_github_pkg_browser-v0.0.0-20240102092130-5ac0b6a4141c.zip", ], ) go_repository( @@ -6105,39 +6196,39 @@ def go_deps(): name = "com_github_prometheus_exporter_toolkit", build_file_proto_mode = "disable_global", importpath = "github.com/prometheus/exporter-toolkit", - sha256 = "d6d1eee3a082bd82744db81a52b01e4923932b498f92411ca57390e7489cf34b", - strip_prefix = "github.com/prometheus/exporter-toolkit@v0.10.0", + sha256 = "30605de7fc911194ff45289870b4a56d9da2aca9e8f60de810bc65af77cdd2cb", + strip_prefix = "github.com/prometheus/exporter-toolkit@v0.11.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.10.0.zip", - "http://ats.apps.svc/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.10.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.10.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.10.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.11.0.zip", + "http://ats.apps.svc/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.11.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.11.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/exporter-toolkit/com_github_prometheus_exporter_toolkit-v0.11.0.zip", ], ) go_repository( name = "com_github_prometheus_procfs", build_file_proto_mode = "disable_global", importpath = "github.com/prometheus/procfs", - sha256 = "7f23eba0928dc2eb4d052180eb0a0c014153d728141931d2aac2daf9380dfff5", - strip_prefix = "github.com/prometheus/procfs@v0.12.0", + sha256 = "6fe923fc0ca170a60524d1031cf9ee634cb2eba16798edcbe1ed02e591994cfa", + strip_prefix = "github.com/prometheus/procfs@v0.13.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.12.0.zip", - "http://ats.apps.svc/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.12.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.12.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.12.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip", + "http://ats.apps.svc/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip", ], ) go_repository( name = "com_github_prometheus_prometheus", build_file_proto_mode = "disable_global", importpath = "github.com/prometheus/prometheus", - sha256 = "70aa4f4a5d34ccb0425e89461637c0962502ad994ddcf1a48e87a75ebc0232b1", - strip_prefix = "github.com/prometheus/prometheus@v0.49.1", + sha256 = "10d025d507269bf23714992664755f8fed449dd1c8b9cb113a72e29baebcaa64", + strip_prefix = "github.com/prometheus/prometheus@v0.50.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.49.1.zip", - "http://ats.apps.svc/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.49.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.49.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.49.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip", + "http://ats.apps.svc/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip", ], ) go_repository( @@ -6430,13 +6521,13 @@ def go_deps(): name = "com_github_scaleway_scaleway_sdk_go", build_file_proto_mode = "disable_global", importpath = "github.com/scaleway/scaleway-sdk-go", - sha256 = "c1c638a823b55c10a89bf55a501c55dc91ee2aced5e677d66748363923d34108", - strip_prefix = "github.com/scaleway/scaleway-sdk-go@v1.0.0-beta.21", + sha256 = "b7f9a702ee1e899d81bc23ca5e761cd2bc6c0202797d9b5193b83a50bad16698", + strip_prefix = "github.com/scaleway/scaleway-sdk-go@v1.0.0-beta.22", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.21.zip", - "http://ats.apps.svc/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.21.zip", - "https://cache.hawkingrei.com/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.21.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.21.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.22.zip", + "http://ats.apps.svc/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.22.zip", + "https://cache.hawkingrei.com/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.22.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/scaleway/scaleway-sdk-go/com_github_scaleway_scaleway_sdk_go-v1.0.0-beta.22.zip", ], ) go_repository( @@ -7054,13 +7145,13 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sha256 = "7b6accba538659d8c03705405b7b3249712751028586f679cc45dfdc87a54898", - strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240308052415-af4f9a9b6e41", + sha256 = "1838f5b1e46ccef68f651efd1a01a2b70037062ba6a08d58b2bd6aa8d3580e0c", + strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240318065517-a9128e8200ab", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240308052415-af4f9a9b6e41.zip", - "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240308052415-af4f9a9b6e41.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240308052415-af4f9a9b6e41.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240308052415-af4f9a9b6e41.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240318065517-a9128e8200ab.zip", + "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240318065517-a9128e8200ab.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240318065517-a9128e8200ab.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240318065517-a9128e8200ab.zip", ], ) go_repository( @@ -9632,39 +9723,39 @@ def go_deps(): name = "io_k8s_api", build_file_proto_mode = "disable_global", importpath = "k8s.io/api", - sha256 = "eff8202063496b701784d305dc90c235c1b6df8b394a04161f1890f3ee8164e3", - strip_prefix = "k8s.io/api@v0.28.4", + sha256 = "2255428d2347df0b3a9cf6ac2791f5be6653b3c642359736e46733584d917335", + strip_prefix = "k8s.io/api@v0.28.6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/api/io_k8s_api-v0.28.4.zip", - "http://ats.apps.svc/gomod/k8s.io/api/io_k8s_api-v0.28.4.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/api/io_k8s_api-v0.28.4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/api/io_k8s_api-v0.28.4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/api/io_k8s_api-v0.28.6.zip", + "http://ats.apps.svc/gomod/k8s.io/api/io_k8s_api-v0.28.6.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/api/io_k8s_api-v0.28.6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/api/io_k8s_api-v0.28.6.zip", ], ) go_repository( name = "io_k8s_apimachinery", build_file_proto_mode = "disable_global", importpath = "k8s.io/apimachinery", - sha256 = "731a1f026cbb3cefcca91fc0a758d6249207f3b36cc004ac2e498b3828745b48", - strip_prefix = "k8s.io/apimachinery@v0.28.4", + sha256 = "efc7e38cb4662d0b6c5648772e1ae92040a4d03af0a3a7731aedf17f8eab7359", + strip_prefix = "k8s.io/apimachinery@v0.28.6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.4.zip", - "http://ats.apps.svc/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.4.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.6.zip", + "http://ats.apps.svc/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.6.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/apimachinery/io_k8s_apimachinery-v0.28.6.zip", ], ) go_repository( name = "io_k8s_client_go", build_file_proto_mode = "disable_global", importpath = "k8s.io/client-go", - sha256 = "901547d5808428fee28f3541113980ccfed7ee11dd0141976d8bfeac9fba4736", - strip_prefix = "k8s.io/client-go@v0.28.4", + sha256 = "d7ba046e2a7574042492b9986943f782e48284fffbd4e86405b5ae011da37bf3", + strip_prefix = "k8s.io/client-go@v0.28.6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.4.zip", - "http://ats.apps.svc/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.4.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.6.zip", + "http://ats.apps.svc/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.6.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/client-go/io_k8s_client_go-v0.28.6.zip", ], ) go_repository( @@ -9684,26 +9775,26 @@ def go_deps(): name = "io_k8s_klog_v2", build_file_proto_mode = "disable_global", importpath = "k8s.io/klog/v2", - sha256 = "b52957447658f4e86f649fa802adcacac8c550f8e997bd972b25e24a9c6c9d1a", - strip_prefix = "k8s.io/klog/v2@v2.110.1", + sha256 = "f6793076147daa5aefcc1ef0f5536c063dd8e209af9f95f8fed4efbd57665a49", + strip_prefix = "k8s.io/klog/v2@v2.120.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.110.1.zip", - "http://ats.apps.svc/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.110.1.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.110.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.110.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.120.1.zip", + "http://ats.apps.svc/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.120.1.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.120.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/klog/v2/io_k8s_klog_v2-v2.120.1.zip", ], ) go_repository( name = "io_k8s_kube_openapi", build_file_proto_mode = "disable_global", importpath = "k8s.io/kube-openapi", - sha256 = "1439fcbc0a04bbf7edf72712288e1cc4d2497fd28279c76a01a366910b65d6c7", - strip_prefix = "k8s.io/kube-openapi@v0.0.0-20230717233707-2695361300d9", + sha256 = "57cb5b3e2bdcb29c209e7b543bc61f4f8d7dd390fc21a883a7c0f388771931b2", + strip_prefix = "k8s.io/kube-openapi@v0.0.0-20231010175941-2dd684a91f00", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20230717233707-2695361300d9.zip", - "http://ats.apps.svc/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20230717233707-2695361300d9.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20230717233707-2695361300d9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20230717233707-2695361300d9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20231010175941-2dd684a91f00.zip", + "http://ats.apps.svc/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20231010175941-2dd684a91f00.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20231010175941-2dd684a91f00.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/kube-openapi/io_k8s_kube_openapi-v0.0.0-20231010175941-2dd684a91f00.zip", ], ) go_repository( @@ -9723,13 +9814,13 @@ def go_deps(): name = "io_k8s_sigs_structured_merge_diff_v4", build_file_proto_mode = "disable_global", importpath = "sigs.k8s.io/structured-merge-diff/v4", - sha256 = "0a5107d9269d3fc45d3abb9a1fc0c3c4788b82d848679416cb4c2bc49cad89de", - strip_prefix = "sigs.k8s.io/structured-merge-diff/v4@v4.3.0", + sha256 = "ce80f7bbeca9f900e8ec311744007cd764c679ecc8d3a6aaa6abf1223dde91f2", + strip_prefix = "sigs.k8s.io/structured-merge-diff/v4@v4.4.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.3.0.zip", - "http://ats.apps.svc/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.3.0.zip", - "https://cache.hawkingrei.com/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.3.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.3.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.4.1.zip", + "http://ats.apps.svc/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.4.1.zip", + "https://cache.hawkingrei.com/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.4.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/sigs.k8s.io/structured-merge-diff/v4/io_k8s_sigs_structured_merge_diff_v4-v4.4.1.zip", ], ) go_repository( @@ -9749,13 +9840,13 @@ def go_deps(): name = "io_k8s_utils", build_file_proto_mode = "disable_global", importpath = "k8s.io/utils", - sha256 = "755df44d714f0c28df51b94f096c1ff5af7625a00c92ca03b5914217a759b391", - strip_prefix = "k8s.io/utils@v0.0.0-20230711102312-30195339c3c7", + sha256 = "b215ab848276200af441edc56a920338d00299cf69005e94bdbe9df76c7bd0ba", + strip_prefix = "k8s.io/utils@v0.0.0-20230726121419-3b25d923346b", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230711102312-30195339c3c7.zip", - "http://ats.apps.svc/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230711102312-30195339c3c7.zip", - "https://cache.hawkingrei.com/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230711102312-30195339c3c7.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230711102312-30195339c3c7.zip", + "http://bazel-cache.pingcap.net:8080/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230726121419-3b25d923346b.zip", + "http://ats.apps.svc/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230726121419-3b25d923346b.zip", + "https://cache.hawkingrei.com/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230726121419-3b25d923346b.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/k8s.io/utils/io_k8s_utils-v0.0.0-20230726121419-3b25d923346b.zip", ], ) go_repository( @@ -9775,39 +9866,39 @@ def go_deps(): name = "io_opentelemetry_go_collector_featuregate", build_file_proto_mode = "disable_global", importpath = "go.opentelemetry.io/collector/featuregate", - sha256 = "edd801853ad91428eed8d717c00b6b042c1f6aa4c0c90428a79148156c33886b", - strip_prefix = "go.opentelemetry.io/collector/featuregate@v1.0.0", + sha256 = "b8f74a6e4e4e68dc630b54917755c1604525ce161b9f8a2730079fa19bc11676", + strip_prefix = "go.opentelemetry.io/collector/featuregate@v1.0.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.0.zip", - "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.0.zip", - "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.1.zip", + "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.1.zip", + "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/featuregate/io_opentelemetry_go_collector_featuregate-v1.0.1.zip", ], ) go_repository( name = "io_opentelemetry_go_collector_pdata", build_file_proto_mode = "disable_global", importpath = "go.opentelemetry.io/collector/pdata", - sha256 = "7705242dc35df16befd2a69a79f9f9797b2412623a7a81e47622d69a98219f0c", - strip_prefix = "go.opentelemetry.io/collector/pdata@v1.0.0", + sha256 = "696f8737e0dd15c76a683c7ab00f373a50a4c1f27890ed288ffc994b1bb19d15", + strip_prefix = "go.opentelemetry.io/collector/pdata@v1.0.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.0.zip", - "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.0.zip", - "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.1.zip", + "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.1.zip", + "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/pdata/io_opentelemetry_go_collector_pdata-v1.0.1.zip", ], ) go_repository( name = "io_opentelemetry_go_collector_semconv", build_file_proto_mode = "disable_global", importpath = "go.opentelemetry.io/collector/semconv", - sha256 = "75958ce453b97b61b25de8c214f0006cedf200df94ece196b9f52a600a749cfd", - strip_prefix = "go.opentelemetry.io/collector/semconv@v0.90.1", + sha256 = "7ee5e8d4b9f9bbdefdec35bc49866ab628ca740344a8940d33874868debfb034", + strip_prefix = "go.opentelemetry.io/collector/semconv@v0.93.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.90.1.zip", - "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.90.1.zip", - "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.90.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.90.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.93.0.zip", + "http://ats.apps.svc/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.93.0.zip", + "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.93.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/collector/semconv/io_opentelemetry_go_collector_semconv-v0.93.0.zip", ], ) go_repository( @@ -9879,13 +9970,13 @@ def go_deps(): name = "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp", build_file_proto_mode = "disable_global", importpath = "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp", - sha256 = "0cf89afaad577e3d43b7071994a0bf57e18a1c27a0d6e98a60b9e995fa9e97fb", - strip_prefix = "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.21.0", + sha256 = "6b7c01ca38a5b1c4216adcf54c422e0a78c257d918e89b8dd02721c6098b9dec", + strip_prefix = "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.22.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.21.0.zip", - "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.21.0.zip", - "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.21.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.21.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.22.0.zip", + "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.22.0.zip", + "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.22.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp-v1.22.0.zip", ], ) go_repository( @@ -10074,13 +10165,13 @@ def go_deps(): name = "org_golang_google_genproto_googleapis_api", build_file_proto_mode = "disable_global", importpath = "google.golang.org/genproto/googleapis/api", - sha256 = "408760d148f392b4952687f847e54b361fb39db7168020b4813f268313035b4a", - strip_prefix = "google.golang.org/genproto/googleapis/api@v0.0.0-20240205150955-31a09d347014", + sha256 = "1ebe3c1107c126819cd1b27f2eb3966df4fcd434bbe45d5ef2cba514091a8c80", + strip_prefix = "google.golang.org/genproto/googleapis/api@v0.0.0-20240304212257-790db918fca8", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240205150955-31a09d347014.zip", - "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240205150955-31a09d347014.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240205150955-31a09d347014.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240205150955-31a09d347014.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip", + "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip", ], ) go_repository( @@ -10100,26 +10191,26 @@ def go_deps(): name = "org_golang_google_genproto_googleapis_rpc", build_file_proto_mode = "disable_global", importpath = "google.golang.org/genproto/googleapis/rpc", - sha256 = "b860319d7742b707cac31e956d4873478d45fe063c3a874d5cf65185f74a823a", - strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240221002015-b0ce06bbee7c", + sha256 = "755a36227e2551491d44533ef50df7f47964db44911e6d4d84e4d4842a5339d6", + strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240308144416-29370a3891b7", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240221002015-b0ce06bbee7c.zip", - "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240221002015-b0ce06bbee7c.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240221002015-b0ce06bbee7c.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240221002015-b0ce06bbee7c.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240308144416-29370a3891b7.zip", + "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240308144416-29370a3891b7.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240308144416-29370a3891b7.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240308144416-29370a3891b7.zip", ], ) go_repository( name = "org_golang_google_grpc", build_file_proto_mode = "disable_global", importpath = "google.golang.org/grpc", - sha256 = "0a79f61fcc876b8bcf3581afd1a41a1d1a8a95b6554d90238ec4321242db9325", - strip_prefix = "google.golang.org/grpc@v1.62.0", + sha256 = "95226c98d052d7e4cd308c4df972464ba90e302cf12b5f180e245c4c3d2cc02f", + strip_prefix = "google.golang.org/grpc@v1.62.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.0.zip", - "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.0.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.1.zip", + "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.1.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.62.1.zip", ], ) go_repository( @@ -10139,13 +10230,13 @@ def go_deps(): name = "org_golang_google_protobuf", build_file_proto_mode = "disable_global", importpath = "google.golang.org/protobuf", - sha256 = "c2c117cf29abee8697dabdc69662accf66171bea0efa2749988867ae8ef2362d", - strip_prefix = "google.golang.org/protobuf@v1.32.0", + sha256 = "2cc1c98e12903009bd4bf0d5e938a421ca2f88ae87b0fc50004b2c7598b1fd24", + strip_prefix = "google.golang.org/protobuf@v1.33.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.32.0.zip", - "http://ats.apps.svc/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.32.0.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.32.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.32.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip", + "http://ats.apps.svc/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip", ], ) go_repository( @@ -10230,13 +10321,13 @@ def go_deps(): name = "org_golang_x_mod", build_file_proto_mode = "disable_global", importpath = "golang.org/x/mod", - sha256 = "81c61d043854b5242ac4a9ff92fe3b275b033cc5ec32c46b46a40a143c1658e7", - strip_prefix = "golang.org/x/mod@v0.15.0", + sha256 = "44bb0b60a305036c6402f9b2d58f7d3cfca310cff57241f37c6e0a6bdafacb15", + strip_prefix = "golang.org/x/mod@v0.16.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.15.0.zip", - "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.15.0.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.15.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.15.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.16.0.zip", + "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.16.0.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.16.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.16.0.zip", ], ) go_repository( @@ -10256,13 +10347,13 @@ def go_deps(): name = "org_golang_x_oauth2", build_file_proto_mode = "disable_global", importpath = "golang.org/x/oauth2", - sha256 = "e59eee832df8784517e9d77cf608784a659d2001c9af66591c1787e17401622e", - strip_prefix = "golang.org/x/oauth2@v0.17.0", + sha256 = "7716d06cf89e1736b74b94584cdcad120c769519f0c41e6d77d0f12657870772", + strip_prefix = "golang.org/x/oauth2@v0.18.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.17.0.zip", - "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.17.0.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.17.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.17.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip", + "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip", ], ) go_repository( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock deleted file mode 100644 index 0e3a8c4dc4778..0000000000000 --- a/MODULE.bazel.lock +++ /dev/null @@ -1,1267 +0,0 @@ -{ - "lockFileVersion": 3, - "moduleFileHash": "0e3e315145ac7ee7a4e0ac825e1c5e03c068ec1254dd42c3caaecb27e921dc4d", - "flags": { - "cmdRegistries": [ - "https://bcr.bazel.build/" - ], - "cmdModuleOverrides": {}, - "allowedYankedVersions": [], - "envVarAllowedYankedVersions": "", - "ignoreDevDependency": false, - "directDependenciesMode": "WARNING", - "compatibilityMode": "ERROR" - }, - "localOverrideHashes": { - "bazel_tools": "922ea6752dc9105de5af957f7a99a6933c0a6a712d23df6aad16a9c399f7e787" - }, - "moduleDepGraph": { - "": { - "name": "", - "version": "", - "key": "", - "repoName": "", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - } - }, - "bazel_tools@_": { - "name": "bazel_tools", - "version": "", - "key": "bazel_tools@_", - "repoName": "bazel_tools", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_cc_toolchains//:all", - "@local_config_sh//:local_sh_toolchain" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@bazel_tools//tools/cpp:cc_configure.bzl", - "extensionName": "cc_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 17, - "column": 29 - }, - "imports": { - "local_config_cc": "local_config_cc", - "local_config_cc_toolchains": "local_config_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/osx:xcode_configure.bzl", - "extensionName": "xcode_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 21, - "column": 32 - }, - "imports": { - "local_config_xcode": "local_config_xcode" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@rules_java//java:extensions.bzl", - "extensionName": "toolchains", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 24, - "column": 32 - }, - "imports": { - "local_jdk": "local_jdk", - "remote_java_tools": "remote_java_tools", - "remote_java_tools_linux": "remote_java_tools_linux", - "remote_java_tools_windows": "remote_java_tools_windows", - "remote_java_tools_darwin_x86_64": "remote_java_tools_darwin_x86_64", - "remote_java_tools_darwin_arm64": "remote_java_tools_darwin_arm64" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/sh:sh_configure.bzl", - "extensionName": "sh_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 35, - "column": 39 - }, - "imports": { - "local_config_sh": "local_config_sh" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/test:extensions.bzl", - "extensionName": "remote_coverage_tools_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 39, - "column": 48 - }, - "imports": { - "remote_coverage_tools": "remote_coverage_tools" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/android:android_extensions.bzl", - "extensionName": "remote_android_tools_extensions", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 42, - "column": 42 - }, - "imports": { - "android_gmaven_r8": "android_gmaven_r8", - "android_tools": "android_tools" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "rules_cc": "rules_cc@0.0.9", - "rules_java": "rules_java@7.1.0", - "rules_license": "rules_license@0.0.7", - "rules_proto": "rules_proto@4.0.0", - "rules_python": "rules_python@0.4.0", - "platforms": "platforms@0.0.7", - "com_google_protobuf": "protobuf@3.19.6", - "zlib": "zlib@1.3", - "build_bazel_apple_support": "apple_support@1.5.0", - "local_config_platform": "local_config_platform@_" - } - }, - "local_config_platform@_": { - "name": "local_config_platform", - "version": "", - "key": "local_config_platform@_", - "repoName": "local_config_platform", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.7", - "bazel_tools": "bazel_tools@_" - } - }, - "rules_cc@0.0.9": { - "name": "rules_cc", - "version": "0.0.9", - "key": "rules_cc@0.0.9", - "repoName": "rules_cc", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_cc_toolchains//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@bazel_tools//tools/cpp:cc_configure.bzl", - "extensionName": "cc_configure_extension", - "usingModule": "rules_cc@0.0.9", - "location": { - "file": "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel", - "line": 9, - "column": 29 - }, - "imports": { - "local_config_cc_toolchains": "local_config_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "platforms": "platforms@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_cc~0.0.9", - "urls": [ - "https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz" - ], - "integrity": "sha256-IDeHW5pEVtzkp50RKorohbvEqtlo5lh9ym5k86CQDN8=", - "strip_prefix": "rules_cc-0.0.9", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_cc/0.0.9/patches/module_dot_bazel_version.patch": "sha256-mM+qzOI0SgAdaJBlWOSMwMPKpaA9b7R37Hj/tp5bb4g=" - }, - "remote_patch_strip": 0 - } - } - }, - "rules_java@7.1.0": { - "name": "rules_java", - "version": "7.1.0", - "key": "rules_java@7.1.0", - "repoName": "rules_java", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "//toolchains:all", - "@local_jdk//:runtime_toolchain_definition", - "@local_jdk//:bootstrap_runtime_toolchain_definition", - "@remotejdk11_linux_toolchain_config_repo//:all", - "@remotejdk11_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk11_linux_ppc64le_toolchain_config_repo//:all", - "@remotejdk11_linux_s390x_toolchain_config_repo//:all", - "@remotejdk11_macos_toolchain_config_repo//:all", - "@remotejdk11_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk11_win_toolchain_config_repo//:all", - "@remotejdk11_win_arm64_toolchain_config_repo//:all", - "@remotejdk17_linux_toolchain_config_repo//:all", - "@remotejdk17_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk17_linux_ppc64le_toolchain_config_repo//:all", - "@remotejdk17_linux_s390x_toolchain_config_repo//:all", - "@remotejdk17_macos_toolchain_config_repo//:all", - "@remotejdk17_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk17_win_toolchain_config_repo//:all", - "@remotejdk17_win_arm64_toolchain_config_repo//:all", - "@remotejdk21_linux_toolchain_config_repo//:all", - "@remotejdk21_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk21_macos_toolchain_config_repo//:all", - "@remotejdk21_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk21_win_toolchain_config_repo//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_java//java:extensions.bzl", - "extensionName": "toolchains", - "usingModule": "rules_java@7.1.0", - "location": { - "file": "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel", - "line": 19, - "column": 27 - }, - "imports": { - "remote_java_tools": "remote_java_tools", - "remote_java_tools_linux": "remote_java_tools_linux", - "remote_java_tools_windows": "remote_java_tools_windows", - "remote_java_tools_darwin_x86_64": "remote_java_tools_darwin_x86_64", - "remote_java_tools_darwin_arm64": "remote_java_tools_darwin_arm64", - "local_jdk": "local_jdk", - "remotejdk11_linux_toolchain_config_repo": "remotejdk11_linux_toolchain_config_repo", - "remotejdk11_linux_aarch64_toolchain_config_repo": "remotejdk11_linux_aarch64_toolchain_config_repo", - "remotejdk11_linux_ppc64le_toolchain_config_repo": "remotejdk11_linux_ppc64le_toolchain_config_repo", - "remotejdk11_linux_s390x_toolchain_config_repo": "remotejdk11_linux_s390x_toolchain_config_repo", - "remotejdk11_macos_toolchain_config_repo": "remotejdk11_macos_toolchain_config_repo", - "remotejdk11_macos_aarch64_toolchain_config_repo": "remotejdk11_macos_aarch64_toolchain_config_repo", - "remotejdk11_win_toolchain_config_repo": "remotejdk11_win_toolchain_config_repo", - "remotejdk11_win_arm64_toolchain_config_repo": "remotejdk11_win_arm64_toolchain_config_repo", - "remotejdk17_linux_toolchain_config_repo": "remotejdk17_linux_toolchain_config_repo", - "remotejdk17_linux_aarch64_toolchain_config_repo": "remotejdk17_linux_aarch64_toolchain_config_repo", - "remotejdk17_linux_ppc64le_toolchain_config_repo": "remotejdk17_linux_ppc64le_toolchain_config_repo", - "remotejdk17_linux_s390x_toolchain_config_repo": "remotejdk17_linux_s390x_toolchain_config_repo", - "remotejdk17_macos_toolchain_config_repo": "remotejdk17_macos_toolchain_config_repo", - "remotejdk17_macos_aarch64_toolchain_config_repo": "remotejdk17_macos_aarch64_toolchain_config_repo", - "remotejdk17_win_toolchain_config_repo": "remotejdk17_win_toolchain_config_repo", - "remotejdk17_win_arm64_toolchain_config_repo": "remotejdk17_win_arm64_toolchain_config_repo", - "remotejdk21_linux_toolchain_config_repo": "remotejdk21_linux_toolchain_config_repo", - "remotejdk21_linux_aarch64_toolchain_config_repo": "remotejdk21_linux_aarch64_toolchain_config_repo", - "remotejdk21_macos_toolchain_config_repo": "remotejdk21_macos_toolchain_config_repo", - "remotejdk21_macos_aarch64_toolchain_config_repo": "remotejdk21_macos_aarch64_toolchain_config_repo", - "remotejdk21_win_toolchain_config_repo": "remotejdk21_win_toolchain_config_repo" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "platforms": "platforms@0.0.7", - "rules_cc": "rules_cc@0.0.9", - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_proto": "rules_proto@4.0.0", - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0", - "urls": [ - "https://github.com/bazelbuild/rules_java/releases/download/7.1.0/rules_java-7.1.0.tar.gz" - ], - "integrity": "sha256-o3pOX2OrgnFuXdau75iO2EYcegC46TYnImKJn1h81OE=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_license@0.0.7": { - "name": "rules_license", - "version": "0.0.7", - "key": "rules_license@0.0.7", - "repoName": "rules_license", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_license~0.0.7", - "urls": [ - "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" - ], - "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_proto@4.0.0": { - "name": "rules_proto", - "version": "4.0.0", - "key": "rules_proto@4.0.0", - "repoName": "rules_proto", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_cc": "rules_cc@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_proto~4.0.0", - "urls": [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0.zip" - ], - "integrity": "sha256-Lr5z6xyuRA19pNtRYMGjKaynwQpck4H/lwYyVjyhoq4=", - "strip_prefix": "rules_proto-4.0.0", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_proto/4.0.0/patches/module_dot_bazel.patch": "sha256-MclJO7tIAM2ElDAmscNId9pKTpOuDGHgVlW/9VBOIp0=" - }, - "remote_patch_strip": 0 - } - } - }, - "rules_python@0.4.0": { - "name": "rules_python", - "version": "0.4.0", - "key": "rules_python@0.4.0", - "repoName": "rules_python", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@bazel_tools//tools/python:autodetecting_toolchain" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_python//bzlmod:extensions.bzl", - "extensionName": "pip_install", - "usingModule": "rules_python@0.4.0", - "location": { - "file": "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel", - "line": 7, - "column": 28 - }, - "imports": { - "pypi__click": "pypi__click", - "pypi__pip": "pypi__pip", - "pypi__pip_tools": "pypi__pip_tools", - "pypi__pkginfo": "pypi__pkginfo", - "pypi__setuptools": "pypi__setuptools", - "pypi__wheel": "pypi__wheel" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_python~0.4.0", - "urls": [ - "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz" - ], - "integrity": "sha256-lUqom0kb5KCDMEosuDgBnIuMNyCnq7nEy4GseiQjDOo=", - "strip_prefix": "", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_python/0.4.0/patches/propagate_pip_install_dependencies.patch": "sha256-v7S/dem/mixg63MF4KoRGDA4KEol9ab/tIVp+6Xq0D0=", - "https://bcr.bazel.build/modules/rules_python/0.4.0/patches/module_dot_bazel.patch": "sha256-kG4VIfWxQazzTuh50mvsx6pmyoRVA4lfH5rkto/Oq+Y=" - }, - "remote_patch_strip": 1 - } - } - }, - "platforms@0.0.7": { - "name": "platforms", - "version": "0.0.7", - "key": "platforms@0.0.7", - "repoName": "platforms", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "platforms", - "urls": [ - "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz" - ], - "integrity": "sha256-OlYcmee9vpFzqmU/1Xn+hJ8djWc5V4CrR3Cx84FDHVE=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "protobuf@3.19.6": { - "name": "protobuf", - "version": "3.19.6", - "key": "protobuf@3.19.6", - "repoName": "protobuf", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "zlib": "zlib@1.3", - "rules_python": "rules_python@0.4.0", - "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@4.0.0", - "rules_java": "rules_java@7.1.0", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "protobuf~3.19.6", - "urls": [ - "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.6.zip" - ], - "integrity": "sha256-OH4sVZuyx8G8N5jE5s/wFTgaebJ1hpavy/johzC0c4k=", - "strip_prefix": "protobuf-3.19.6", - "remote_patches": { - "https://bcr.bazel.build/modules/protobuf/3.19.6/patches/relative_repo_names.patch": "sha256-w/5gw/zGv8NFId+669hcdw1Uus2lxgYpulATHIwIByI=", - "https://bcr.bazel.build/modules/protobuf/3.19.6/patches/remove_dependency_on_rules_jvm_external.patch": "sha256-THUTnVgEBmjA0W7fKzIyZOVG58DnW9HQTkr4D2zKUUc=", - "https://bcr.bazel.build/modules/protobuf/3.19.6/patches/add_module_dot_bazel_for_examples.patch": "sha256-s/b1gi3baK3LsXefI2rQilhmkb2R5jVJdnT6zEcdfHY=", - "https://bcr.bazel.build/modules/protobuf/3.19.6/patches/module_dot_bazel.patch": "sha256-S0DEni8zgx7rHscW3z/rCEubQnYec0XhNet640cw0h4=" - }, - "remote_patch_strip": 1 - } - } - }, - "zlib@1.3": { - "name": "zlib", - "version": "1.3", - "key": "zlib@1.3", - "repoName": "zlib", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.7", - "rules_cc": "rules_cc@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "zlib~1.3", - "urls": [ - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz" - ], - "integrity": "sha256-/wukwpIBPbwnUws6geH5qBPNOd4Byl4Pi/NVcC76WT4=", - "strip_prefix": "zlib-1.3", - "remote_patches": { - "https://bcr.bazel.build/modules/zlib/1.3/patches/add_build_file.patch": "sha256-Ei+FYaaOo7A3jTKunMEodTI0Uw5NXQyZEcboMC8JskY=", - "https://bcr.bazel.build/modules/zlib/1.3/patches/module_dot_bazel.patch": "sha256-fPWLM+2xaF/kuy+kZc1YTfW6hNjrkG400Ho7gckuyJk=" - }, - "remote_patch_strip": 0 - } - } - }, - "apple_support@1.5.0": { - "name": "apple_support", - "version": "1.5.0", - "key": "apple_support@1.5.0", - "repoName": "build_bazel_apple_support", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_apple_cc_toolchains//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@build_bazel_apple_support//crosstool:setup.bzl", - "extensionName": "apple_cc_configure_extension", - "usingModule": "apple_support@1.5.0", - "location": { - "file": "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel", - "line": 17, - "column": 35 - }, - "imports": { - "local_config_apple_cc": "local_config_apple_cc", - "local_config_apple_cc_toolchains": "local_config_apple_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "platforms": "platforms@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "apple_support~1.5.0", - "urls": [ - "https://github.com/bazelbuild/apple_support/releases/download/1.5.0/apple_support.1.5.0.tar.gz" - ], - "integrity": "sha256-miM41vja0yRPgj8txghKA+TQ+7J8qJLclw5okNW0gYQ=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "bazel_skylib@1.3.0": { - "name": "bazel_skylib", - "version": "1.3.0", - "key": "bazel_skylib@1.3.0", - "repoName": "bazel_skylib", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "//toolchains/unittest:cmd_toolchain", - "//toolchains/unittest:bash_toolchain" - ], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "bazel_skylib~1.3.0", - "urls": [ - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz" - ], - "integrity": "sha256-dNVE2W9KW7Yw1GXKi7z+Ix41lOWq5X4e2/F6brPKJQY=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - } - }, - "moduleExtensions": { - "@@apple_support~1.5.0//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "pMLFCYaRPkgXPQ8vtuNkMfiHfPmRBy6QJfnid4sWfv0=", - "accumulatedFileDigests": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc": { - "bzlFile": "@@apple_support~1.5.0//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", - "attributes": { - "name": "apple_support~1.5.0~apple_cc_configure_extension~local_config_apple_cc" - } - }, - "local_config_apple_cc_toolchains": { - "bzlFile": "@@apple_support~1.5.0//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", - "attributes": { - "name": "apple_support~1.5.0~apple_cc_configure_extension~local_config_apple_cc_toolchains" - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@bazel_tools//tools/cpp:cc_configure.bzl%cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "mcsWHq3xORJexV5/4eCvNOLxFOQKV6eli3fkr+tEaqE=", - "accumulatedFileDigests": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_cc": { - "bzlFile": "@@bazel_tools//tools/cpp:cc_configure.bzl", - "ruleClassName": "cc_autoconf", - "attributes": { - "name": "bazel_tools~cc_configure_extension~local_config_cc" - } - }, - "local_config_cc_toolchains": { - "bzlFile": "@@bazel_tools//tools/cpp:cc_configure.bzl", - "ruleClassName": "cc_autoconf_toolchains", - "attributes": { - "name": "bazel_tools~cc_configure_extension~local_config_cc_toolchains" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_tools", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@bazel_tools//tools/osx:xcode_configure.bzl%xcode_configure_extension": { - "general": { - "bzlTransitiveDigest": "Qh2bWTU6QW6wkrd87qrU4YeY+SG37Nvw3A0PR4Y0L2Y=", - "accumulatedFileDigests": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_xcode": { - "bzlFile": "@@bazel_tools//tools/osx:xcode_configure.bzl", - "ruleClassName": "xcode_autoconf", - "attributes": { - "name": "bazel_tools~xcode_configure_extension~local_config_xcode", - "xcode_locator": "@bazel_tools//tools/osx:xcode_locator.m", - "remote_xcode": "" - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@bazel_tools//tools/sh:sh_configure.bzl%sh_configure_extension": { - "general": { - "bzlTransitiveDigest": "hp4NgmNjEg5+xgvzfh6L83bt9/aiiWETuNpwNuF1MSU=", - "accumulatedFileDigests": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_sh": { - "bzlFile": "@@bazel_tools//tools/sh:sh_configure.bzl", - "ruleClassName": "sh_config", - "attributes": { - "name": "bazel_tools~sh_configure_extension~local_config_sh" - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@rules_java~7.1.0//java:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "D02GmifxnV/IhYgspsJMDZ/aE8HxAjXgek5gi6FSto4=", - "accumulatedFileDigests": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "remotejdk21_linux_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_linux_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_linux//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_linux//:jdk\",\n)\n" - } - }, - "remotejdk17_linux_s390x_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_s390x_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:s390x\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_s390x//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:s390x\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_s390x//:jdk\",\n)\n" - } - }, - "remotejdk17_macos_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_macos_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_macos//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_macos//:jdk\",\n)\n" - } - }, - "remotejdk21_macos_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_macos_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_macos_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_macos_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk17_linux_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk21_macos_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_macos_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 21,\n)\n", - "sha256": "2a7a99a3ea263dbd8d32a67d1e6e363ba8b25c645c826f5e167a02bbafaff1fa", - "strip_prefix": "zulu21.28.85-ca-jdk21.0.0-macosx_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz" - ] - } - }, - "remotejdk17_linux_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux//:jdk\",\n)\n" - } - }, - "remotejdk17_macos_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_macos_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "314b04568ec0ae9b36ba03c9cbd42adc9e1265f74678923b19297d66eb84dcca", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-macosx_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-macosx_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-macosx_aarch64.tar.gz" - ] - } - }, - "remote_java_tools_windows": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remote_java_tools_windows", - "sha256": "c5c70c214a350f12cbf52da8270fa43ba629b795f3dd328028a38f8f0d39c2a1", - "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v13.1/java_tools_windows-v13.1.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v13.1/java_tools_windows-v13.1.zip" - ] - } - }, - "remotejdk11_win": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_win", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "43408193ce2fa0862819495b5ae8541085b95660153f2adcf91a52d3a1710e83", - "strip_prefix": "zulu11.66.15-ca-jdk11.0.20-win_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-win_x64.zip", - "https://cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-win_x64.zip" - ] - } - }, - "remotejdk11_win_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_win_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_win//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_win//:jdk\",\n)\n" - } - }, - "remotejdk11_linux_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "54174439f2b3fddd11f1048c397fe7bb45d4c9d66d452d6889b013d04d21c4de", - "strip_prefix": "zulu11.66.15-ca-jdk11.0.20-linux_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-linux_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-linux_aarch64.tar.gz" - ] - } - }, - "remotejdk17_linux": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "b9482f2304a1a68a614dfacddcf29569a72f0fac32e6c74f83dc1b9a157b8340", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-linux_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-linux_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-linux_x64.tar.gz" - ] - } - }, - "remotejdk11_linux_s390x_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_s390x_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:s390x\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_s390x//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:s390x\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_s390x//:jdk\",\n)\n" - } - }, - "remotejdk11_linux_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux//:jdk\",\n)\n" - } - }, - "remotejdk11_macos": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_macos", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "bcaab11cfe586fae7583c6d9d311c64384354fb2638eb9a012eca4c3f1a1d9fd", - "strip_prefix": "zulu11.66.15-ca-jdk11.0.20-macosx_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-macosx_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-macosx_x64.tar.gz" - ] - } - }, - "remotejdk11_win_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_win_arm64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "b8a28e6e767d90acf793ea6f5bed0bb595ba0ba5ebdf8b99f395266161e53ec2", - "strip_prefix": "jdk-11.0.13+8", - "urls": [ - "https://mirror.bazel.build/aka.ms/download-jdk/microsoft-jdk-11.0.13.8.1-windows-aarch64.zip" - ] - } - }, - "remotejdk17_macos": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_macos", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "640453e8afe8ffe0fb4dceb4535fb50db9c283c64665eebb0ba68b19e65f4b1f", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-macosx_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-macosx_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-macosx_x64.tar.gz" - ] - } - }, - "remotejdk21_macos": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_macos", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 21,\n)\n", - "sha256": "9639b87db586d0c89f7a9892ae47f421e442c64b97baebdff31788fbe23265bd", - "strip_prefix": "zulu21.28.85-ca-jdk21.0.0-macosx_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz" - ] - } - }, - "remotejdk21_macos_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_macos_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_macos//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_macos//:jdk\",\n)\n" - } - }, - "remotejdk17_macos_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_macos_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_macos_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_macos_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk17_win": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_win", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "192f2afca57701de6ec496234f7e45d971bf623ff66b8ee4a5c81582054e5637", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-win_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-win_x64.zip", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-win_x64.zip" - ] - } - }, - "remotejdk11_macos_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_macos_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_macos_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_macos_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk11_linux_ppc64le_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_ppc64le_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:ppc\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_ppc64le//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:ppc\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_ppc64le//:jdk\",\n)\n" - } - }, - "remotejdk21_linux": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_linux", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 21,\n)\n", - "sha256": "0c0eadfbdc47a7ca64aeab51b9c061f71b6e4d25d2d87674512e9b6387e9e3a6", - "strip_prefix": "zulu21.28.85-ca-jdk21.0.0-linux_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz" - ] - } - }, - "remote_java_tools_linux": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remote_java_tools_linux", - "sha256": "d134da9b04c9023fb6e56a5d4bffccee73f7bc9572ddc4e747778dacccd7a5a7", - "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v13.1/java_tools_linux-v13.1.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v13.1/java_tools_linux-v13.1.zip" - ] - } - }, - "remotejdk21_win": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_win", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 21,\n)\n", - "sha256": "e9959d500a0d9a7694ac243baf657761479da132f0f94720cbffd092150bd802", - "strip_prefix": "zulu21.28.85-ca-jdk21.0.0-win_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip" - ] - } - }, - "remotejdk21_linux_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_linux_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 21,\n)\n", - "sha256": "1fb64b8036c5d463d8ab59af06bf5b6b006811e6012e3b0eb6bccf57f1c55835", - "strip_prefix": "zulu21.28.85-ca-jdk21.0.0-linux_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz" - ] - } - }, - "remotejdk11_linux_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_linux_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk11_linux_s390x": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_s390x", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "a58fc0361966af0a5d5a31a2d8a208e3c9bb0f54f345596fd80b99ea9a39788b", - "strip_prefix": "jdk-11.0.15+10", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15+10/OpenJDK11U-jdk_s390x_linux_hotspot_11.0.15_10.tar.gz", - "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15+10/OpenJDK11U-jdk_s390x_linux_hotspot_11.0.15_10.tar.gz" - ] - } - }, - "remotejdk17_linux_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "6531cef61e416d5a7b691555c8cf2bdff689201b8a001ff45ab6740062b44313", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-linux_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-linux_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-linux_aarch64.tar.gz" - ] - } - }, - "remotejdk17_win_arm64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_win_arm64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:arm64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_win_arm64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:arm64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_win_arm64//:jdk\",\n)\n" - } - }, - "remotejdk11_linux": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "a34b404f87a08a61148b38e1416d837189e1df7a040d949e743633daf4695a3c", - "strip_prefix": "zulu11.66.15-ca-jdk11.0.20-linux_x64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-linux_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-linux_x64.tar.gz" - ] - } - }, - "remotejdk11_macos_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_macos_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_macos//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:macos\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_macos//:jdk\",\n)\n" - } - }, - "remotejdk17_linux_ppc64le_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_ppc64le_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:ppc\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_ppc64le//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:ppc\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_linux_ppc64le//:jdk\",\n)\n" - } - }, - "remotejdk17_win_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_win_arm64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "6802c99eae0d788e21f52d03cab2e2b3bf42bc334ca03cbf19f71eb70ee19f85", - "strip_prefix": "zulu17.44.53-ca-jdk17.0.8.1-win_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-win_aarch64.zip", - "https://cdn.azul.com/zulu/bin/zulu17.44.53-ca-jdk17.0.8.1-win_aarch64.zip" - ] - } - }, - "remote_java_tools_darwin_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remote_java_tools_darwin_arm64", - "sha256": "dab5bb87ec43e980faea6e1cec14bafb217b8e2f5346f53aa784fd715929a930", - "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v13.1/java_tools_darwin_arm64-v13.1.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v13.1/java_tools_darwin_arm64-v13.1.zip" - ] - } - }, - "remotejdk17_linux_ppc64le": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_ppc64le", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "00a4c07603d0218cd678461b5b3b7e25b3253102da4022d31fc35907f21a2efd", - "strip_prefix": "jdk-17.0.8.1+1", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_ppc64le_linux_hotspot_17.0.8.1_1.tar.gz", - "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_ppc64le_linux_hotspot_17.0.8.1_1.tar.gz" - ] - } - }, - "remotejdk21_linux_aarch64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_linux_aarch64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_linux_aarch64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:linux\", \"@platforms//cpu:aarch64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_linux_aarch64//:jdk\",\n)\n" - } - }, - "remotejdk11_win_arm64_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_win_arm64_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_11\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"11\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:arm64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk11_win_arm64//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:arm64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk11_win_arm64//:jdk\",\n)\n" - } - }, - "local_jdk": { - "bzlFile": "@@rules_java~7.1.0//toolchains:local_java_repository.bzl", - "ruleClassName": "_local_java_repository_rule", - "attributes": { - "name": "rules_java~7.1.0~toolchains~local_jdk", - "java_home": "", - "version": "", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = {RUNTIME_VERSION},\n)\n" - } - }, - "remote_java_tools_darwin_x86_64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remote_java_tools_darwin_x86_64", - "sha256": "0db40d8505a2b65ef0ed46e4256757807db8162f7acff16225be57c1d5726dbc", - "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v13.1/java_tools_darwin_x86_64-v13.1.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v13.1/java_tools_darwin_x86_64-v13.1.zip" - ] - } - }, - "remote_java_tools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remote_java_tools", - "sha256": "286bdbbd66e616fc4ed3f90101418729a73baa7e8c23a98ffbef558f74c0ad14", - "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v13.1/java_tools-v13.1.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v13.1/java_tools-v13.1.zip" - ] - } - }, - "remotejdk17_linux_s390x": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_linux_s390x", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 17,\n)\n", - "sha256": "ffacba69c6843d7ca70d572489d6cc7ab7ae52c60f0852cedf4cf0d248b6fc37", - "strip_prefix": "jdk-17.0.8.1+1", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_s390x_linux_hotspot_17.0.8.1_1.tar.gz", - "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_s390x_linux_hotspot_17.0.8.1_1.tar.gz" - ] - } - }, - "remotejdk17_win_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk17_win_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_17\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"17\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk17_win//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk17_win//:jdk\",\n)\n" - } - }, - "remotejdk11_linux_ppc64le": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_linux_ppc64le", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "a8fba686f6eb8ae1d1a9566821dbd5a85a1108b96ad857fdbac5c1e4649fc56f", - "strip_prefix": "jdk-11.0.15+10", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15+10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.15_10.tar.gz", - "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15+10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.15_10.tar.gz" - ] - } - }, - "remotejdk11_macos_aarch64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk11_macos_aarch64", - "build_file_content": "load(\"@rules_java//java:defs.bzl\", \"java_runtime\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files([\"WORKSPACE\", \"BUILD.bazel\"])\n\nfilegroup(\n name = \"jre\",\n srcs = glob(\n [\n \"jre/bin/**\",\n \"jre/lib/**\",\n ],\n allow_empty = True,\n # In some configurations, Java browser plugin is considered harmful and\n # common antivirus software blocks access to npjp2.dll interfering with Bazel,\n # so do not include it in JRE on Windows.\n exclude = [\"jre/bin/plugin2/**\"],\n ),\n)\n\nfilegroup(\n name = \"jdk-bin\",\n srcs = glob(\n [\"bin/**\"],\n # The JDK on Windows sometimes contains a directory called\n # \"%systemroot%\", which is not a valid label.\n exclude = [\"**/*%*/**\"],\n ),\n)\n\n# This folder holds security policies.\nfilegroup(\n name = \"jdk-conf\",\n srcs = glob(\n [\"conf/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-include\",\n srcs = glob(\n [\"include/**\"],\n allow_empty = True,\n ),\n)\n\nfilegroup(\n name = \"jdk-lib\",\n srcs = glob(\n [\"lib/**\", \"release\"],\n allow_empty = True,\n exclude = [\n \"lib/missioncontrol/**\",\n \"lib/visualvm/**\",\n ],\n ),\n)\n\njava_runtime(\n name = \"jdk\",\n srcs = [\n \":jdk-bin\",\n \":jdk-conf\",\n \":jdk-include\",\n \":jdk-lib\",\n \":jre\",\n ],\n # Provide the 'java` binary explicitly so that the correct path is used by\n # Bazel even when the host platform differs from the execution platform.\n # Exactly one of the two globs will be empty depending on the host platform.\n # When --incompatible_disallow_empty_glob is enabled, each individual empty\n # glob will fail without allow_empty = True, even if the overall result is\n # non-empty.\n java = glob([\"bin/java.exe\", \"bin/java\"], allow_empty = True)[0],\n version = 11,\n)\n", - "sha256": "7632bc29f8a4b7d492b93f3bc75a7b61630894db85d136456035ab2a24d38885", - "strip_prefix": "zulu11.66.15-ca-jdk11.0.20-macosx_aarch64", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-macosx_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu11.66.15-ca-jdk11.0.20-macosx_aarch64.tar.gz" - ] - } - }, - "remotejdk21_win_toolchain_config_repo": { - "bzlFile": "@@rules_java~7.1.0//toolchains:remote_java_repository.bzl", - "ruleClassName": "_toolchain_config", - "attributes": { - "name": "rules_java~7.1.0~toolchains~remotejdk21_win_toolchain_config_repo", - "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_win//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_win//:jdk\",\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_java~7.1.0", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_java~7.1.0", - "remote_java_tools", - "rules_java~7.1.0~toolchains~remote_java_tools" - ] - ] - } - } - } -} diff --git a/br/cmd/br/debug.go b/br/cmd/br/debug.go index df9795fd32482..2ca774b287159 100644 --- a/br/cmd/br/debug.go +++ b/br/cmd/br/debug.go @@ -80,7 +80,7 @@ func newCheckSumCommand() *cobra.Command { } reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo) - dbs, err := metautil.LoadBackupTables(ctx, reader) + dbs, err := metautil.LoadBackupTables(ctx, reader, false) if err != nil { return errors.Trace(err) } @@ -173,7 +173,7 @@ func newBackupMetaValidateCommand() *cobra.Command { return errors.Trace(err) } reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo) - dbs, err := metautil.LoadBackupTables(ctx, reader) + dbs, err := metautil.LoadBackupTables(ctx, reader, false) if err != nil { log.Error("load tables failed", zap.Error(err)) return errors.Trace(err) diff --git a/br/cmd/tidb-lightning-ctl/main.go b/br/cmd/tidb-lightning-ctl/main.go index 4ed895da3c32c..8df8d471bd4ca 100644 --- a/br/cmd/tidb-lightning-ctl/main.go +++ b/br/cmd/tidb-lightning-ctl/main.go @@ -193,14 +193,14 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common return errors.Trace(err) } - var lastErr error + var errs []error for _, table := range targetTables { fmt.Fprintln(os.Stderr, "Dropping table:", table.TableName) err := target.DropTable(ctx, table.TableName) if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while dropping table:", err) - lastErr = err + errs = append(errs, err) } } @@ -218,18 +218,18 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common err := engine.Cleanup(cfg.TikvImporter.SortedKVDir) if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while cleanup engine:", err) - lastErr = err + errs = append(errs, err) } } } } // try clean up metas - if lastErr == nil { - lastErr = lightning.CleanupMetas(ctx, cfg, tableName) + if len(errs) == 0 { + errs = append(errs, lightning.CleanupMetas(ctx, cfg, tableName)) } - return errors.Trace(lastErr) + return errors.Trace(errors.Join(errs...)) } func checkpointDump(ctx context.Context, cfg *config.Config, dumpFolder string) error { diff --git a/br/pkg/backup/prepare_snap/BUILD.bazel b/br/pkg/backup/prepare_snap/BUILD.bazel index ce61679db0c53..65f2772fc3858 100644 --- a/br/pkg/backup/prepare_snap/BUILD.bazel +++ b/br/pkg/backup/prepare_snap/BUILD.bazel @@ -35,7 +35,7 @@ go_test( timeout = "short", srcs = ["prepare_test.go"], flaky = True, - shard_count = 7, + shard_count = 8, deps = [ ":prepare_snap", "//br/pkg/utils", diff --git a/br/pkg/backup/prepare_snap/prepare.go b/br/pkg/backup/prepare_snap/prepare.go index f3ccdff2b1163..e799f8c346444 100644 --- a/br/pkg/backup/prepare_snap/prepare.go +++ b/br/pkg/backup/prepare_snap/prepare.go @@ -155,7 +155,7 @@ func (p *Preparer) DriveLoopAndWaitPrepare(ctx context.Context) error { zap.Int("retry_limit", p.RetryLimit), zap.Duration("lease_duration", p.LeaseDuration)) p.retryTime = 0 - if err := p.prepareConnections(ctx); err != nil { + if err := p.PrepareConnections(ctx); err != nil { log.Error("failed to prepare connections", logutil.ShortError(err)) return errors.Annotate(err, "failed to prepare connections") } @@ -386,23 +386,31 @@ func (p *Preparer) sendWaitApply(ctx context.Context, reqs pendingRequests) erro } func (p *Preparer) streamOf(ctx context.Context, storeID uint64) (*prepareStream, error) { - s, ok := p.clients[storeID] + _, ok := p.clients[storeID] if !ok { + log.Warn("stream of store found a store not established connection", zap.Uint64("store", storeID)) cli, err := p.env.ConnectToStore(ctx, storeID) if err != nil { return nil, errors.Annotatef(err, "failed to dial store %d", storeID) } - s = new(prepareStream) - s.storeID = storeID - s.output = p.eventChan - s.leaseDuration = p.LeaseDuration - err = s.InitConn(ctx, cli) - if err != nil { - return nil, err + if err := p.createAndCacheStream(ctx, cli, storeID); err != nil { + return nil, errors.Annotatef(err, "failed to create and cache stream for store %d", storeID) } - p.clients[storeID] = s } - return s, nil + return p.clients[storeID], nil +} + +func (p *Preparer) createAndCacheStream(ctx context.Context, cli PrepareClient, storeID uint64) error { + s := new(prepareStream) + s.storeID = storeID + s.output = p.eventChan + s.leaseDuration = p.LeaseDuration + err := s.InitConn(ctx, cli) + if err != nil { + return err + } + p.clients[storeID] = s + return nil } func (p *Preparer) pushWaitApply(reqs pendingRequests, region Region) { @@ -415,17 +423,31 @@ func (p *Preparer) pushWaitApply(reqs pendingRequests, region Region) { p.inflightReqs[region.GetMeta().Id] = *region.GetMeta() } -func (p *Preparer) prepareConnections(ctx context.Context) error { +// PrepareConnections prepares the connections for each store. +// This will pause the admin commands for each store. +func (p *Preparer) PrepareConnections(ctx context.Context) error { log.Info("Preparing connections to stores.") stores, err := p.env.GetAllLiveStores(ctx) if err != nil { return errors.Annotate(err, "failed to get all live stores") } + + log.Info("Start to initialize the connections.", zap.Int("stores", len(stores))) + clients := map[uint64]PrepareClient{} for _, store := range stores { - _, err := p.streamOf(ctx, store.Id) + cli, err := p.env.ConnectToStore(ctx, store.Id) if err != nil { - return errors.Annotatef(err, "failed to prepare connection to store %d", store.Id) + return errors.Annotatef(err, "failed to dial the store %d", store.Id) + } + clients[store.Id] = cli + } + + for id, cli := range clients { + log.Info("Start to pause the admin commands.", zap.Uint64("store", id)) + if err := p.createAndCacheStream(ctx, cli, id); err != nil { + return errors.Annotatef(err, "failed to create and cache stream for store %d", id) } } + return nil } diff --git a/br/pkg/backup/prepare_snap/prepare_test.go b/br/pkg/backup/prepare_snap/prepare_test.go index 163d446e7314b..5f3d2e28d44dc 100644 --- a/br/pkg/backup/prepare_snap/prepare_test.go +++ b/br/pkg/backup/prepare_snap/prepare_test.go @@ -110,6 +110,7 @@ type mockStores struct { mu sync.Mutex stores map[uint64]*mockStore onCreateStore func(*mockStore) + connectDelay func(uint64) <-chan struct{} onConnectToStore func(uint64) error pdc *tikv.RegionCache @@ -117,8 +118,16 @@ type mockStores struct { func newTestEnv(pdc pd.Client) *mockStores { r := tikv.NewRegionCache(pdc) + stores, err := pdc.GetAllStores(context.Background()) + if err != nil { + panic(err) + } + ss := map[uint64]*mockStore{} + for _, store := range stores { + ss[store.Id] = nil + } ms := &mockStores{ - stores: map[uint64]*mockStore{}, + stores: ss, pdc: r, onCreateStore: func(ms *mockStore) {}, } @@ -138,7 +147,14 @@ func (m *mockStores) GetAllLiveStores(ctx context.Context) ([]*metapb.Store, err func (m *mockStores) ConnectToStore(ctx context.Context, storeID uint64) (PrepareClient, error) { m.mu.Lock() - defer m.mu.Unlock() + defer func() { + m.mu.Unlock() + if m.connectDelay != nil { + if ch := m.connectDelay(storeID); ch != nil { + <-ch + } + } + }() if m.onConnectToStore != nil { err := m.onConnectToStore(storeID) @@ -147,8 +163,8 @@ func (m *mockStores) ConnectToStore(ctx context.Context, storeID uint64) (Prepar } } - _, ok := m.stores[storeID] - if !ok { + s, ok := m.stores[storeID] + if !ok || s == nil { m.stores[storeID] = &mockStore{ output: make(chan brpb.PrepareSnapshotBackupResponse, 16), successRegions: []metapb.Region{}, @@ -456,3 +472,41 @@ func TestSplitEnv(t *testing.T) { require.Equal(t, cc.PrepareClient.(*counterClient).send, 1) require.ElementsMatch(t, cc.PrepareClient.(*counterClient).regions, tinyRequest.Regions) } + +func TestConnectionDelay(t *testing.T) { + log.SetLevel(zapcore.DebugLevel) + req := require.New(t) + pdc := fakeCluster(t, 3, dummyRegions(100)...) + ms := newTestEnv(pdc) + called := 0 + delayConn := make(chan struct{}) + blocked := make(chan struct{}, 64) + ms.connectDelay = func(i uint64) <-chan struct{} { + called += 1 + if called == 2 { + blocked <- struct{}{} + return delayConn + } + return nil + } + ctx := context.Background() + prep := New(ms) + connectionPrepareResult := make(chan error) + go func() { + connectionPrepareResult <- prep.PrepareConnections(ctx) + }() + <-blocked + ms.mu.Lock() + nonNilStore := 0 + for id, store := range ms.stores { + // We must not create and lease (i.e. reject admin command from any tikv) here. + if store != nil { + req.True(store.leaseUntil.Before(time.Now()), "%d->%s", id, store.leaseUntil) + nonNilStore += 1 + } + } + req.GreaterOrEqual(nonNilStore, 2) + ms.mu.Unlock() + delayConn <- struct{}{} + req.NoError(<-connectionPrepareResult) +} diff --git a/br/pkg/backup/prepare_snap/stream.go b/br/pkg/backup/prepare_snap/stream.go index 9e253fc4a4d37..1108731fa5002 100644 --- a/br/pkg/backup/prepare_snap/stream.go +++ b/br/pkg/backup/prepare_snap/stream.go @@ -70,6 +70,7 @@ func (p *prepareStream) InitConn(ctx context.Context, cli PrepareClient) error { p.cli = cli p.clientLoopHandle, ctx = errgroup.WithContext(ctx) ctx, p.stopBgTasks = context.WithCancel(ctx) + log.Info("initializing", zap.Uint64("store", p.storeID)) return p.GoLeaseLoop(ctx, p.leaseDuration) } diff --git a/br/pkg/checksum/validate.go b/br/pkg/checksum/validate.go index 427d30200c073..c23ff8884d5d0 100644 --- a/br/pkg/checksum/validate.go +++ b/br/pkg/checksum/validate.go @@ -33,7 +33,7 @@ func FastChecksum( errCh := make(chan error) go func() { reader := metautil.NewMetaReader(backupMeta, storage, cipher) - if err := reader.ReadSchemasFiles(ctx, ch); err != nil { + if err := reader.ReadSchemasFiles(ctx, ch, metautil.SkipStats); err != nil { errCh <- errors.Trace(err) } close(ch) diff --git a/br/pkg/lightning/backend/backend.go b/br/pkg/lightning/backend/backend.go index 3cbae8a97cdb2..55d01247702cb 100644 --- a/br/pkg/lightning/backend/backend.go +++ b/br/pkg/lightning/backend/backend.go @@ -127,6 +127,10 @@ type CheckCtx struct { // TargetInfoGetter defines the interfaces to get target information. type TargetInfoGetter interface { + // FetchRemoteDBModels obtains the models of all databases. Currently, only + // the database name is filled. + FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) + // FetchRemoteTableModels obtains the models of all tables given the schema // name. The returned table info does not need to be precise if the encoder, // is not requiring them, but must at least fill in the following fields for diff --git a/br/pkg/lightning/backend/external/engine.go b/br/pkg/lightning/backend/external/engine.go index 98fdc27dd1ff1..b86b45feed694 100644 --- a/br/pkg/lightning/backend/external/engine.go +++ b/br/pkg/lightning/backend/external/engine.go @@ -17,7 +17,6 @@ package external import ( "bytes" "context" - "encoding/hex" "sort" "sync" "time" @@ -25,7 +24,6 @@ import ( "github.com/cockroachdb/pebble" "github.com/docker/go-units" "github.com/jfcg/sorty/v2" - "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/config" @@ -227,7 +225,7 @@ func getFilesReadConcurrency( startKey, endKey []byte, ) ([]uint64, []uint64, error) { result := make([]uint64, len(statsFiles)) - offsets, err := seekPropsOffsets(ctx, []kv.Key{startKey, endKey}, statsFiles, storage, false) + offsets, err := seekPropsOffsets(ctx, []kv.Key{startKey, endKey}, statsFiles, storage) if err != nil { return nil, nil, err } @@ -387,43 +385,6 @@ func (e *Engine) buildIngestData(keys, values [][]byte, buf []*membuf.Buffer) *M // LargeRegionSplitDataThreshold is exposed for test. var LargeRegionSplitDataThreshold = int(config.SplitRegionSize) -// createMergeIter is unused now. -// TODO(lance6716): check the performance of new design and remove it. -func (e *Engine) createMergeIter(ctx context.Context, start kv.Key) (*MergeKVIter, error) { - logger := logutil.Logger(ctx) - - var offsets []uint64 - if len(e.statsFiles) == 0 { - offsets = make([]uint64, len(e.dataFiles)) - logger.Info("no stats files", - zap.String("startKey", hex.EncodeToString(start))) - } else { - offs, err := seekPropsOffsets(ctx, []kv.Key{start}, e.statsFiles, e.storage, e.checkHotspot) - if err != nil { - return nil, errors.Trace(err) - } - offsets = offs[0] - logger.Debug("seek props offsets", - zap.Uint64s("offsets", offsets), - zap.String("startKey", hex.EncodeToString(start)), - zap.Strings("dataFiles", e.dataFiles), - zap.Strings("statsFiles", e.statsFiles)) - } - iter, err := NewMergeKVIter( - ctx, - e.dataFiles, - offsets, - e.storage, - 64*1024, - e.checkHotspot, - e.mergerIterConcurrency, - ) - if err != nil { - return nil, errors.Trace(err) - } - return iter, nil -} - // KVStatistics returns the total kv size and total kv count. func (e *Engine) KVStatistics() (totalKVSize int64, totalKVCount int64) { return e.totalKVSize, e.totalKVCount diff --git a/br/pkg/lightning/backend/external/engine_test.go b/br/pkg/lightning/backend/external/engine_test.go index f810fb5c1f952..6750f22fc5329 100644 --- a/br/pkg/lightning/backend/external/engine_test.go +++ b/br/pkg/lightning/backend/external/engine_test.go @@ -15,106 +15,18 @@ package external import ( - "bytes" "context" "fmt" "path" - "slices" - "strconv" "testing" - "time" "github.com/cockroachdb/pebble" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/membuf" - "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/util/codec" "github.com/stretchr/testify/require" - "golang.org/x/exp/rand" ) -func TestIter(t *testing.T) { - seed := time.Now().Unix() - rand.Seed(uint64(seed)) - t.Logf("seed: %d", seed) - - totalKV := 300 - kvPairs := make([]common.KvPair, totalKV) - for i := range kvPairs { - keyBuf := make([]byte, rand.Intn(10)+1) - rand.Read(keyBuf) - // make sure the key is unique - kvPairs[i].Key = append(keyBuf, byte(i/255), byte(i%255)) - valBuf := make([]byte, rand.Intn(10)+1) - rand.Read(valBuf) - kvPairs[i].Val = valBuf - } - - sortedKVPairs := make([]common.KvPair, totalKV) - copy(sortedKVPairs, kvPairs) - slices.SortFunc(sortedKVPairs, func(i, j common.KvPair) int { - return bytes.Compare(i.Key, j.Key) - }) - - ctx := context.Background() - store := storage.NewMemStorage() - - for i := 0; i < 3; i++ { - w := NewWriterBuilder(). - SetMemorySizeLimit(uint64(rand.Intn(100)+1)). - SetPropSizeDistance(uint64(rand.Intn(50)+1)). - SetPropKeysDistance(uint64(rand.Intn(10)+1)). - Build(store, "/subtask", strconv.Itoa(i)) - kvStart := i * 100 - kvEnd := (i + 1) * 100 - for j := kvStart; j < kvEnd; j++ { - err := w.WriteRow(ctx, kvPairs[j].Key, kvPairs[j].Val, nil) - require.NoError(t, err) - } - err := w.Close(ctx) - require.NoError(t, err) - } - - dataFiles, statFiles, err := GetAllFileNames(ctx, store, "/subtask") - require.NoError(t, err) - - engine := Engine{ - storage: store, - dataFiles: dataFiles, - statsFiles: statFiles, - } - iter, err := engine.createMergeIter(ctx, sortedKVPairs[0].Key) - require.NoError(t, err) - got := make([]common.KvPair, 0, totalKV) - for iter.Next() { - got = append(got, common.KvPair{ - Key: iter.Key(), - Val: iter.Value(), - }) - } - require.NoError(t, iter.Error()) - require.Equal(t, sortedKVPairs, got) - - pickStartIdx := rand.Intn(len(sortedKVPairs)) - startKey := sortedKVPairs[pickStartIdx].Key - iter, err = engine.createMergeIter(ctx, startKey) - require.NoError(t, err) - got = make([]common.KvPair, 0, totalKV) - for iter.Next() { - got = append(got, common.KvPair{ - Key: iter.Key(), - Val: iter.Value(), - }) - } - require.NoError(t, iter.Error()) - // got keys must be ascending - for i := 1; i < len(got); i++ { - require.True(t, bytes.Compare(got[i-1].Key, got[i].Key) < 0) - } - // the first key must be less than or equal to startKey - require.True(t, bytes.Compare(got[0].Key, startKey) <= 0) -} - func testGetFirstAndLastKey( t *testing.T, data common.IngestData, diff --git a/br/pkg/lightning/backend/external/iter.go b/br/pkg/lightning/backend/external/iter.go index ec83bca3d859c..5560bb463c059 100644 --- a/br/pkg/lightning/backend/external/iter.go +++ b/br/pkg/lightning/backend/external/iter.go @@ -793,7 +793,6 @@ func NewMergePropIter( ctx context.Context, multiStat []MultipleFilesStat, exStorage storage.ExternalStorage, - _ bool, ) (*MergePropIter, error) { closeReaderFlag := false readerOpeners := make([]readerOpenerFn[*rangeProperty, mergePropBaseIter], 0, len(multiStat)) diff --git a/br/pkg/lightning/backend/external/merge_v2.go b/br/pkg/lightning/backend/external/merge_v2.go index 5f5a6833bb050..1d694f0e2b2ae 100644 --- a/br/pkg/lightning/backend/external/merge_v2.go +++ b/br/pkg/lightning/backend/external/merge_v2.go @@ -65,7 +65,6 @@ func MergeOverlappingFilesV2( math.MaxInt64, int64(4*size.GB), math.MaxInt64, - checkHotspot, ) if err != nil { return err diff --git a/br/pkg/lightning/backend/external/split.go b/br/pkg/lightning/backend/external/split.go index 489d3f3433a3e..66ad46862ab62 100644 --- a/br/pkg/lightning/backend/external/split.go +++ b/br/pkg/lightning/backend/external/split.go @@ -96,7 +96,6 @@ func NewRangeSplitter( externalStorage storage.ExternalStorage, rangesGroupSize, rangesGroupKeys int64, maxRangeSize, maxRangeKeys int64, - checkHotSpot bool, ) (*RangeSplitter, error) { logger := logutil.Logger(ctx) overlaps := make([]int64, 0, len(multiFileStat)) @@ -112,9 +111,8 @@ func NewRangeSplitter( zap.Int64("rangesGroupKeys", rangesGroupKeys), zap.Int64("maxRangeSize", maxRangeSize), zap.Int64("maxRangeKeys", maxRangeKeys), - zap.Bool("checkHotSpot", checkHotSpot), ) - propIter, err := NewMergePropIter(ctx, multiFileStat, externalStorage, checkHotSpot) + propIter, err := NewMergePropIter(ctx, multiFileStat, externalStorage) if err != nil { return nil, err } diff --git a/br/pkg/lightning/backend/external/split_test.go b/br/pkg/lightning/backend/external/split_test.go index ef675b032f516..9141f07f534d8 100644 --- a/br/pkg/lightning/backend/external/split_test.go +++ b/br/pkg/lightning/backend/external/split_test.go @@ -64,7 +64,7 @@ func TestGeneralProperties(t *testing.T) { require.NoError(t, err) multiFileStat := mockOneMultiFileStat(dataFiles, statFiles) splitter, err := NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 30, 1000, 1, true, + ctx, multiFileStat, memStore, 1000, 30, 1000, 1, ) var lastEndKey []byte notExhausted: @@ -124,7 +124,7 @@ func TestOnlyOneGroup(t *testing.T) { multiFileStat := mockOneMultiFileStat(dataFiles, statFiles) splitter, err := NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 30, 1000, 10, true, + ctx, multiFileStat, memStore, 1000, 30, 1000, 10, ) require.NoError(t, err) endKey, dataFiles, statFiles, splitKeys, err := splitter.SplitOneRangesGroup() @@ -136,7 +136,7 @@ func TestOnlyOneGroup(t *testing.T) { require.NoError(t, splitter.Close()) splitter, err = NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 30, 1000, 1, true, + ctx, multiFileStat, memStore, 1000, 30, 1000, 1, ) require.NoError(t, err) endKey, dataFiles, statFiles, splitKeys, err = splitter.SplitOneRangesGroup() @@ -170,7 +170,7 @@ func TestSortedData(t *testing.T) { multiFileStat := mockOneMultiFileStat(dataFiles, statFiles) splitter, err := NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, int64(rangesGroupKV), 1000, 10, true, + ctx, multiFileStat, memStore, 1000, int64(rangesGroupKV), 1000, 10, ) require.NoError(t, err) @@ -254,7 +254,7 @@ func TestRangeSplitterStrictCase(t *testing.T) { multiFileStat := []MultipleFilesStat{multi[0], multi2[0]} // group keys = 2, region keys = 1 splitter, err := NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 2, 1000, 1, true, + ctx, multiFileStat, memStore, 1000, 2, 1000, 1, ) require.NoError(t, err) @@ -337,7 +337,7 @@ func TestExactlyKeyNum(t *testing.T) { // maxRangeKeys = 3 splitter, err := NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 100, 1000, 3, true, + ctx, multiFileStat, memStore, 1000, 100, 1000, 3, ) require.NoError(t, err) endKey, splitDataFiles, splitStatFiles, splitKeys, err := splitter.SplitOneRangesGroup() @@ -349,7 +349,7 @@ func TestExactlyKeyNum(t *testing.T) { // rangesGroupKeys = 3 splitter, err = NewRangeSplitter( - ctx, multiFileStat, memStore, 1000, 3, 1000, 1, true, + ctx, multiFileStat, memStore, 1000, 3, 1000, 1, ) require.NoError(t, err) endKey, splitDataFiles, splitStatFiles, splitKeys, err = splitter.SplitOneRangesGroup() @@ -482,7 +482,6 @@ func Test3KFilesRangeSplitter(t *testing.T) { int64(math.MaxInt64), int64(config.SplitRegionSize), int64(config.SplitRegionKeys), - false, ) require.NoError(t, err) var lastEndKey []byte diff --git a/br/pkg/lightning/backend/external/testutil.go b/br/pkg/lightning/backend/external/testutil.go index 38de84aaf9c55..53ebb45194b03 100644 --- a/br/pkg/lightning/backend/external/testutil.go +++ b/br/pkg/lightning/backend/external/testutil.go @@ -54,7 +54,6 @@ func testReadAndCompare( math.MaxInt64, 4*1024*1024*1024, math.MaxInt64, - true, ) require.NoError(t, err) diff --git a/br/pkg/lightning/backend/external/util.go b/br/pkg/lightning/backend/external/util.go index 3a6fa7e41e08d..8433dc262aeae 100644 --- a/br/pkg/lightning/backend/external/util.go +++ b/br/pkg/lightning/backend/external/util.go @@ -18,31 +18,41 @@ import ( "bytes" "context" "fmt" + "io" "slices" "sort" "strconv" "strings" "github.com/docker/go-units" + "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/hack" "github.com/pingcap/tidb/pkg/util/logutil" - "go.uber.org/zap" "go.uber.org/zap/zapcore" ) -// seekPropsOffsets seeks the statistic files to find the largest offset of -// sorted data file offsets such that the key at offset is less than or equal to -// the given start keys. Caller can specify multiple ascending keys and -// seekPropsOffsets will return the offsets list for each key. +// seekPropsOffsets reads the statistic files to find the largest offset of +// corresponding sorted data file such that the key at offset is less than or +// equal to the given start keys. These returned offsets can be used to seek data +// file reader, read, parse and skip few smaller keys, and then locate the needed +// data. +// +// To avoid potential data loss, it also checks at least one statistic file has a +// key larger than or equal to the start key. If not, we are afraid that some +// paths are missing, and the data between [start key, min(first key of +// statistic files)) are lost. +// +// Caller can specify multiple ascending keys and seekPropsOffsets will return +// the offsets list per file for each key. func seekPropsOffsets( ctx context.Context, starts []kv.Key, paths []string, exStorage storage.ExternalStorage, - checkHotSpot bool, ) (_ [][]uint64, err error) { logger := logutil.Logger(ctx) task := log.BeginTask(logger, "seek props offsets") @@ -50,60 +60,99 @@ func seekPropsOffsets( task.End(zapcore.ErrorLevel, err) }() - // adapt the NewMergePropIter argument types - multiFileStat := MultipleFilesStat{Filenames: make([][2]string, 0, len(paths))} - for _, path := range paths { - multiFileStat.Filenames = append(multiFileStat.Filenames, [2]string{"", path}) + offsetsPerFile := make([][]uint64, len(paths)) + for i := range offsetsPerFile { + offsetsPerFile[i] = make([]uint64, len(starts)) } - iter, err := NewMergePropIter(ctx, []MultipleFilesStat{multiFileStat}, exStorage, checkHotSpot) - if err != nil { + // Record first key if it is smaller than first key of "starts" key argument for + // each file, and check all files afterward. + firstKeyTooSmallCheckers := make([]kv.Key, len(paths)) + eg, egCtx := util.NewErrorGroupWithRecoverWithCtx(ctx) + for i := range paths { + i := i + eg.Go(func() error { + r, err2 := newStatsReader(egCtx, exStorage, paths[i], 250*1024) + if err2 != nil { + if err2 == io.EOF { + return nil + } + return errors.Trace(err2) + } + defer r.Close() + + moved := false + keyIdx := 0 + curKey := starts[keyIdx] + + p, err3 := r.nextProp() + for { + switch err3 { + case nil: + case io.EOF: + // fill the rest of the offsets with the last offset + currOffset := offsetsPerFile[i][keyIdx] + for keyIdx++; keyIdx < len(starts); keyIdx++ { + offsetsPerFile[i][keyIdx] = currOffset + } + return nil + default: + return errors.Trace(err3) + } + propKey := kv.Key(p.firstKey) + for propKey.Cmp(curKey) > 0 { + if !moved { + if firstKeyTooSmallCheckers[i] == nil { + firstKeyTooSmallCheckers[i] = propKey + } + } + keyIdx++ + if keyIdx >= len(starts) { + return nil + } + offsetsPerFile[i][keyIdx] = offsetsPerFile[i][keyIdx-1] + curKey = starts[keyIdx] + } + moved = true + offsetsPerFile[i][keyIdx] = p.offset + p, err3 = r.nextProp() + } + }) + } + + if err = eg.Wait(); err != nil { return nil, err } - defer func() { - if err := iter.Close(); err != nil { - logger.Warn("failed to close merge prop iterator", zap.Error(err)) + + hasNil := false + for _, k := range firstKeyTooSmallCheckers { + if k == nil { + hasNil = true + break } - }() - offsets4AllKey := make([][]uint64, 0, len(starts)) - offsets := make([]uint64, len(paths)) - offsets4AllKey = append(offsets4AllKey, offsets) - moved := false - - keyIdx := 0 - curKey := starts[keyIdx] - for iter.Next() { - p := iter.prop() - propKey := kv.Key(p.firstKey) - for propKey.Cmp(curKey) > 0 { - if !moved { - return nil, fmt.Errorf("start key %s is too small for stat files %v, propKey %s", - curKey.String(), - paths, - propKey.String(), - ) - } - keyIdx++ - if keyIdx >= len(starts) { - return offsets4AllKey, nil + } + if !hasNil { + minKey := firstKeyTooSmallCheckers[0] + for _, k := range firstKeyTooSmallCheckers[1:] { + if k.Cmp(minKey) < 0 { + minKey = k } - curKey = starts[keyIdx] - newOffsets := slices.Clone(offsets) - offsets4AllKey = append(offsets4AllKey, newOffsets) - offsets = newOffsets } - moved = true - _, idx := iter.readerIndex() - offsets[idx] = p.offset - } - if iter.Error() != nil { - return nil, iter.Error() + return nil, fmt.Errorf("start key %s is too small for stat files %v, propKey %s", + starts[0].String(), + paths, + minKey.String(), + ) } - for len(offsets4AllKey) < len(starts) { - newOffsets := slices.Clone(offsets) - offsets4AllKey = append(offsets4AllKey, newOffsets) - offsets = newOffsets + + // TODO(lance6716): change the caller so we don't need to transpose the result + offsetsPerKey := make([][]uint64, len(starts)) + for i := range starts { + offsetsPerKey[i] = make([]uint64, len(paths)) + for j := range paths { + offsetsPerKey[i][j] = offsetsPerFile[j][i] + } } - return offsets4AllKey, nil + return offsetsPerKey, nil } // GetAllFileNames returns data file paths and stat file paths. Both paths are diff --git a/br/pkg/lightning/backend/external/util_test.go b/br/pkg/lightning/backend/external/util_test.go index 8dc271022dc39..1800a27d23c1a 100644 --- a/br/pkg/lightning/backend/external/util_test.go +++ b/br/pkg/lightning/backend/external/util_test.go @@ -71,37 +71,37 @@ func TestSeekPropsOffsets(t *testing.T) { err = w2.Close(ctx) require.NoError(t, err) - got, err := seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5")}, []string{file1, file2}, store, true) + got, err := seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{10, 20}}, got) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5"), []byte("key2.6")}, []string{file1, file2}, store, true) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5"), []byte("key2.6")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{10, 20}, {10, 20}}, got) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3")}, []string{file1, file2}, store, true) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{30, 20}}, got) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5"), []byte("key3")}, []string{file1, file2}, store, true) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key2.5"), []byte("key3")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{10, 20}, {30, 20}}, got) - _, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key0")}, []string{file1, file2}, store, true) + _, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key0")}, []string{file1, file2}, store) require.ErrorContains(t, err, "start key 6b657930 is too small for stat files [/test1 /test2]") - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key1")}, []string{file1, file2}, store, false) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key1")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{10, 0}}, got) - _, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key0"), []byte("key1")}, []string{file1, file2}, store, true) + _, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key0"), []byte("key1")}, []string{file1, file2}, store) require.ErrorContains(t, err, "start key 6b657930 is too small for stat files [/test1 /test2]") - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key999")}, []string{file1, file2}, store, false) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key999")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{50, 40}}, got) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key999"), []byte("key999")}, []string{file1, file2}, store, false) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key999"), []byte("key999")}, []string{file1, file2}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{50, 40}, {50, 40}}, got) @@ -118,11 +118,11 @@ func TestSeekPropsOffsets(t *testing.T) { require.NoError(t, err) err = w4.Close(ctx) require.NoError(t, err) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3")}, []string{file1, file2, file3, file4}, store, true) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3")}, []string{file1, file2, file3, file4}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{30, 20, 0, 30}}, got) - got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3"), []byte("key999")}, []string{file1, file2, file3, file4}, store, true) + got, err = seekPropsOffsets(ctx, []kv.Key{[]byte("key3"), []byte("key999")}, []string{file1, file2, file3, file4}, store) require.NoError(t, err) require.Equal(t, [][]uint64{{30, 20, 0, 30}, {50, 40, 0, 50}}, got) } diff --git a/br/pkg/lightning/backend/kv/session.go b/br/pkg/lightning/backend/kv/session.go index 0f5b4a1ba9f5c..396698881de4b 100644 --- a/br/pkg/lightning/backend/kv/session.go +++ b/br/pkg/lightning/backend/kv/session.go @@ -180,9 +180,9 @@ func (*MemBuf) Staging() kv.StagingHandle { // If the changes are not published by `Release`, they will be discarded. func (*MemBuf) Cleanup(_ kv.StagingHandle) {} -// MayFlush implements the kv.MemBuffer interface. -func (*MemBuf) MayFlush() error { - return nil +// GetLocal implements the kv.MemBuffer interface. +func (mb *MemBuf) GetLocal(ctx context.Context, key []byte) ([]byte, error) { + return mb.Get(ctx, key) } // Size returns sum of keys and values length. @@ -272,6 +272,16 @@ func (*transaction) SetAssertion(_ []byte, _ ...kv.FlagsOp) error { return nil } +// IsPipelined implements the kv.Transaction interface. +func (*transaction) IsPipelined() bool { + return false +} + +// MayFlush implements the kv.Transaction interface. +func (*transaction) MayFlush() error { + return nil +} + type planCtxImpl struct { *Session *planctximpl.PlanCtxExtendedImpl diff --git a/br/pkg/lightning/backend/local/BUILD.bazel b/br/pkg/lightning/backend/local/BUILD.bazel index db1b9c69365e2..9d0e96a140fe1 100644 --- a/br/pkg/lightning/backend/local/BUILD.bazel +++ b/br/pkg/lightning/backend/local/BUILD.bazel @@ -50,8 +50,11 @@ go_library( "//pkg/kv", "//pkg/metrics", "//pkg/parser/model", + "//pkg/parser/mysql", + "//pkg/parser/terror", "//pkg/sessionctx/variable", "//pkg/table", + "//pkg/table/tables", "//pkg/tablecodec", "//pkg/util", "//pkg/util/codec", @@ -141,6 +144,7 @@ go_test( "//pkg/parser/mysql", "//pkg/sessionctx/stmtctx", "//pkg/store/pdtypes", + "//pkg/table", "//pkg/table/tables", "//pkg/tablecodec", "//pkg/testkit/testsetup", @@ -162,7 +166,6 @@ go_test( "@com_github_pingcap_kvproto//pkg/errorpb", "@com_github_pingcap_kvproto//pkg/import_sstpb", "@com_github_pingcap_kvproto//pkg/metapb", - "@com_github_pingcap_kvproto//pkg/pdpb", "@com_github_pingcap_tipb//go-tipb", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//oracle", diff --git a/br/pkg/lightning/backend/local/duplicate.go b/br/pkg/lightning/backend/local/duplicate.go index 6cd78cd254934..1fd8fac55df77 100644 --- a/br/pkg/lightning/backend/local/duplicate.go +++ b/br/pkg/lightning/backend/local/duplicate.go @@ -17,6 +17,7 @@ package local import ( "bytes" "context" + "encoding/hex" "encoding/json" "fmt" "io" @@ -42,7 +43,10 @@ import ( "github.com/pingcap/tidb/pkg/distsql" tidbkv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/table" + "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/hack" @@ -459,7 +463,7 @@ func (m *DupeDetector) HasDuplicate() bool { } // RecordDataConflictError records data conflicts to errorMgr. The key received from stream must be a row key. -func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKVStream) error { +func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKVStream, algorithm config.DuplicateResolutionAlgorithm) error { //nolint: errcheck defer stream.Close() var dataConflictInfos []errormanager.DataConflictInfo @@ -481,6 +485,7 @@ func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKV if err != nil { return errors.Trace(err) } + conflictInfo := errormanager.DataConflictInfo{ RawKey: key, RawValue: val, @@ -494,6 +499,10 @@ func (m *DupeDetector) RecordDataConflictError(ctx context.Context, stream DupKV } dataConflictInfos = dataConflictInfos[:0] } + + if algorithm == config.ErrorOnDup { + return errors.Trace(common.ErrFoundDataConflictRecords.FastGenByArgs(m.tbl.Meta().Name, h.String(), m.decoder.DecodeRawRowDataAsStr(h, val))) + } } if len(dataConflictInfos) > 0 { if err := m.errorMgr.RecordDataConflictError(ctx, m.logger, m.tableName, dataConflictInfos); err != nil { @@ -528,7 +537,7 @@ func (m *DupeDetector) saveIndexHandles(ctx context.Context, handles pendingInde } // RecordIndexConflictError records index conflicts to errorMgr. The key received from stream must be an index key. -func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupKVStream, tableID int64, indexInfo *model.IndexInfo) error { +func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupKVStream, tableID int64, indexInfo *model.IndexInfo, algorithm config.DuplicateResolutionAlgorithm) error { //nolint: errcheck defer stream.Close() indexHandles := makePendingIndexHandlesWithCapacity(0) @@ -550,6 +559,7 @@ func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupK if err != nil { return errors.Trace(err) } + conflictInfo := errormanager.DataConflictInfo{ RawKey: key, RawValue: val, @@ -564,6 +574,10 @@ func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupK } indexHandles.truncate() } + + if algorithm == config.ErrorOnDup { + return newErrFoundIndexConflictRecords(key, val, m.tbl, indexInfo) + } } if indexHandles.Len() > 0 { if err := m.saveIndexHandles(ctx, indexHandles); err != nil { @@ -573,6 +587,99 @@ func (m *DupeDetector) RecordIndexConflictError(ctx context.Context, stream DupK return nil } +// RetrieveKeyAndValueFromErrFoundDuplicateKeys retrieves the key and value +// from ErrFoundDuplicateKeys error. +func RetrieveKeyAndValueFromErrFoundDuplicateKeys(err error) ([]byte, []byte, error) { + if !common.ErrFoundDuplicateKeys.Equal(err) { + return nil, nil, err + } + tErr, ok := errors.Cause(err).(*terror.Error) + if !ok { + return nil, nil, err + } + if len(tErr.Args()) != 2 { + return nil, nil, err + } + key, keyIsByte := tErr.Args()[0].([]byte) + value, valIsByte := tErr.Args()[1].([]byte) + if !keyIsByte || !valIsByte { + return nil, nil, err + } + return key, value, nil +} + +// newErrFoundConflictRecords generate an error ErrFoundDataConflictRecords / ErrFoundIndexConflictRecords +// according to key and value. +func newErrFoundConflictRecords(key []byte, value []byte, tbl table.Table) error { + sessionOpts := encode.SessionOptions{ + SQLMode: mysql.ModeStrictAllTables, + } + + decoder, err := kv.NewTableKVDecoder(tbl, tbl.Meta().Name.L, &sessionOpts, log.L()) + if err != nil { + return errors.Trace(err) + } + + if tablecodec.IsRecordKey(key) { + // for data KV + handle, err := tablecodec.DecodeRowKey(key) + if err != nil { + return errors.Trace(err) + } + + rowData := decoder.DecodeRawRowDataAsStr(handle, value) + + return errors.Trace(common.ErrFoundDataConflictRecords.FastGenByArgs(tbl.Meta().Name, handle.String(), rowData)) + } + + // for index KV + _, idxID, _, err := tablecodec.DecodeIndexKey(key) + if err != nil { + return errors.Trace(err) + } + + idxInfo := model.FindIndexInfoByID(tbl.Meta().Indices, idxID) + return newErrFoundIndexConflictRecords(key, value, tbl, idxInfo) +} + +// newErrFoundIndexConflictRecords generate an error ErrFoundIndexConflictRecords +// according to key and value. +func newErrFoundIndexConflictRecords(key []byte, value []byte, tbl table.Table, idxInfo *model.IndexInfo) error { + sessionOpts := encode.SessionOptions{ + SQLMode: mysql.ModeStrictAllTables, + } + + decoder, err := kv.NewTableKVDecoder(tbl, tbl.Meta().Name.L, &sessionOpts, log.L()) + if err != nil { + return errors.Trace(err) + } + + indexName := fmt.Sprintf("%s.%s", tbl.Meta().Name.String(), idxInfo.Name.String()) + valueStr, err := tables.GenIndexValueFromIndex(key, value, tbl.Meta(), idxInfo) + if err != nil { + log.L().Warn("decode index key value / column value failed", zap.String("index", indexName), + zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err)) + return errors.Trace(common.ErrFoundIndexConflictRecords.FastGenByArgs(tbl.Meta().Name, indexName, key, value)) + } + + h, err := decoder.DecodeHandleFromIndex(idxInfo, key, value) + if err != nil { + return errors.Trace(err) + } + return errors.Trace(common.ErrFoundIndexConflictRecords.FastGenByArgs(tbl.Meta().Name, indexName, valueStr, h)) +} + +// ConvertToErrFoundConflictRecords converts ErrFoundDuplicateKeys +// to ErrFoundDataConflictRecords or ErrFoundIndexConflictRecords error. +func ConvertToErrFoundConflictRecords(originalErr error, tbl table.Table) error { + rawKey, rawValue, err := RetrieveKeyAndValueFromErrFoundDuplicateKeys(originalErr) + if err != nil { + return errors.Trace(err) + } + + return newErrFoundConflictRecords(rawKey, rawValue, tbl) +} + // BuildDuplicateTaskForTest is only used for test. var BuildDuplicateTaskForTest = func(m *DupeDetector) ([]dupTask, error) { return m.buildDupTasks() @@ -702,7 +809,7 @@ func (m *DupeDetector) buildLocalDupTasks(dupDB *pebble.DB, keyAdapter common.Ke } // CollectDuplicateRowsFromDupDB collects duplicates from the duplicate DB and records all duplicate row info into errorMgr. -func (m *DupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB *pebble.DB, keyAdapter common.KeyAdapter) error { +func (m *DupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB *pebble.DB, keyAdapter common.KeyAdapter, algorithm config.DuplicateResolutionAlgorithm) error { tasks, err := m.buildLocalDupTasks(dupDB, keyAdapter) if err != nil { return errors.Trace(err) @@ -719,9 +826,9 @@ func (m *DupeDetector) CollectDuplicateRowsFromDupDB(ctx context.Context, dupDB stream := NewLocalDupKVStream(dupDB, keyAdapter, task.KeyRange) var err error if task.indexInfo == nil { - err = m.RecordDataConflictError(gCtx, stream) + err = m.RecordDataConflictError(gCtx, stream, algorithm) } else { - err = m.RecordIndexConflictError(gCtx, stream, task.tableID, task.indexInfo) + err = m.RecordIndexConflictError(gCtx, stream, task.tableID, task.indexInfo, algorithm) } return errors.Trace(err) }); err != nil { @@ -788,6 +895,7 @@ func (m *DupeDetector) processRemoteDupTaskOnce( importClientFactory ImportClientFactory, regionPool *utils.WorkerPool, remainKeyRanges *pendingKeyRanges, + algorithm config.DuplicateResolutionAlgorithm, ) (madeProgress bool, err error) { //nolint: prealloc var regions []*split.RegionInfo @@ -828,9 +936,9 @@ func (m *DupeDetector) processRemoteDupTaskOnce( return errors.Annotatef(err, "failed to create remote duplicate kv stream") } if task.indexInfo == nil { - err = m.RecordDataConflictError(ctx, stream) + err = m.RecordDataConflictError(ctx, stream, algorithm) } else { - err = m.RecordIndexConflictError(ctx, stream, task.tableID, task.indexInfo) + err = m.RecordIndexConflictError(ctx, stream, task.tableID, task.indexInfo, algorithm) } if err != nil { return errors.Annotatef(err, "failed to record conflict errors") @@ -864,12 +972,13 @@ func (m *DupeDetector) processRemoteDupTask( logger log.Logger, importClientFactory ImportClientFactory, regionPool *utils.WorkerPool, + algorithm config.DuplicateResolutionAlgorithm, ) error { regionErrRetryAttempts := split.WaitRegionOnlineAttemptTimes remainAttempts := maxDupCollectAttemptTimes remainKeyRanges := newPendingKeyRanges(task.KeyRange) for { - madeProgress, err := m.processRemoteDupTaskOnce(ctx, task, logger, importClientFactory, regionPool, remainKeyRanges) + madeProgress, err := m.processRemoteDupTaskOnce(ctx, task, logger, importClientFactory, regionPool, remainKeyRanges, algorithm) if err == nil { if !remainKeyRanges.empty() { remainKeyRanges.list() @@ -904,7 +1013,7 @@ func (m *DupeDetector) processRemoteDupTask( } // CollectDuplicateRowsFromTiKV collects duplicates from the remote TiKV and records all duplicate row info into errorMgr. -func (m *DupeDetector) CollectDuplicateRowsFromTiKV(ctx context.Context, importClientFactory ImportClientFactory) error { +func (m *DupeDetector) CollectDuplicateRowsFromTiKV(ctx context.Context, importClientFactory ImportClientFactory, algorithm config.DuplicateResolutionAlgorithm) error { tasks, err := m.buildDupTasks() if err != nil { return errors.Trace(err) @@ -929,7 +1038,7 @@ func (m *DupeDetector) CollectDuplicateRowsFromTiKV(ctx context.Context, importC zap.Int64("indexID", task.indexInfo.ID), ) } - err := m.processRemoteDupTask(gCtx, task, taskLogger, importClientFactory, regionPool) + err := m.processRemoteDupTask(gCtx, task, taskLogger, importClientFactory, regionPool, algorithm) return errors.Trace(err) }) } @@ -954,7 +1063,7 @@ type DupeController struct { // CollectLocalDuplicateRows collect duplicate keys from local db. We will store the duplicate keys which // may be repeated with other keys in local data source. -func (local *DupeController) CollectLocalDuplicateRows(ctx context.Context, tbl table.Table, tableName string, opts *encode.SessionOptions) (hasDupe bool, err error) { +func (local *DupeController) CollectLocalDuplicateRows(ctx context.Context, tbl table.Table, tableName string, opts *encode.SessionOptions, algorithm config.DuplicateResolutionAlgorithm) (hasDupe bool, err error) { logger := log.FromContext(ctx).With(zap.String("table", tableName)).Begin(zap.InfoLevel, "[detect-dupe] collect local duplicate keys") defer func() { logger.End(zap.ErrorLevel, err) @@ -965,7 +1074,7 @@ func (local *DupeController) CollectLocalDuplicateRows(ctx context.Context, tbl if err != nil { return false, errors.Trace(err) } - if err := duplicateManager.CollectDuplicateRowsFromDupDB(ctx, local.duplicateDB, local.keyAdapter); err != nil { + if err := duplicateManager.CollectDuplicateRowsFromDupDB(ctx, local.duplicateDB, local.keyAdapter, algorithm); err != nil { return false, errors.Trace(err) } return duplicateManager.HasDuplicate(), nil @@ -973,7 +1082,8 @@ func (local *DupeController) CollectLocalDuplicateRows(ctx context.Context, tbl // CollectRemoteDuplicateRows collect duplicate keys from remote TiKV storage. This keys may be duplicate with // the data import by other lightning. -func (local *DupeController) CollectRemoteDuplicateRows(ctx context.Context, tbl table.Table, tableName string, opts *encode.SessionOptions) (hasDupe bool, err error) { +// TODO: revise the returned arguments to (hasDupe bool, dupInfo *DupInfo, err error) to distinguish the conflict error and the common error +func (local *DupeController) CollectRemoteDuplicateRows(ctx context.Context, tbl table.Table, tableName string, opts *encode.SessionOptions, algorithm config.DuplicateResolutionAlgorithm) (hasDupe bool, err error) { logger := log.FromContext(ctx).With(zap.String("table", tableName)).Begin(zap.InfoLevel, "[detect-dupe] collect remote duplicate keys") defer func() { logger.End(zap.ErrorLevel, err) @@ -984,8 +1094,9 @@ func (local *DupeController) CollectRemoteDuplicateRows(ctx context.Context, tbl if err != nil { return false, errors.Trace(err) } - if err := duplicateManager.CollectDuplicateRowsFromTiKV(ctx, local.importClientFactory); err != nil { - return false, errors.Trace(err) + err = duplicateManager.CollectDuplicateRowsFromTiKV(ctx, local.importClientFactory, algorithm) + if err != nil { + return common.ErrFoundDataConflictRecords.Equal(err) || common.ErrFoundIndexConflictRecords.Equal(err), errors.Trace(err) } return duplicateManager.HasDuplicate(), nil } @@ -999,12 +1110,12 @@ func (local *DupeController) ResolveDuplicateRows(ctx context.Context, tbl table }() switch algorithm { - case config.DupeResAlgNone: + case config.NoneOnDup: logger.Warn("skipping resolution due to selected algorithm. this table will become inconsistent!", zap.String("category", "resolve-dupe"), zap.Stringer("algorithm", algorithm)) return nil - case config.DupeResAlgReplace, config.DupeResAlgErr: + case config.ReplaceOnDup: default: - panic(fmt.Sprintf("[resolve-dupe] unknown resolution algorithm %v", algorithm)) + panic(fmt.Sprintf("[resolve-dupe] unknown conflict.strategy algorithm %v", algorithm)) } pool := utils.NewWorkerPool(uint(local.dupeConcurrency), "resolve duplicate rows") @@ -1016,31 +1127,24 @@ func (local *DupeController) ResolveDuplicateRows(ctx context.Context, tbl table logger.Debug("got tblInfo from tbl", zap.ByteString("tblInfo", tblInfo)) - switch algorithm { - case config.DupeResAlgReplace: - err = local.errorMgr.ReplaceConflictKeys( - ctx, tbl, tableName, pool, - func(ctx context.Context, key []byte) ([]byte, error) { - value, err := local.getLatestValue(ctx, logger, key) - if err != nil { - return nil, errors.Trace(err) - } - return value, nil - }, - func(ctx context.Context, key []byte) error { - err := local.deleteDuplicateRow(ctx, logger, key) - if err != nil { - logger.Warn("delete duplicate rows encounter error", log.ShortError(err)) - return common.ErrResolveDuplicateRows.Wrap(errors.Trace(err)).GenWithStackByArgs(tableName) - } - return nil - }, - ) - case config.DupeResAlgErr: - err = local.errorMgr.ResolveConflictKeysError( - ctx, tableName, - ) - } + err = local.errorMgr.ReplaceConflictKeys( + ctx, tbl, tableName, pool, + func(ctx context.Context, key []byte) ([]byte, error) { + value, err := local.getLatestValue(ctx, logger, key) + if err != nil { + return nil, errors.Trace(err) + } + return value, nil + }, + func(ctx context.Context, key []byte) error { + err := local.deleteDuplicateRow(ctx, logger, key) + if err != nil { + logger.Warn("delete duplicate rows encounter error", log.ShortError(err)) + return common.ErrResolveDuplicateRows.Wrap(errors.Trace(err)).GenWithStackByArgs(tableName) + } + return nil + }, + ) return errors.Trace(err) } diff --git a/br/pkg/lightning/backend/local/duplicate_test.go b/br/pkg/lightning/backend/local/duplicate_test.go index 268e19b8047c6..fd244c53e3eb3 100644 --- a/br/pkg/lightning/backend/local/duplicate_test.go +++ b/br/pkg/lightning/backend/local/duplicate_test.go @@ -21,14 +21,18 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" lkv "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/br/pkg/lightning/backend/local" + "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/keyspace" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" + "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/mock" "github.com/stretchr/testify/require" ) @@ -70,3 +74,115 @@ func TestBuildDupTask(t *testing.T) { require.Equal(t, tc.hasTableRange, hasRecordKey) } } + +func buildTableForTestConvertToErrFoundConflictRecords(t *testing.T, node []ast.StmtNode) (table.Table, *lkv.Pairs) { + mockSctx := mock.NewContext() + info, err := ddl.MockTableInfo(mockSctx, node[0].(*ast.CreateTableStmt), 108) + require.NoError(t, err) + info.State = model.StatePublic + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), info) + require.NoError(t, err) + + sessionOpts := encode.SessionOptions{ + SQLMode: mysql.ModeStrictAllTables, + Timestamp: 1234567890, + } + + encoder, err := lkv.NewBaseKVEncoder(&encode.EncodingConfig{ + Table: tbl, + SessionOptions: sessionOpts, + Logger: log.L(), + }) + require.NoError(t, err) + encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true + + data1 := []types.Datum{ + types.NewIntDatum(1), + types.NewIntDatum(6), + types.NewStringDatum("1.csv"), + types.NewIntDatum(101), + } + data2 := []types.Datum{ + types.NewIntDatum(2), + types.NewIntDatum(6), + types.NewStringDatum("2.csv"), + types.NewIntDatum(102), + } + data3 := []types.Datum{ + types.NewIntDatum(3), + types.NewIntDatum(7), + types.NewStringDatum("3.csv"), + types.NewIntDatum(103), + } + tctx := encoder.SessionCtx.GetTableCtx() + _, err = encoder.Table.AddRecord(tctx, data1) + require.NoError(t, err) + _, err = encoder.Table.AddRecord(tctx, data2) + require.NoError(t, err) + _, err = encoder.Table.AddRecord(tctx, data3) + require.NoError(t, err) + return tbl, encoder.SessionCtx.TakeKvPairs() +} + +func TestRetrieveKeyAndValueFromErrFoundDuplicateKeys(t *testing.T) { + p := parser.New() + node, _, err := p.ParseSQL("create table a (a int primary key, b int not null, c text, d int, key key_b(b));") + require.NoError(t, err) + + _, kvPairs := buildTableForTestConvertToErrFoundConflictRecords(t, node) + + data1RowKey := kvPairs.Pairs[0].Key + data1RowValue := kvPairs.Pairs[0].Val + + originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data1RowKey, data1RowValue) + rawKey, rawValue, err := local.RetrieveKeyAndValueFromErrFoundDuplicateKeys(originalErr) + require.NoError(t, err) + require.Equal(t, data1RowKey, rawKey) + require.Equal(t, data1RowValue, rawValue) +} + +func TestConvertToErrFoundConflictRecordsSingleColumnsIndex(t *testing.T) { + p := parser.New() + node, _, err := p.ParseSQL("create table a (a int primary key, b int not null, c text, d int, unique key key_b(b));") + require.NoError(t, err) + + tbl, kvPairs := buildTableForTestConvertToErrFoundConflictRecords(t, node) + + data2RowKey := kvPairs.Pairs[2].Key + data2RowValue := kvPairs.Pairs[2].Val + data3IndexKey := kvPairs.Pairs[5].Key + data3IndexValue := kvPairs.Pairs[5].Val + + originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data2RowKey, data2RowValue) + + newErr := local.ConvertToErrFoundConflictRecords(originalErr, tbl) + require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a, primary key is '2', row data is '(2, 6, \"2.csv\", 102)'") + + originalErr = common.ErrFoundDuplicateKeys.FastGenByArgs(data3IndexKey, data3IndexValue) + + newErr = local.ConvertToErrFoundConflictRecords(originalErr, tbl) + require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_b', unique key is '[7]', primary key is '3'") +} + +func TestConvertToErrFoundConflictRecordsMultipleColumnsIndex(t *testing.T) { + p := parser.New() + node, _, err := p.ParseSQL("create table a (a int primary key, b int not null, c text, d int, unique key key_bd(b,d));") + require.NoError(t, err) + + tbl, kvPairs := buildTableForTestConvertToErrFoundConflictRecords(t, node) + + data2RowKey := kvPairs.Pairs[2].Key + data2RowValue := kvPairs.Pairs[2].Val + data3IndexKey := kvPairs.Pairs[5].Key + data3IndexValue := kvPairs.Pairs[5].Val + + originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data2RowKey, data2RowValue) + + newErr := local.ConvertToErrFoundConflictRecords(originalErr, tbl) + require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a, primary key is '2', row data is '(2, 6, \"2.csv\", 102)'") + + originalErr = common.ErrFoundDuplicateKeys.FastGenByArgs(data3IndexKey, data3IndexValue) + + newErr = local.ConvertToErrFoundConflictRecords(originalErr, tbl) + require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_bd', unique key is '[7 103]', primary key is '3'") +} diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go index 9a53d47d93b51..dbcb70f315504 100644 --- a/br/pkg/lightning/backend/local/local.go +++ b/br/pkg/lightning/backend/local/local.go @@ -271,6 +271,11 @@ func NewTargetInfoGetter(tls *common.TLS, db *sql.DB, pdHTTPCli pdhttp.Client) b } } +// FetchRemoteDBModels implements the `backend.TargetInfoGetter` interface. +func (g *targetInfoGetter) FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) { + return tikv.FetchRemoteDBModelsFromTLS(ctx, g.tls) +} + // FetchRemoteTableModels obtains the models of all tables given the schema name. // It implements the `TargetInfoGetter` interface. func (g *targetInfoGetter) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) { @@ -440,8 +445,8 @@ func NewBackendConfig(cfg *config.Config, maxOpenFiles int, keyspaceName, resour MemTableSize: int(cfg.TikvImporter.EngineMemCacheSize), LocalWriterMemCacheSize: int64(cfg.TikvImporter.LocalWriterMemCacheSize), ShouldCheckTiKV: cfg.App.CheckRequirements, - DupeDetectEnabled: cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone, - DuplicateDetectOpt: common.DupDetectOpt{ReportErrOnDup: cfg.TikvImporter.DuplicateResolution == config.DupeResAlgErr}, + DupeDetectEnabled: cfg.Conflict.Strategy != config.NoneOnDup, + DuplicateDetectOpt: common.DupDetectOpt{ReportErrOnDup: cfg.Conflict.Strategy == config.ErrorOnDup}, StoreWriteBWLimit: int(cfg.TikvImporter.StoreWriteBWLimit), ShouldCheckWriteStall: cfg.Cron.SwitchMode.Duration == 0, MaxOpenFiles: maxOpenFiles, @@ -517,7 +522,6 @@ func NewBackend( // If the time too short, we may scatter a region many times, because // the interface `ScatterRegions` may time out. pd.WithCustomTimeoutOption(60*time.Second), - pd.WithMaxErrorRetry(3), ) if err != nil { return nil, common.NormalizeOrWrapErr(common.ErrCreatePDClient, err) diff --git a/br/pkg/lightning/backend/local/local_test.go b/br/pkg/lightning/backend/local/local_test.go index e45b48a2fc7cf..d084c505b089c 100644 --- a/br/pkg/lightning/backend/local/local_test.go +++ b/br/pkg/lightning/backend/local/local_test.go @@ -679,6 +679,10 @@ func (c *mockPdClient) GetTS(ctx context.Context) (int64, int64, error) { return 1, 2, nil } +func (c *mockPdClient) GetClusterID(ctx context.Context) uint64 { + return 1 +} + type mockGrpcErr struct{} func (e mockGrpcErr) GRPCStatus() *status.Status { diff --git a/br/pkg/lightning/backend/local/localhelper.go b/br/pkg/lightning/backend/local/localhelper.go index 595554fff2bc8..3e749172576ea 100644 --- a/br/pkg/lightning/backend/local/localhelper.go +++ b/br/pkg/lightning/backend/local/localhelper.go @@ -307,14 +307,14 @@ func (local *Backend) SplitAndScatterRegionByRanges( } startTime := time.Now() - scatterCount, err := local.waitForScatterRegions(ctx, scatterRegions) - if scatterCount == len(scatterRegions) { + unScatteredCount, err := local.splitCli.WaitRegionsScattered(ctx, scatterRegions) + if unScatteredCount == 0 { log.FromContext(ctx).Info("waiting for scattering regions done", zap.Int("regions", len(scatterRegions)), zap.Duration("take", time.Since(startTime))) } else { log.FromContext(ctx).Info("waiting for scattering regions timeout", - zap.Int("scatterCount", scatterCount), - zap.Int("regions", len(scatterRegions)), + zap.Int("unScatteredCount", unScatteredCount), + zap.Int("allRegionCount", len(scatterRegions)), zap.Duration("take", time.Since(startTime)), zap.Error(err)) } @@ -323,7 +323,7 @@ func (local *Backend) SplitAndScatterRegionByRanges( // ScatterRegion scatter the regions and retry if it fails. It returns error if can not scatter after max_retry. func (local *Backend) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error { - backoffer := split.NewWaitRegionOnlineBackoffer().(*split.WaitRegionOnlineBackoffer) + backoffer := split.NewWaitRegionOnlineBackoffer() err := utils.WithRetry(ctx, func() error { var failedErr error err := local.splitCli.ScatterRegion(ctx, regionInfo) @@ -356,7 +356,7 @@ func (local *Backend) BatchSplitRegions( var failedErr error splitRegions := newRegions // wait for regions to be split - backoffer := split.NewWaitRegionOnlineBackoffer().(*split.WaitRegionOnlineBackoffer) + backoffer := split.NewWaitRegionOnlineBackoffer() failedErr = utils.WithRetry(ctx, func() error { retryRegions := make([]*split.RegionInfo, 0) for _, region := range splitRegions { @@ -418,62 +418,6 @@ func (local *Backend) hasRegion(ctx context.Context, regionID uint64) (bool, err return regionInfo != nil, nil } -func (local *Backend) waitForScatterRegions(ctx context.Context, regions []*split.RegionInfo) (scatterCount int, _ error) { - var ( - retErr error - backoffer = split.NewWaitRegionOnlineBackoffer().(*split.WaitRegionOnlineBackoffer) - ) - // WithRetry will return multierr which is hard to use, so we use `retErr` - // to save the error needed to return. - _ = utils.WithRetry(ctx, func() error { - var retryRegions []*split.RegionInfo - for _, region := range regions { - scattered, err := local.checkRegionScatteredOrReScatter(ctx, region) - if scattered { - scatterCount++ - continue - } - if err != nil { - if !common.IsRetryableError(err) { - log.FromContext(ctx).Warn("wait for scatter region encountered non-retryable error", logutil.Region(region.Region), zap.Error(err)) - retErr = err - // return nil to stop retry, the error is saved in `retErr` - return nil - } - log.FromContext(ctx).Warn("wait for scatter region encountered error, will retry again", logutil.Region(region.Region), zap.Error(err)) - } - retryRegions = append(retryRegions, region) - } - if len(retryRegions) == 0 { - regions = retryRegions - return nil - } - if len(retryRegions) < len(regions) { - backoffer.Stat.ReduceRetry() - } - - regions = retryRegions - return errors.Annotatef(berrors.ErrPDBatchScanRegion, "wait for scatter region failed") - }, backoffer) - - if len(regions) > 0 && retErr == nil { - retErr = errors.Errorf("wait for scatter region timeout, print the first unfinished region %v", - regions[0].Region.String()) - } - return scatterCount, retErr -} - -func (local *Backend) checkRegionScatteredOrReScatter(ctx context.Context, regionInfo *split.RegionInfo) (bool, error) { - ok, rescatter, err := local.splitCli.IsScatterRegionFinished(ctx, regionInfo.Region.GetId()) - if err != nil { - return false, errors.Trace(err) - } - if !rescatter { - return ok, nil - } - return false, local.ScatterRegion(ctx, regionInfo) -} - func getSplitKeysByRanges(ranges []common.Range, regions []*split.RegionInfo, logger log.Logger) map[uint64][][]byte { checkKeys := make([][]byte, 0) var lastEnd []byte diff --git a/br/pkg/lightning/backend/local/localhelper_test.go b/br/pkg/lightning/backend/local/localhelper_test.go index 6f1e4fd9b530e..79e86d049c3c6 100644 --- a/br/pkg/lightning/backend/local/localhelper_test.go +++ b/br/pkg/lightning/backend/local/localhelper_test.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/restore/split" @@ -272,11 +271,8 @@ func (c *testSplitClient) ScanRegions(ctx context.Context, key, endKey []byte, l return regions, err } -func (c *testSplitClient) IsScatterRegionFinished( - ctx context.Context, - regionID uint64, -) (scatterDone bool, needRescatter bool, scatterErr error) { - return true, false, nil +func (c *testSplitClient) WaitRegionsScattered(context.Context, []*split.RegionInfo) (int, error) { + return 0, nil } func cloneRegion(region *split.RegionInfo) *split.RegionInfo { @@ -934,153 +930,3 @@ func TestStoreWriteLimiter(t *testing.T) { } wg.Wait() } - -type scatterRegionCli struct { - split.SplitClient - - respM map[uint64][]*pdpb.GetOperatorResponse - scatterM map[uint64]int -} - -func newScatterRegionCli() *scatterRegionCli { - return &scatterRegionCli{ - respM: make(map[uint64][]*pdpb.GetOperatorResponse), - scatterM: make(map[uint64]int), - } -} - -func (c *scatterRegionCli) ScatterRegion(_ context.Context, regionInfo *split.RegionInfo) error { - c.scatterM[regionInfo.Region.Id]++ - return nil -} - -func (c *scatterRegionCli) GetOperator(_ context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { - ret := c.respM[regionID][0] - c.respM[regionID] = c.respM[regionID][1:] - return ret, nil -} - -func (c *scatterRegionCli) IsScatterRegionFinished(ctx context.Context, regionID uint64) (scatterDone bool, needRescatter bool, scatterErr error) { - resp, _ := c.GetOperator(ctx, regionID) - return split.IsScatterRegionFinished(resp) -} - -func TestWaitForScatterRegions(t *testing.T) { - ctx := context.Background() - - regionCnt := 6 - regions := make([]*split.RegionInfo, 0, regionCnt) - for i := 1; i <= regionCnt; i++ { - regions = append(regions, &split.RegionInfo{ - Region: &metapb.Region{ - Id: uint64(i), - }, - }) - } - checkRespDrained := func(cli *scatterRegionCli) { - for i := 1; i <= regionCnt; i++ { - require.Len(t, cli.respM[uint64(i)], 0) - } - } - checkNoRetry := func(cli *scatterRegionCli) { - for i := 1; i <= regionCnt; i++ { - require.Equal(t, 0, cli.scatterM[uint64(i)]) - } - } - - cli := newScatterRegionCli() - cli.respM[1] = []*pdpb.GetOperatorResponse{ - {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, - } - cli.respM[2] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("not-scatter-region")}, - } - cli.respM[3] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, - } - cli.respM[4] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, - } - cli.respM[5] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, - {Desc: []byte("not-scatter-region")}, - } - // should trigger a retry - cli.respM[6] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_REPLACE}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, - } - - local := &Backend{splitCli: cli} - cnt, err := local.waitForScatterRegions(ctx, regions) - require.NoError(t, err) - require.Equal(t, 6, cnt) - for i := 1; i <= 4; i++ { - require.Equal(t, 0, cli.scatterM[uint64(i)]) - } - require.Equal(t, 3, cli.scatterM[uint64(5)]) - require.Equal(t, 1, cli.scatterM[uint64(6)]) - checkRespDrained(cli) - - // test non-retryable error - - cli = newScatterRegionCli() - cli.respM[1] = []*pdpb.GetOperatorResponse{ - {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, - } - cli.respM[2] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("not-scatter-region")}, - } - // mimic non-retryable error - cli.respM[3] = []*pdpb.GetOperatorResponse{ - {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_DATA_COMPACTED}}}, - } - local = &Backend{splitCli: cli} - cnt, err = local.waitForScatterRegions(ctx, regions) - require.ErrorContains(t, err, "get operator error: DATA_COMPACTED") - require.Equal(t, 2, cnt) - checkRespDrained(cli) - checkNoRetry(cli) - - // test backoff is timed-out - - backup := split.WaitRegionOnlineAttemptTimes - split.WaitRegionOnlineAttemptTimes = 2 - t.Cleanup(func() { - split.WaitRegionOnlineAttemptTimes = backup - }) - - cli = newScatterRegionCli() - cli.respM[1] = []*pdpb.GetOperatorResponse{ - {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, - } - cli.respM[2] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("not-scatter-region")}, - } - cli.respM[3] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, - } - cli.respM[4] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, // first retry - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, // second retry - } - cli.respM[5] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("not-scatter-region")}, - } - cli.respM[6] = []*pdpb.GetOperatorResponse{ - {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, - } - local = &Backend{splitCli: cli} - cnt, err = local.waitForScatterRegions(ctx, regions) - require.ErrorContains(t, err, "wait for scatter region timeout, print the first unfinished region id:4") - require.Equal(t, 5, cnt) - checkRespDrained(cli) - checkNoRetry(cli) -} diff --git a/br/pkg/lightning/backend/tidb/tidb.go b/br/pkg/lightning/backend/tidb/tidb.go index 4773a564fdc1c..e26ebd42bac19 100644 --- a/br/pkg/lightning/backend/tidb/tidb.go +++ b/br/pkg/lightning/backend/tidb/tidb.go @@ -138,6 +138,38 @@ func NewTargetInfoGetter(db *sql.DB) backend.TargetInfoGetter { } } +// FetchRemoteDBModels implements the `backend.TargetInfoGetter` interface. +func (b *targetInfoGetter) FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) { + results := []*model.DBInfo{} + logger := log.FromContext(ctx) + s := common.SQLWithRetry{ + DB: b.db, + Logger: logger, + } + err := s.Transact(ctx, "fetch db models", func(_ context.Context, tx *sql.Tx) error { + results = results[:0] + + rows, e := tx.Query("SHOW DATABASES") + if e != nil { + return e + } + defer rows.Close() + + for rows.Next() { + var dbName string + if e := rows.Scan(&dbName); e != nil { + return e + } + dbInfo := &model.DBInfo{ + Name: model.NewCIStr(dbName), + } + results = append(results, dbInfo) + } + return rows.Err() + }) + return results, err +} + // FetchRemoteTableModels obtains the models of all tables given the schema name. // It implements the `backend.TargetInfoGetter` interface. // TODO: refactor @@ -270,7 +302,7 @@ type tidbBackend struct { // onDuplicate is the type of INSERT SQL. It may be different with // conflictCfg.Strategy to implement other feature, but the behaviour in caller's // view should be the same. - onDuplicate string + onDuplicate config.DuplicateResolutionAlgorithm errorMgr *errormanager.ErrorManager // maxChunkSize and maxChunkRows are the target size and number of rows of each INSERT SQL // statement to be sent to downstream. Sometimes we want to reduce the txn size to avoid @@ -292,7 +324,7 @@ func NewTiDBBackend( errorMgr *errormanager.ErrorManager, ) backend.Backend { conflict := cfg.Conflict - var onDuplicate string + var onDuplicate config.DuplicateResolutionAlgorithm switch conflict.Strategy { case config.ErrorOnDup: onDuplicate = config.ErrorOnDup diff --git a/br/pkg/lightning/common/errors.go b/br/pkg/lightning/common/errors.go index 4cf79ccac0457..2ba7bc95e01f1 100644 --- a/br/pkg/lightning/common/errors.go +++ b/br/pkg/lightning/common/errors.go @@ -99,9 +99,12 @@ var ( ErrInvalidMetaStatus = errors.Normalize("invalid meta status: '%s'", errors.RFCCodeText("Lightning:Restore:ErrInvalidMetaStatus")) ErrTableIsChecksuming = errors.Normalize("table '%s' is checksuming", errors.RFCCodeText("Lightning:Restore:ErrTableIsChecksuming")) ErrResolveDuplicateRows = errors.Normalize("resolve duplicate rows error on table '%s'", errors.RFCCodeText("Lightning:Restore:ErrResolveDuplicateRows")) - ErrFoundDuplicateKeys = errors.Normalize("found duplicate key '%s', value '%s'", errors.RFCCodeText("Lightning:Restore:ErrFoundDuplicateKey")) - ErrAddIndexFailed = errors.Normalize("add index on table %s failed", errors.RFCCodeText("Lightning:Restore:ErrAddIndexFailed")) - ErrDropIndexFailed = errors.Normalize("drop index %s on table %s failed", errors.RFCCodeText("Lightning:Restore:ErrDropIndexFailed")) + // ErrFoundDuplicateKeys shoud be replaced with ErrFoundDataConflictRecords and ErrFoundIndexConflictRecords (TODO) + ErrFoundDuplicateKeys = errors.Normalize("found duplicate key '%s', value '%s'", errors.RFCCodeText("Lightning:Restore:ErrFoundDuplicateKey")) + ErrAddIndexFailed = errors.Normalize("add index on table %s failed", errors.RFCCodeText("Lightning:Restore:ErrAddIndexFailed")) + ErrDropIndexFailed = errors.Normalize("drop index %s on table %s failed", errors.RFCCodeText("Lightning:Restore:ErrDropIndexFailed")) + ErrFoundDataConflictRecords = errors.Normalize("found data conflict records in table %s, primary key is '%s', row data is '%s'", errors.RFCCodeText("Lightning:Restore:ErrFoundDataConflictRecords")) + ErrFoundIndexConflictRecords = errors.Normalize("found index conflict records in table %s, index name is '%s', unique key is '%s', primary key is '%s'", errors.RFCCodeText("Lightning:Restore:ErrFoundIndexConflictRecords")) ) type withStack struct { diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index 8aa6b3ecd3dfd..4c59962282d1c 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -63,13 +63,6 @@ const ( // CheckpointDriverFile is a constant for choosing the "File" checkpoint driver in the configuration. CheckpointDriverFile = "file" - // ReplaceOnDup indicates using REPLACE INTO to insert data for TiDB backend. - ReplaceOnDup = "replace" - // IgnoreOnDup indicates using INSERT IGNORE INTO to insert data for TiDB backend. - IgnoreOnDup = "ignore" - // ErrorOnDup indicates using INSERT INTO to insert data, which would violate PK or UNIQUE constraint for TiDB backend. - ErrorOnDup = "error" - // KVWriteBatchSize batch size when write to TiKV. // this is the default value of linux send buffer size(net.ipv4.tcp_wmem) too. KVWriteBatchSize = 16 * units.KiB @@ -597,17 +590,20 @@ const ( type DuplicateResolutionAlgorithm int const ( - // DupeResAlgNone doesn't detect duplicate. - DupeResAlgNone DuplicateResolutionAlgorithm = iota - - // DupeResAlgReplace records all duplicate records like the 'record' algorithm, and remove some rows with conflict - // and reserve other rows that can be kept and not cause conflict anymore. Users need to analyze the - // lightning_task_info.conflict_error_v2 table to check whether the reserved data cater to their need and check whether - // they need to add back the correct rows. - DupeResAlgReplace - - // DupeResAlgErr reports an error after detecting the first conflict and stops the import process. - DupeResAlgErr + // NoneOnDup does nothing when detecting duplicate. + NoneOnDup DuplicateResolutionAlgorithm = iota + // ReplaceOnDup indicates using REPLACE INTO to insert data for TiDB backend. + // ReplaceOnDup records all duplicate records, remove some rows with conflict + // and reserve other rows that can be kept and not cause conflict anymore for local backend. + // Users need to analyze the lightning_task_info.conflict_error_v2 table to check whether the reserved data + // cater to their need and check whether they need to add back the correct rows. + ReplaceOnDup + // IgnoreOnDup indicates using INSERT IGNORE INTO to insert data for TiDB backend. + // Local backend does not support IgnoreOnDup. + IgnoreOnDup + // ErrorOnDup indicates using INSERT INTO to insert data for TiDB backend, which would violate PK or UNIQUE constraint when detecting duplicate. + // ErrorOnDup reports an error after detecting the first conflict and stops the import process for local backend. + ErrorOnDup ) // UnmarshalTOML implements the toml.Unmarshaler interface. @@ -615,7 +611,7 @@ func (dra *DuplicateResolutionAlgorithm) UnmarshalTOML(v any) error { if val, ok := v.(string); ok { return dra.FromStringValue(val) } - return errors.Errorf("invalid duplicate-resolution '%v', please choose valid option between ['none', 'replace', 'error']", v) + return errors.Errorf("invalid conflict.strategy '%v', please choose valid option between ['', 'replace', 'ignore', 'error']", v) } // MarshalText implements the encoding.TextMarshaler interface. @@ -626,14 +622,19 @@ func (dra DuplicateResolutionAlgorithm) MarshalText() ([]byte, error) { // FromStringValue parses the string value to the DuplicateResolutionAlgorithm. func (dra *DuplicateResolutionAlgorithm) FromStringValue(s string) error { switch strings.ToLower(s) { - case "none": - *dra = DupeResAlgNone + case "", "none": + *dra = NoneOnDup case "replace": - *dra = DupeResAlgReplace + *dra = ReplaceOnDup + case "ignore": + *dra = IgnoreOnDup case "error": - *dra = DupeResAlgErr + *dra = ErrorOnDup + case "remove", "record": + log.L().Warn("\"conflict.strategy '%s' is no longer supported, has been converted to 'replace'") + *dra = ReplaceOnDup default: - return errors.Errorf("invalid duplicate-resolution '%s', please choose valid option between ['none', 'replace', 'error']", s) + return errors.Errorf("invalid conflict.strategy '%s', please choose valid option between ['', 'replace', 'ignore', 'error']", s) } return nil } @@ -651,14 +652,16 @@ func (dra *DuplicateResolutionAlgorithm) UnmarshalJSON(data []byte) error { // String implements the fmt.Stringer interface. func (dra DuplicateResolutionAlgorithm) String() string { switch dra { - case DupeResAlgNone: - return "none" - case DupeResAlgReplace: + case NoneOnDup: + return "" + case ReplaceOnDup: return "replace" - case DupeResAlgErr: + case IgnoreOnDup: + return "ignore" + case ErrorOnDup: return "error" default: - panic(fmt.Sprintf("invalid duplicate-resolution type '%d'", dra)) + panic(fmt.Sprintf("invalid conflict.strategy type '%d'", dra)) } } @@ -1045,21 +1048,22 @@ type TikvImporter struct { Addr string `toml:"addr" json:"addr"` Backend string `toml:"backend" json:"backend"` // deprecated, use Conflict.Strategy instead. - OnDuplicate string `toml:"on-duplicate" json:"on-duplicate"` - MaxKVPairs int `toml:"max-kv-pairs" json:"max-kv-pairs"` + OnDuplicate DuplicateResolutionAlgorithm `toml:"on-duplicate" json:"on-duplicate"` + MaxKVPairs int `toml:"max-kv-pairs" json:"max-kv-pairs"` // deprecated - SendKVPairs int `toml:"send-kv-pairs" json:"send-kv-pairs"` - SendKVSize ByteSize `toml:"send-kv-size" json:"send-kv-size"` - CompressKVPairs CompressionType `toml:"compress-kv-pairs" json:"compress-kv-pairs"` - RegionSplitSize ByteSize `toml:"region-split-size" json:"region-split-size"` - RegionSplitKeys int `toml:"region-split-keys" json:"region-split-keys"` - RegionSplitBatchSize int `toml:"region-split-batch-size" json:"region-split-batch-size"` - RegionSplitConcurrency int `toml:"region-split-concurrency" json:"region-split-concurrency"` - RegionCheckBackoffLimit int `toml:"region-check-backoff-limit" json:"region-check-backoff-limit"` - SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"` - DiskQuota ByteSize `toml:"disk-quota" json:"disk-quota"` - RangeConcurrency int `toml:"range-concurrency" json:"range-concurrency"` - DuplicateResolution DuplicateResolutionAlgorithm `toml:"duplicate-resolution" json:"duplicate-resolution"` + SendKVPairs int `toml:"send-kv-pairs" json:"send-kv-pairs"` + SendKVSize ByteSize `toml:"send-kv-size" json:"send-kv-size"` + CompressKVPairs CompressionType `toml:"compress-kv-pairs" json:"compress-kv-pairs"` + RegionSplitSize ByteSize `toml:"region-split-size" json:"region-split-size"` + RegionSplitKeys int `toml:"region-split-keys" json:"region-split-keys"` + RegionSplitBatchSize int `toml:"region-split-batch-size" json:"region-split-batch-size"` + RegionSplitConcurrency int `toml:"region-split-concurrency" json:"region-split-concurrency"` + RegionCheckBackoffLimit int `toml:"region-check-backoff-limit" json:"region-check-backoff-limit"` + SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"` + DiskQuota ByteSize `toml:"disk-quota" json:"disk-quota"` + RangeConcurrency int `toml:"range-concurrency" json:"range-concurrency"` + // deprecated, use Conflict.Strategy instead. + DuplicateResolution DuplicateResolutionAlgorithm `toml:"duplicate-resolution" json:"duplicate-resolution"` // deprecated, use ParallelImport instead. IncrementalImport bool `toml:"incremental-import" json:"incremental-import"` ParallelImport bool `toml:"parallel-import" json:"parallel-import"` @@ -1098,7 +1102,6 @@ func (t *TikvImporter) adjust() error { "`tikv-importer.logical-import-batch-rows` got %d, should be larger than 0", t.LogicalImportBatchRows) } - t.DuplicateResolution = DupeResAlgNone case BackendLocal: if t.RegionSplitBatchSize <= 0 { return common.ErrInvalidConfig.GenWithStack( @@ -1328,43 +1331,57 @@ func ParseCharset(dataCharacterSet string) (Charset, error) { // Conflict is the config section for PK/UK conflict related configurations. type Conflict struct { - Strategy string `toml:"strategy" json:"strategy"` - Threshold int64 `toml:"threshold" json:"threshold"` - MaxRecordRows int64 `toml:"max-record-rows" json:"max-record-rows"` + Strategy DuplicateResolutionAlgorithm `toml:"strategy" json:"strategy"` + PrecheckConflictBeforeImport bool `toml:"precheck-conflict-before-import" json:"precheck-conflict-before-import"` + Threshold int64 `toml:"threshold" json:"threshold"` + MaxRecordRows int64 `toml:"max-record-rows" json:"max-record-rows"` } // adjust assigns default values and check illegal values. The arguments must be // adjusted before calling this function. func (c *Conflict) adjust(i *TikvImporter, l *Lightning) error { strategyConfigFrom := "conflict.strategy" - if c.Strategy == "" { - if i.OnDuplicate == "" && i.Backend == BackendTiDB { + if c.Strategy == NoneOnDup { + if i.OnDuplicate == NoneOnDup && i.Backend == BackendTiDB { c.Strategy = ErrorOnDup } - if i.OnDuplicate != "" { + if i.OnDuplicate != NoneOnDup { strategyConfigFrom = "tikv-importer.on-duplicate" c.Strategy = i.OnDuplicate } } - c.Strategy = strings.ToLower(c.Strategy) + strategyFromDuplicateResolution := false + if c.Strategy == NoneOnDup && i.DuplicateResolution != NoneOnDup { + c.Strategy = i.DuplicateResolution + strategyFromDuplicateResolution = true + } switch c.Strategy { - case ReplaceOnDup, IgnoreOnDup, ErrorOnDup, "": + case ReplaceOnDup, IgnoreOnDup, ErrorOnDup, NoneOnDup: default: return common.ErrInvalidConfig.GenWithStack( "unsupported `%s` (%s)", strategyConfigFrom, c.Strategy) } - if c.Strategy != "" { - if i.ParallelImport && i.Backend == BackendLocal { + if c.Strategy != NoneOnDup { + if i.ParallelImport && i.Backend == BackendLocal && c.PrecheckConflictBeforeImport { return common.ErrInvalidConfig.GenWithStack( - `%s cannot be used with tikv-importer.parallel-import and tikv-importer.backend = "local"`, + `%s cannot be used with tikv-importer.parallel-import and tikv-importer.backend = "local" and conflict.precheck-conflict-before-import = true`, strategyConfigFrom) } - if i.DuplicateResolution != DupeResAlgNone { + if !strategyFromDuplicateResolution && i.DuplicateResolution != NoneOnDup { return common.ErrInvalidConfig.GenWithStack( "%s cannot be used with tikv-importer.duplicate-resolution", strategyConfigFrom) } } + if c.Strategy == IgnoreOnDup && i.Backend == BackendLocal { + return common.ErrInvalidConfig.GenWithStack( + `%s cannot be set to "ignore" when use tikv-importer.backend = "local"`, + strategyConfigFrom) + } + if c.PrecheckConflictBeforeImport && i.Backend == BackendTiDB { + return common.ErrInvalidConfig.GenWithStack( + `conflict.precheck-conflict-before-import cannot be set to true when use tikv-importer.backend = "tidb"`) + } if c.Threshold < 0 { switch c.Strategy { @@ -1372,9 +1389,9 @@ func (c *Conflict) adjust(i *TikvImporter, l *Lightning) error { c.Threshold = 0 case IgnoreOnDup, ReplaceOnDup: c.Threshold = math.MaxInt64 - case "": + case NoneOnDup: c.Threshold = 0 - if i.DuplicateResolution != DupeResAlgNone { + if i.Backend == BackendLocal && c.Strategy != NoneOnDup { c.Threshold = math.MaxInt64 } } @@ -1473,7 +1490,7 @@ func NewConfig() *Config { RegionSplitConcurrency: runtime.GOMAXPROCS(0), RegionCheckBackoffLimit: DefaultRegionCheckBackoffLimit, DiskQuota: ByteSize(math.MaxInt64), - DuplicateResolution: DupeResAlgNone, + DuplicateResolution: NoneOnDup, PausePDSchedulerScope: PausePDSchedulerScopeTable, BlockSize: 16 * 1024, LogicalImportBatchSize: ByteSize(defaultLogicalImportBatchSize), @@ -1486,9 +1503,10 @@ func NewConfig() *Config { ChecksumViaSQL: false, }, Conflict: Conflict{ - Strategy: "", - Threshold: -1, - MaxRecordRows: -1, + Strategy: NoneOnDup, + PrecheckConflictBeforeImport: false, + Threshold: -1, + MaxRecordRows: -1, }, } } diff --git a/br/pkg/lightning/config/config_test.go b/br/pkg/lightning/config/config_test.go index 07f77abf37178..4c152fe1370eb 100644 --- a/br/pkg/lightning/config/config_test.go +++ b/br/pkg/lightning/config/config_test.go @@ -737,16 +737,25 @@ func TestDurationMarshalJSON(t *testing.T) { func TestDuplicateResolutionAlgorithm(t *testing.T) { var dra DuplicateResolutionAlgorithm + require.NoError(t, dra.FromStringValue("")) + require.Equal(t, NoneOnDup, dra) require.NoError(t, dra.FromStringValue("none")) - require.Equal(t, DupeResAlgNone, dra) + require.Equal(t, NoneOnDup, dra) require.NoError(t, dra.FromStringValue("replace")) - require.Equal(t, DupeResAlgReplace, dra) + require.Equal(t, ReplaceOnDup, dra) + require.NoError(t, dra.FromStringValue("ignore")) + require.Equal(t, IgnoreOnDup, dra) require.NoError(t, dra.FromStringValue("error")) - require.Equal(t, DupeResAlgErr, dra) - - require.Equal(t, "none", DupeResAlgNone.String()) - require.Equal(t, "replace", DupeResAlgReplace.String()) - require.Equal(t, "error", DupeResAlgErr.String()) + require.Equal(t, ErrorOnDup, dra) + require.NoError(t, dra.FromStringValue("remove")) + require.Equal(t, ReplaceOnDup, dra) + require.NoError(t, dra.FromStringValue("record")) + require.Equal(t, ReplaceOnDup, dra) + + require.Equal(t, "", NoneOnDup.String()) + require.Equal(t, "replace", ReplaceOnDup.String()) + require.Equal(t, "ignore", IgnoreOnDup.String()) + require.Equal(t, "error", ErrorOnDup.String()) } func TestLoadConfig(t *testing.T) { @@ -966,12 +975,12 @@ func TestAdjustConflictStrategy(t *testing.T) { ctx := context.Background() cfg.TikvImporter.Backend = BackendTiDB - cfg.Conflict.Strategy = "" + cfg.Conflict.Strategy = NoneOnDup require.NoError(t, cfg.Adjust(ctx)) require.Equal(t, ErrorOnDup, cfg.Conflict.Strategy) cfg.TikvImporter.Backend = BackendLocal - cfg.Conflict.Strategy = "" + cfg.Conflict.Strategy = NoneOnDup require.NoError(t, cfg.Adjust(ctx)) require.Empty(t, cfg.Conflict.Strategy) @@ -983,7 +992,14 @@ func TestAdjustConflictStrategy(t *testing.T) { cfg.TikvImporter.Backend = BackendLocal cfg.Conflict.Strategy = ReplaceOnDup cfg.TikvImporter.ParallelImport = true - require.ErrorContains(t, cfg.Adjust(ctx), `conflict.strategy cannot be used with tikv-importer.parallel-import and tikv-importer.backend = "local"`) + cfg.Conflict.PrecheckConflictBeforeImport = true + require.ErrorContains(t, cfg.Adjust(ctx), `conflict.strategy cannot be used with tikv-importer.parallel-import and tikv-importer.backend = "local" and conflict.precheck-conflict-before-import = true`) + + cfg.TikvImporter.Backend = BackendLocal + cfg.Conflict.Strategy = ReplaceOnDup + cfg.TikvImporter.ParallelImport = true + cfg.Conflict.PrecheckConflictBeforeImport = false + require.NoError(t, cfg.Adjust(ctx)) cfg.TikvImporter.Backend = BackendTiDB cfg.Conflict.Strategy = IgnoreOnDup @@ -992,15 +1008,33 @@ func TestAdjustConflictStrategy(t *testing.T) { cfg.TikvImporter.Backend = BackendLocal cfg.Conflict.Strategy = ReplaceOnDup cfg.TikvImporter.ParallelImport = false - cfg.TikvImporter.DuplicateResolution = DupeResAlgReplace - require.ErrorContains(t, cfg.Adjust(ctx), "conflict.strategy cannot be used with tikv-importer.duplicate-resolution") + cfg.TikvImporter.DuplicateResolution = ReplaceOnDup + require.ErrorContains(t, cfg.Adjust(ctx), `conflict.strategy cannot be used with tikv-importer.duplicate-resolution`) cfg.TikvImporter.Backend = BackendLocal - cfg.Conflict.Strategy = "" + cfg.Conflict.Strategy = NoneOnDup cfg.TikvImporter.OnDuplicate = ReplaceOnDup cfg.TikvImporter.ParallelImport = false - cfg.TikvImporter.DuplicateResolution = DupeResAlgReplace - require.ErrorContains(t, cfg.Adjust(ctx), "tikv-importer.on-duplicate cannot be used with tikv-importer.duplicate-resolution") + cfg.TikvImporter.DuplicateResolution = ReplaceOnDup + require.ErrorContains(t, cfg.Adjust(ctx), `tikv-importer.on-duplicate cannot be used with tikv-importer.duplicate-resolution`) + + cfg.TikvImporter.Backend = BackendLocal + cfg.Conflict.Strategy = IgnoreOnDup + cfg.TikvImporter.DuplicateResolution = NoneOnDup + require.ErrorContains(t, cfg.Adjust(ctx), `conflict.strategy cannot be set to "ignore" when use tikv-importer.backend = "local"`) + + cfg.TikvImporter.Backend = BackendTiDB + cfg.Conflict.Strategy = IgnoreOnDup + cfg.Conflict.PrecheckConflictBeforeImport = true + require.ErrorContains(t, cfg.Adjust(ctx), `conflict.precheck-conflict-before-import cannot be set to true when use tikv-importer.backend = "tidb"`) + + cfg.TikvImporter.Backend = BackendLocal + cfg.Conflict.Strategy = NoneOnDup + cfg.TikvImporter.ParallelImport = false + cfg.TikvImporter.DuplicateResolution = ReplaceOnDup + cfg.TikvImporter.OnDuplicate = NoneOnDup + require.NoError(t, cfg.Adjust(ctx)) + require.Equal(t, ReplaceOnDup, cfg.Conflict.Strategy) } func TestAdjustMaxRecordRows(t *testing.T) { @@ -1252,13 +1286,17 @@ func TestCompressionType(t *testing.T) { func TestAdjustConflict(t *testing.T) { cfg := NewConfig() assignMinimalLegalValue(cfg) - cfg.Conflict.Strategy = "123" - require.ErrorContains(t, cfg.Conflict.adjust(&cfg.TikvImporter, &cfg.App), "unsupported `conflict.strategy` (123)") + var dra DuplicateResolutionAlgorithm - cfg.Conflict.Strategy = "IGNORE" + require.NoError(t, dra.FromStringValue("REPLACE")) + cfg.Conflict.Strategy = dra require.NoError(t, cfg.Conflict.adjust(&cfg.TikvImporter, &cfg.App)) require.Equal(t, int64(math.MaxInt64), cfg.Conflict.Threshold) + require.NoError(t, dra.FromStringValue("IGNORE")) + cfg.Conflict.Strategy = dra + require.ErrorContains(t, cfg.Conflict.adjust(&cfg.TikvImporter, &cfg.App), `conflict.strategy cannot be set to "ignore" when use tikv-importer.backend = "local"`) + cfg.Conflict.Strategy = ErrorOnDup cfg.Conflict.Threshold = 1 require.ErrorContains(t, cfg.Conflict.adjust(&cfg.TikvImporter, &cfg.App), `conflict.threshold cannot be set when use conflict.strategy = "error"`) diff --git a/br/pkg/lightning/errormanager/BUILD.bazel b/br/pkg/lightning/errormanager/BUILD.bazel index 0c37f9aea9882..0bea375253695 100644 --- a/br/pkg/lightning/errormanager/BUILD.bazel +++ b/br/pkg/lightning/errormanager/BUILD.bazel @@ -40,11 +40,10 @@ go_test( ], embed = [":errormanager"], flaky = True, - shard_count = 10, + shard_count = 9, deps = [ "//br/pkg/lightning/backend/encode", "//br/pkg/lightning/backend/kv", - "//br/pkg/lightning/common", "//br/pkg/lightning/config", "//br/pkg/lightning/log", "//br/pkg/utils", diff --git a/br/pkg/lightning/errormanager/errormanager.go b/br/pkg/lightning/errormanager/errormanager.go index 86a08c0f8cb47..55ec02a9daa59 100644 --- a/br/pkg/lightning/errormanager/errormanager.go +++ b/br/pkg/lightning/errormanager/errormanager.go @@ -155,19 +155,6 @@ const ( WHERE key_data = "" and row_data = ""; ` - selectConflictKeysCountError = ` - SELECT COUNT(*) - FROM %s.` + ConflictErrorTableName + ` - WHERE table_name = ?; - ` - - selectConflictKeysError = ` - SELECT raw_key, raw_row - FROM %s.` + ConflictErrorTableName + ` - WHERE table_name = ? - LIMIT 1; - ` - insertIntoDupRecord = ` INSERT INTO %s.` + DupRecordTable + ` (task_id, table_name, path, offset, error, row_id, row_data) @@ -221,7 +208,7 @@ func New(db *sql.DB, cfg *config.Config, logger log.Logger) *ErrorManager { taskID: cfg.TaskID, configError: &cfg.App.MaxError, remainingError: cfg.App.MaxError, - conflictV1Enabled: cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone, + conflictV1Enabled: cfg.TikvImporter.Backend == config.BackendLocal && cfg.Conflict.Strategy != config.NoneOnDup, configConflict: &cfg.Conflict, conflictErrRemain: conflictErrRemain, conflictRecordsRemain: conflictRecordsRemain, @@ -230,7 +217,7 @@ func New(db *sql.DB, cfg *config.Config, logger log.Logger) *ErrorManager { } switch cfg.TikvImporter.Backend { case config.BackendLocal: - if cfg.Conflict.Strategy != "" { + if cfg.Conflict.PrecheckConflictBeforeImport && cfg.Conflict.Strategy != config.NoneOnDup { em.conflictV2Enabled = true } case config.BackendTiDB: @@ -769,64 +756,6 @@ func (em *ErrorManager) ReplaceConflictKeys( return errors.Trace(g.Wait()) } -// ResolveConflictKeysError query all conflicting rows (handle and their -// values) from the current error report and return error -// if the number of the conflicting rows is larger than 0. -func (em *ErrorManager) ResolveConflictKeysError( - ctx context.Context, - tableName string, -) error { - if em.db == nil { - return nil - } - - _, gCtx := errgroup.WithContext(ctx) - - kvRows, err := em.db.QueryContext( - gCtx, common.SprintfWithIdentifiers(selectConflictKeysCountError, em.schema), - tableName) - if err != nil { - return errors.Trace(err) - } - defer kvRows.Close() - var kvRowsCount int64 - for kvRows.Next() { - if err := kvRows.Scan(&kvRowsCount); err != nil { - return errors.Trace(err) - } - } - if err := kvRows.Err(); err != nil { - return errors.Trace(err) - } - - em.logger.Debug("got kv rows count from table", - zap.Int64("kv rows count", kvRowsCount)) - if kvRowsCount > 0 { - rows, err := em.db.QueryContext( - gCtx, common.SprintfWithIdentifiers(selectConflictKeysError, em.schema), - tableName) - if err != nil { - return errors.Trace(err) - } - defer rows.Close() - - var rawKey, rawRow []byte - for rows.Next() { - if err := rows.Scan(&rawKey, &rawRow); err != nil { - return errors.Trace(err) - } - em.logger.Debug("got raw_key, raw_row from table", - logutil.Key("raw_key", rawKey), - zap.Binary("raw_row", rawRow)) - } - if err := rows.Err(); err != nil { - return errors.Trace(err) - } - return common.ErrFoundDuplicateKeys.FastGenByArgs(rawKey, rawRow) - } - return nil -} - // RecordDuplicateCount reduce the counter of "duplicate entry" errors. // Currently, the count will not be shared for multiple lightning instances. func (em *ErrorManager) RecordDuplicateCount(cnt int64) error { @@ -978,7 +907,8 @@ func (em *ErrorManager) LogErrorDetails() { if errCnt := em.conflictError(); errCnt > 0 { if em.conflictV1Enabled { em.logger.Warn(fmtErrMsg(errCnt, "conflict", ConflictErrorTableName)) - } else { + } + if em.conflictV2Enabled { em.logger.Warn(fmtErrMsg(errCnt, "conflict", DupRecordTable)) } } @@ -1024,7 +954,8 @@ func (em *ErrorManager) Output() string { count++ if em.conflictV1Enabled { t.AppendRow(table.Row{count, "Unique Key Conflict", errCnt, em.fmtTableName(ConflictErrorTableName)}) - } else { + } + if em.conflictV2Enabled { t.AppendRow(table.Row{count, "Unique Key Conflict", errCnt, em.fmtTableName(DupRecordTable)}) } } diff --git a/br/pkg/lightning/errormanager/errormanager_test.go b/br/pkg/lightning/errormanager/errormanager_test.go index a0eec6e80b934..247054fd304aa 100644 --- a/br/pkg/lightning/errormanager/errormanager_test.go +++ b/br/pkg/lightning/errormanager/errormanager_test.go @@ -42,14 +42,15 @@ func TestInit(t *testing.T) { cfg := config.NewConfig() cfg.TikvImporter.Backend = config.BackendLocal - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgNone + cfg.TikvImporter.DuplicateResolution = config.NoneOnDup cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.Conflict.PrecheckConflictBeforeImport = true cfg.App.MaxError.Type.Store(10) cfg.Conflict.Threshold = 20 cfg.App.TaskInfoSchemaName = "lightning_errors" em := New(db, cfg, log.L()) - require.False(t, em.conflictV1Enabled) + require.True(t, em.conflictV1Enabled) require.True(t, em.conflictV2Enabled) require.Equal(t, cfg.App.MaxError.Type.Load(), em.remainingError.Type.Load()) require.Equal(t, cfg.Conflict.Threshold, em.conflictErrRemain.Load()) @@ -297,7 +298,8 @@ func TestReplaceConflictOneKey(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := New(db, cfg, log.L()) err = em.Init(ctx) @@ -501,7 +503,8 @@ func TestReplaceConflictOneUniqueKey(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := New(db, cfg, log.L()) err = em.Init(ctx) diff --git a/br/pkg/lightning/errormanager/resolveconflict_test.go b/br/pkg/lightning/errormanager/resolveconflict_test.go index f62cc19f389d3..1f934a882c5fa 100644 --- a/br/pkg/lightning/errormanager/resolveconflict_test.go +++ b/br/pkg/lightning/errormanager/resolveconflict_test.go @@ -24,7 +24,6 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" tidbkv "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/config" "github.com/pingcap/tidb/br/pkg/lightning/errormanager" "github.com/pingcap/tidb/br/pkg/lightning/log" @@ -197,7 +196,8 @@ func TestReplaceConflictMultipleKeysNonclusteredPk(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := errormanager.New(db, cfg, log.L()) err = em.Init(ctx) @@ -364,7 +364,8 @@ func TestReplaceConflictOneKeyNonclusteredPk(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := errormanager.New(db, cfg, log.L()) err = em.Init(ctx) @@ -547,7 +548,8 @@ func TestReplaceConflictOneUniqueKeyNonclusteredPk(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := errormanager.New(db, cfg, log.L()) err = em.Init(ctx) @@ -749,7 +751,8 @@ func TestReplaceConflictOneUniqueKeyNonclusteredVarcharPk(t *testing.T) { mockDB.ExpectCommit() cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgReplace + cfg.Conflict.Strategy = config.ReplaceOnDup + cfg.TikvImporter.Backend = config.BackendLocal cfg.App.TaskInfoSchemaName = "lightning_task_info" em := errormanager.New(db, cfg, log.L()) err = em.Init(ctx) @@ -813,92 +816,3 @@ func TestReplaceConflictOneUniqueKeyNonclusteredVarcharPk(t *testing.T) { err = mockDB.ExpectationsWereMet() require.NoError(t, err) } - -func TestResolveConflictKeysError(t *testing.T) { - p := parser.New() - node, _, err := p.ParseSQL("create table a (a varchar(20) primary key clustered, b int not null, c text, key uni_b(b));") - require.NoError(t, err) - mockSctx := mock.NewContext() - info, err := ddl.MockTableInfo(mockSctx, node[0].(*ast.CreateTableStmt), 108) - require.NoError(t, err) - info.State = model.StatePublic - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), info) - require.NoError(t, err) - - sessionOpts := encode.SessionOptions{ - SQLMode: mysql.ModeStrictAllTables, - Timestamp: 1234567890, - } - - encoder, err := tidbkv.NewBaseKVEncoder(&encode.EncodingConfig{ - Table: tbl, - SessionOptions: sessionOpts, - Logger: log.L(), - }) - require.NoError(t, err) - encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true - - data1 := []types.Datum{ - types.NewIntDatum(1), - types.NewIntDatum(6), - types.NewStringDatum("1.csv"), - types.NewIntDatum(1), - } - data2 := []types.Datum{ - types.NewIntDatum(1), - types.NewIntDatum(6), - types.NewStringDatum("2.csv"), - types.NewIntDatum(2), - } - data3 := []types.Datum{ - types.NewIntDatum(3), - types.NewIntDatum(3), - types.NewStringDatum("3.csv"), - types.NewIntDatum(3), - } - tctx := encoder.SessionCtx.GetTableCtx() - _, err = encoder.Table.AddRecord(tctx, data1) - require.NoError(t, err) - _, err = encoder.Table.AddRecord(tctx, data2) - require.NoError(t, err) - _, err = encoder.Table.AddRecord(tctx, data3) - require.NoError(t, err) - kvPairs := encoder.SessionCtx.TakeKvPairs() - - data1RowKey := kvPairs.Pairs[0].Key - data1RowValue := kvPairs.Pairs[0].Val - - db, mockDB, err := sqlmock.New() - require.NoError(t, err) - defer func() { - _ = db.Close() - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - mockDB.ExpectExec("CREATE SCHEMA IF NOT EXISTS `lightning_task_info`"). - WillReturnResult(sqlmock.NewResult(1, 1)) - mockDB.ExpectExec("CREATE TABLE IF NOT EXISTS `lightning_task_info`\\.conflict_error_v2.*"). - WillReturnResult(sqlmock.NewResult(2, 1)) - mockDB.ExpectQuery("\\QSELECT COUNT(*) FROM `lightning_task_info`.conflict_error_v2 WHERE table_name = ?\\E"). - WillReturnRows(sqlmock.NewRows([]string{"COUNT(*)"}). - AddRow(2)) - mockDB.ExpectQuery("\\QSELECT raw_key, raw_row FROM `lightning_task_info`.conflict_error_v2 WHERE table_name = ? LIMIT 1\\E"). - WillReturnRows(sqlmock.NewRows([]string{"raw_key", "raw_value"}). - AddRow(data1RowKey, data1RowValue)) - - cfg := config.NewConfig() - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgErr - cfg.App.TaskInfoSchemaName = "lightning_task_info" - em := errormanager.New(db, cfg, log.L()) - err = em.Init(ctx) - require.NoError(t, err) - - err = em.ResolveConflictKeysError( - ctx, "a", - ) - require.Error(t, err, common.ErrFoundDuplicateKeys) - err = mockDB.ExpectationsWereMet() - require.NoError(t, err) -} diff --git a/br/pkg/lightning/importer/dup_detect.go b/br/pkg/lightning/importer/dup_detect.go index b6521f50f3ede..0c89d97662035 100644 --- a/br/pkg/lightning/importer/dup_detect.go +++ b/br/pkg/lightning/importer/dup_detect.go @@ -80,7 +80,7 @@ func (d *dupDetector) run( } func makeDupHandlerConstructor( - sorter extsort.ExternalSorter, onDup string, + sorter extsort.ExternalSorter, onDup config.DuplicateResolutionAlgorithm, ) duplicate.HandlerConstructor { switch onDup { case config.ErrorOnDup: @@ -95,16 +95,8 @@ func makeDupHandlerConstructor( } return &replaceOnDup{w: w}, nil } - case config.IgnoreOnDup: - return func(ctx context.Context) (duplicate.Handler, error) { - w, err := sorter.NewWriter(ctx) - if err != nil { - return nil, err - } - return &ignoreOnDup{w: w}, nil - } default: - panic(fmt.Sprintf("unexpected on-duplicate strategy: %s", onDup)) + panic(fmt.Sprintf("unexpected conflict.strategy: %s", onDup)) } } @@ -114,7 +106,6 @@ var ErrDuplicateKey = errors.Normalize("duplicate key detected on indexID %d of var ( _ duplicate.Handler = &errorOnDup{} _ duplicate.Handler = &replaceOnDup{} - _ duplicate.Handler = &ignoreOnDup{} ) type errorOnDup struct { @@ -180,40 +171,6 @@ func (h *replaceOnDup) Close() error { return h.w.Close() } -type ignoreOnDup struct { - // All keyIDs except the first one will be written to w. - // keyID written to w will be ignored during importing. - w extsort.Writer - first bool - idxID []byte // Varint encoded indexID -} - -func (h *ignoreOnDup) Begin(key []byte) error { - h.first = true - idxID, err := decodeIndexID(key) - if err != nil { - return err - } - h.idxID = codec.EncodeVarint(nil, idxID) - return nil -} - -func (h *ignoreOnDup) Append(keyID []byte) error { - if h.first { - h.first = false - return nil - } - return h.w.Put(keyID, h.idxID) -} - -func (*ignoreOnDup) End() error { - return nil -} - -func (h *ignoreOnDup) Close() error { - return h.w.Close() -} - func (d *dupDetector) addKeys(ctx context.Context, detector *duplicate.Detector) error { g, ctx := errgroup.WithContext(ctx) g.SetLimit(d.rc.cfg.App.RegionConcurrency) diff --git a/br/pkg/lightning/importer/dup_detect_test.go b/br/pkg/lightning/importer/dup_detect_test.go index 9753fb4ce5e55..00c55e6953486 100644 --- a/br/pkg/lightning/importer/dup_detect_test.go +++ b/br/pkg/lightning/importer/dup_detect_test.go @@ -73,19 +73,6 @@ func TestReplaceOnDup(t *testing.T) { ) } -func TestIgnoreOnDup(t *testing.T) { - runDupHandlerTest(t, - func(w extsort.Writer) duplicate.Handler { return &ignoreOnDup{w: w} }, - []dupRecord{{ - exampleHandleKey, [][]byte{[]byte("01"), []byte("02"), []byte("03")}}, - {exampleIndexKey, [][]byte{[]byte("11"), []byte("12"), []byte("13")}}}, - map[int64][][]byte{ - conflictOnHandle: {[]byte("02"), []byte("03")}, - exampleIndexID: {[]byte("12"), []byte("13")}, - }, - ) -} - type dupRecord struct { key []byte rowIDs [][]byte diff --git a/br/pkg/lightning/importer/get_pre_info.go b/br/pkg/lightning/importer/get_pre_info.go index 114bd642b36a3..cf327da85bc23 100644 --- a/br/pkg/lightning/importer/get_pre_info.go +++ b/br/pkg/lightning/importer/get_pre_info.go @@ -90,6 +90,8 @@ type PreImportInfoGetter interface { // TargetInfoGetter defines the operations to get information from target. type TargetInfoGetter interface { + // FetchRemoteDBModels fetches the database structures from the remote target. + FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) // FetchRemoteTableModels fetches the table structures from the remote target. FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) // CheckVersionRequirements performs the check whether the target satisfies the version requirements. @@ -155,6 +157,11 @@ func NewTargetInfoGetterImpl( }, nil } +// FetchRemoteDBModels implements TargetInfoGetter. +func (g *TargetInfoGetterImpl) FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) { + return g.backend.FetchRemoteDBModels(ctx) +} + // FetchRemoteTableModels fetches the table structures from the remote target. // It implements the TargetInfoGetter interface. func (g *TargetInfoGetterImpl) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) { @@ -787,6 +794,12 @@ func (p *PreImportInfoGetterImpl) IsTableEmpty(ctx context.Context, schemaName s return p.targetInfoGetter.IsTableEmpty(ctx, schemaName, tableName) } +// FetchRemoteDBModels fetches the database structures from the remote target. +// It implements the PreImportInfoGetter interface. +func (p *PreImportInfoGetterImpl) FetchRemoteDBModels(ctx context.Context) ([]*model.DBInfo, error) { + return p.targetInfoGetter.FetchRemoteDBModels(ctx) +} + // FetchRemoteTableModels fetches the table structures from the remote target. // It implements the PreImportInfoGetter interface. func (p *PreImportInfoGetterImpl) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) { diff --git a/br/pkg/lightning/importer/import.go b/br/pkg/lightning/importer/import.go index b628cb66d5293..1cba2b3473735 100644 --- a/br/pkg/lightning/importer/import.go +++ b/br/pkg/lightning/importer/import.go @@ -124,8 +124,8 @@ const ( ) var ( - minTiKVVersionForDuplicateResolution = *semver.New("5.2.0") - maxTiKVVersionForDuplicateResolution = version.NextMajorVersion() + minTiKVVersionForConflictStrategy = *semver.New("5.2.0") + maxTiKVVersionForConflictStrategy = version.NextMajorVersion() ) // DeliverPauser is a shared pauser to pause progress to (*chunkProcessor).encodeLoop @@ -373,13 +373,13 @@ func NewImportControllerWithPauser( pdhttp.WithTLSConfig(tls.TLSConfig()), ).WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, pdutil.PDRequestRetryTime*time.Second)) - if cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone { - if err := tikv.CheckTiKVVersion(ctx, pdHTTPCli, minTiKVVersionForDuplicateResolution, maxTiKVVersionForDuplicateResolution); err != nil { + if isLocalBackend(cfg) && cfg.Conflict.Strategy != config.NoneOnDup { + if err := tikv.CheckTiKVVersion(ctx, pdHTTPCli, minTiKVVersionForConflictStrategy, maxTiKVVersionForConflictStrategy); err != nil { if !berrors.Is(err, berrors.ErrVersionMismatch) { return nil, common.ErrCheckKVVersion.Wrap(err).GenWithStackByArgs() } - log.FromContext(ctx).Warn("TiKV version doesn't support duplicate resolution. The resolution algorithm will fall back to 'none'", zap.Error(err)) - cfg.TikvImporter.DuplicateResolution = config.DupeResAlgNone + log.FromContext(ctx).Warn("TiKV version doesn't support conflict strategy. The resolution algorithm will fall back to 'none'", zap.Error(err)) + cfg.Conflict.Strategy = config.NoneOnDup } } @@ -625,29 +625,47 @@ type restoreSchemaWorker struct { func (worker *restoreSchemaWorker) addJob(sqlStr string, job *schemaJob) error { stmts, err := createIfNotExistsStmt(worker.parser, sqlStr, job.dbName, job.tblName) if err != nil { - worker.logger.Warn("failed to rewrite statement, will use raw input instead", - zap.String("db", job.dbName), - zap.String("table", job.tblName), - zap.String("statement", sqlStr), - zap.Error(err)) - job.stmts = []string{sqlStr} - } else { - job.stmts = stmts + return errors.Trace(err) } + job.stmts = stmts return worker.appendJob(job) } func (worker *restoreSchemaWorker) makeJobs( dbMetas []*mydump.MDDatabaseMeta, + getDBs func(context.Context) ([]*model.DBInfo, error), getTables func(context.Context, string) ([]*model.TableInfo, error), ) error { defer func() { close(worker.jobCh) worker.quit() }() - var err error + + if len(dbMetas) == 0 { + return nil + } + // 1. restore databases, execute statements concurrency + + dbs, err := getDBs(worker.ctx) + if err != nil { + worker.logger.Warn("get databases from downstream failed", zap.Error(err)) + } + dbSet := make(set.StringSet, len(dbs)) + for _, db := range dbs { + dbSet.Insert(db.Name.L) + } + for _, dbMeta := range dbMetas { + // if downstream already has this database, we can skip ddl job + if dbSet.Exist(strings.ToLower(dbMeta.Name)) { + worker.logger.Info( + "database already exists in downstream, skip processing the source file", + zap.String("db", dbMeta.Name), + ) + continue + } + sql := dbMeta.GetSchema(worker.ctx, worker.store) err = worker.addJob(sql, &schemaJob{ dbName: dbMeta.Name, @@ -662,18 +680,28 @@ func (worker *restoreSchemaWorker) makeJobs( if err != nil { return err } + // 2. restore tables, execute statements concurrency + for _, dbMeta := range dbMetas { // we can ignore error here, and let check failed later if schema not match - tables, _ := getTables(worker.ctx, dbMeta.Name) - tableMap := make(map[string]struct{}) + tables, err := getTables(worker.ctx, dbMeta.Name) + if err != nil { + worker.logger.Warn("get tables from downstream failed", zap.Error(err)) + } + tableSet := make(set.StringSet, len(tables)) for _, t := range tables { - tableMap[t.Name.L] = struct{}{} + tableSet.Insert(t.Name.L) } for _, tblMeta := range dbMeta.Tables { - if _, ok := tableMap[strings.ToLower(tblMeta.Name)]; ok { + if tableSet.Exist(strings.ToLower(tblMeta.Name)) { // we already has this table in TiDB. // we should skip ddl job and let SchemaValid check. + worker.logger.Info( + "table already exists in downstream, skip processing the source file", + zap.String("db", dbMeta.Name), + zap.String("table", tblMeta.Name), + ) continue } else if tblMeta.SchemaFile.FileMeta.Path == "" { return common.ErrSchemaNotExists.GenWithStackByArgs(dbMeta.Name, tblMeta.Name) @@ -748,7 +776,6 @@ loop: var err error if session == nil { session, err = func() (*sql.Conn, error) { - // TODO: support lightning in SQL return worker.db.Conn(worker.ctx) }() if err != nil { @@ -871,7 +898,7 @@ func (rc *Controller) restoreSchema(ctx context.Context) error { for i := 0; i < concurrency; i++ { go worker.doJob() } - err := worker.makeJobs(rc.dbMetas, rc.preInfoGetter.FetchRemoteTableModels) + err := worker.makeJobs(rc.dbMetas, rc.preInfoGetter.FetchRemoteDBModels, rc.preInfoGetter.FetchRemoteTableModels) logTask.End(zap.ErrorLevel, err) if err != nil { return err @@ -1530,7 +1557,7 @@ func (rc *Controller) importTables(ctx context.Context) (finalErr error) { // output error summary defer rc.outputErrorSummary() - if rc.cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone { + if isLocalBackend(rc.cfg) && rc.cfg.Conflict.Strategy != config.NoneOnDup { subCtx, cancel := context.WithCancel(ctx) exitCh, err := rc.keepPauseGCForDupeRes(subCtx) if err != nil { diff --git a/br/pkg/lightning/importer/mock/mock.go b/br/pkg/lightning/importer/mock/mock.go index 27605efb2db50..024c252cdcf3e 100644 --- a/br/pkg/lightning/importer/mock/mock.go +++ b/br/pkg/lightning/importer/mock/mock.go @@ -212,6 +212,15 @@ func (t *TargetInfo) SetTableInfo(schemaName string, tableName string, tblInfo * t.dbTblInfoMap[schemaName][tableName] = tblInfo } +// FetchRemoteDBModels implements the TargetInfoGetter interface. +func (t *TargetInfo) FetchRemoteDBModels(_ context.Context) ([]*model.DBInfo, error) { + resultInfos := []*model.DBInfo{} + for dbName := range t.dbTblInfoMap { + resultInfos = append(resultInfos, &model.DBInfo{Name: model.NewCIStr(dbName)}) + } + return resultInfos, nil +} + // FetchRemoteTableModels fetches the table structures from the remote target. // It implements the TargetInfoGetter interface. func (t *TargetInfo) FetchRemoteTableModels(_ context.Context, schemaName string) ([]*model.TableInfo, error) { diff --git a/br/pkg/lightning/importer/restore_schema_test.go b/br/pkg/lightning/importer/restore_schema_test.go index f6dfbc16ecc6d..9d45ff6417ef7 100644 --- a/br/pkg/lightning/importer/restore_schema_test.go +++ b/br/pkg/lightning/importer/restore_schema_test.go @@ -136,6 +136,10 @@ func (s *restoreSchemaSuite) SetupTest() { s.controller, s.ctx = gomock.WithContext(context.Background(), s.T()) mockTargetInfoGetter := mock.NewMockTargetInfoGetter(s.controller) mockBackend := mock.NewMockBackend(s.controller) + mockTargetInfoGetter.EXPECT(). + FetchRemoteDBModels(gomock.Any()). + AnyTimes(). + Return([]*model.DBInfo{{Name: model.NewCIStr("fakedb")}}, nil) mockTargetInfoGetter.EXPECT(). FetchRemoteTableModels(gomock.Any(), gomock.Any()). AnyTimes(). diff --git a/br/pkg/lightning/importer/table_import.go b/br/pkg/lightning/importer/table_import.go index 2308e3e47520f..0fad14432bc66 100644 --- a/br/pkg/lightning/importer/table_import.go +++ b/br/pkg/lightning/importer/table_import.go @@ -208,7 +208,7 @@ func (tr *TableImporter) importTable( } // 2. Do duplicate detection if needed - if isLocalBackend(rc.cfg) && rc.cfg.Conflict.Strategy != "" { + if isLocalBackend(rc.cfg) && rc.cfg.Conflict.PrecheckConflictBeforeImport && rc.cfg.Conflict.Strategy != config.NoneOnDup { _, uuid := backend.MakeUUID(tr.tableName, common.IndexEngineID) workingDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+local.DupDetectDirSuffix) resultDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+local.DupResultDirSuffix) @@ -1020,13 +1020,13 @@ func (tr *TableImporter) postProcess( localBackend := rc.backend.(*local.Backend) dupeController := localBackend.GetDupeController(rc.cfg.TikvImporter.RangeConcurrency*2, rc.errorMgr) hasDupe := false - if rc.cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone { + if rc.cfg.Conflict.Strategy != config.NoneOnDup { opts := &encode.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, SysVars: rc.sysVars, } var err error - hasLocalDupe, err := dupeController.CollectLocalDuplicateRows(ctx, tr.encTable, tr.tableName, opts) + hasLocalDupe, err := dupeController.CollectLocalDuplicateRows(ctx, tr.encTable, tr.tableName, opts, rc.cfg.Conflict.Strategy) if err != nil { tr.logger.Error("collect local duplicate keys failed", log.ShortError(err)) return false, errors.Trace(err) @@ -1047,12 +1047,12 @@ func (tr *TableImporter) postProcess( needChecksum := !otherHasDupe && needRemoteDupe hasDupe = hasDupe || otherHasDupe - if needRemoteDupe && rc.cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone { + if needRemoteDupe && rc.cfg.Conflict.Strategy != config.NoneOnDup { opts := &encode.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, SysVars: rc.sysVars, } - hasRemoteDupe, e := dupeController.CollectRemoteDuplicateRows(ctx, tr.encTable, tr.tableName, opts) + hasRemoteDupe, e := dupeController.CollectRemoteDuplicateRows(ctx, tr.encTable, tr.tableName, opts, rc.cfg.Conflict.Strategy) if e != nil { tr.logger.Error("collect remote duplicate keys failed", log.ShortError(e)) return false, errors.Trace(e) @@ -1060,7 +1060,7 @@ func (tr *TableImporter) postProcess( hasDupe = hasDupe || hasRemoteDupe if hasDupe { - if err = dupeController.ResolveDuplicateRows(ctx, tr.encTable, tr.tableName, rc.cfg.TikvImporter.DuplicateResolution); err != nil { + if err = dupeController.ResolveDuplicateRows(ctx, tr.encTable, tr.tableName, rc.cfg.Conflict.Strategy); err != nil { tr.logger.Error("resolve remote duplicate keys failed", log.ShortError(err)) return false, errors.Trace(err) } @@ -1340,6 +1340,9 @@ func (tr *TableImporter) importKV( } } err := closedEngine.Import(ctx, regionSplitSize, regionSplitKeys) + if common.ErrFoundDuplicateKeys.Equal(err) { + err = local.ConvertToErrFoundConflictRecords(err, tr.encTable) + } saveCpErr := rc.saveStatusCheckpoint(ctx, tr.tableName, closedEngine.GetID(), err, checkpoints.CheckpointStatusImported) // Don't clean up when save checkpoint failed, because we will verifyLocalFile and import engine again after restart. if err == nil && saveCpErr == nil { diff --git a/br/pkg/lightning/importer/table_import_test.go b/br/pkg/lightning/importer/table_import_test.go index 7b6d3991dc89e..13e311ca16d9d 100644 --- a/br/pkg/lightning/importer/table_import_test.go +++ b/br/pkg/lightning/importer/table_import_test.go @@ -1241,6 +1241,10 @@ type mockPDClient struct { leaderAddr string } +func (m *mockPDClient) GetClusterID(_ context.Context) uint64 { + return 1 +} + func (m *mockPDClient) GetLeaderAddr() string { return m.leaderAddr } diff --git a/br/pkg/metautil/load.go b/br/pkg/metautil/load.go index 6f62a534b337b..e0c2ad717f3b3 100644 --- a/br/pkg/metautil/load.go +++ b/br/pkg/metautil/load.go @@ -38,11 +38,15 @@ func (db *Database) GetTable(name string) *Table { } // LoadBackupTables loads schemas from BackupMeta. -func LoadBackupTables(ctx context.Context, reader *MetaReader) (map[string]*Database, error) { +func LoadBackupTables(ctx context.Context, reader *MetaReader, loadStats bool) (map[string]*Database, error) { ch := make(chan *Table) errCh := make(chan error) go func() { - if err := reader.ReadSchemasFiles(ctx, ch); err != nil { + var opts []ReadSchemaOption + if !loadStats { + opts = []ReadSchemaOption{SkipStats} + } + if err := reader.ReadSchemasFiles(ctx, ch, opts...); err != nil { errCh <- errors.Trace(err) } close(ch) diff --git a/br/pkg/metautil/load_test.go b/br/pkg/metautil/load_test.go index 92a8987ef7ee9..ef885f3bcbf26 100644 --- a/br/pkg/metautil/load_test.go +++ b/br/pkg/metautil/load_test.go @@ -105,6 +105,7 @@ func TestLoadBackupMeta(t *testing.T) { &backuppb.CipherInfo{ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }), + true, ) tbl := dbs[dbName.String()].GetTable(tblName.String()) require.NoError(t, err) @@ -201,6 +202,7 @@ func TestLoadBackupMetaPartionTable(t *testing.T) { CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }, ), + true, ) tbl := dbs[dbName.String()].GetTable(tblName.String()) require.NoError(t, err) @@ -287,6 +289,7 @@ func BenchmarkLoadBackupMeta64(b *testing.B) { CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }, ), + true, ) require.NoError(b, err) require.Len(b, dbs, 1) @@ -319,6 +322,7 @@ func BenchmarkLoadBackupMeta1024(b *testing.B) { CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }, ), + true, ) require.NoError(b, err) require.Len(b, dbs, 1) @@ -351,6 +355,7 @@ func BenchmarkLoadBackupMeta10240(b *testing.B) { CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }, ), + true, ) require.NoError(b, err) require.Len(b, dbs, 1) diff --git a/br/pkg/metautil/metafile.go b/br/pkg/metautil/metafile.go index e8292d32972b3..a5a363c25ba8a 100644 --- a/br/pkg/metautil/metafile.go +++ b/br/pkg/metautil/metafile.go @@ -291,6 +291,7 @@ func (reader *MetaReader) ReadDDLs(ctx context.Context) ([]byte, error) { type readSchemaConfig struct { skipFiles bool + skipStats bool } // ReadSchemaOption describes some extra option of reading the config. @@ -302,6 +303,10 @@ func SkipFiles(conf *readSchemaConfig) { conf.skipFiles = true } +func SkipStats(conf *readSchemaConfig) { + conf.skipStats = true +} + // GetBasic returns a basic copy of the backup meta. func (reader *MetaReader) GetBasic() backuppb.BackupMeta { return *reader.backupMeta @@ -313,6 +318,10 @@ func (reader *MetaReader) ReadSchemasFiles(ctx context.Context, output chan<- *T cctx, cancel := context.WithCancel(ctx) defer cancel() + cfg := readSchemaConfig{} + for _, opt := range opts { + opt(&cfg) + } ch := make(chan any, MaxBatchSize) schemaCh := make(chan *backuppb.Schema, MaxBatchSize) // Make sure these 2 goroutine avoid to blocked by the errCh. @@ -322,6 +331,10 @@ func (reader *MetaReader) ReadSchemasFiles(ctx context.Context, output chan<- *T go func() { defer close(schemaCh) if err := reader.readSchemas(cctx, func(s *backuppb.Schema) { + if cfg.skipStats { + s.Stats = nil + s.StatsIndex = nil + } select { case <-cctx.Done(): case schemaCh <- s: @@ -362,11 +375,6 @@ func (reader *MetaReader) ReadSchemasFiles(ctx context.Context, output chan<- *T } }() - cfg := readSchemaConfig{} - for _, opt := range opts { - opt(&cfg) - } - // It's not easy to balance memory and time costs for current structure. // put all files in memory due to https://github.com/pingcap/br/issues/705 var fileMap map[int64][]*backuppb.File diff --git a/br/pkg/mock/backend.go b/br/pkg/mock/backend.go index 66c829552f5f7..6612fc2b170fd 100644 --- a/br/pkg/mock/backend.go +++ b/br/pkg/mock/backend.go @@ -299,6 +299,21 @@ func (mr *MockTargetInfoGetterMockRecorder) CheckRequirements(arg0, arg1 any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckRequirements", reflect.TypeOf((*MockTargetInfoGetter)(nil).CheckRequirements), arg0, arg1) } +// FetchRemoteDBModels mocks base method. +func (m *MockTargetInfoGetter) FetchRemoteDBModels(arg0 context.Context) ([]*model.DBInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FetchRemoteDBModels", arg0) + ret0, _ := ret[0].([]*model.DBInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FetchRemoteDBModels indicates an expected call of FetchRemoteDBModels. +func (mr *MockTargetInfoGetterMockRecorder) FetchRemoteDBModels(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchRemoteDBModels", reflect.TypeOf((*MockTargetInfoGetter)(nil).FetchRemoteDBModels), arg0) +} + // FetchRemoteTableModels mocks base method. func (m *MockTargetInfoGetter) FetchRemoteTableModels(arg0 context.Context, arg1 string) ([]*model.TableInfo, error) { m.ctrl.T.Helper() diff --git a/br/pkg/mock/encode.go b/br/pkg/mock/encode.go index 0470325d353c9..4c466b549fec9 100644 --- a/br/pkg/mock/encode.go +++ b/br/pkg/mock/encode.go @@ -157,20 +157,6 @@ func (mr *MockRowsMockRecorder) Clear() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clear", reflect.TypeOf((*MockRows)(nil).Clear)) } -// SplitIntoChunks mocks base method. -func (m *MockRows) SplitIntoChunks(arg0 int) []encode.Rows { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SplitIntoChunks", arg0) - ret0, _ := ret[0].([]encode.Rows) - return ret0 -} - -// SplitIntoChunks indicates an expected call of SplitIntoChunks. -func (mr *MockRowsMockRecorder) SplitIntoChunks(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SplitIntoChunks", reflect.TypeOf((*MockRows)(nil).SplitIntoChunks), arg0) -} - // MockRow is a mock of Row interface. type MockRow struct { ctrl *gomock.Controller diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go index 16e484b449687..d110abbc49dcc 100644 --- a/br/pkg/pdutil/pd.go +++ b/br/pkg/pdutil/pd.go @@ -159,7 +159,6 @@ func NewPdController( // If the time too short, we may scatter a region many times, because // the interface `ScatterRegions` may time out. pd.WithCustomTimeoutOption(60*time.Second), - pd.WithMaxErrorRetry(3), ) if err != nil { log.Error("fail to create pd client", zap.Error(err)) diff --git a/br/pkg/restore/batcher.go b/br/pkg/restore/batcher.go index 57bd477822875..033ef13f457f0 100644 --- a/br/pkg/restore/batcher.go +++ b/br/pkg/restore/batcher.go @@ -36,7 +36,6 @@ const ( type Batcher struct { cachedTables []TableWithRange cachedTablesMu *sync.Mutex - rewriteRules *RewriteRules // autoCommitJoiner is for joining the background batch sender. autoCommitJoiner chan<- struct{} @@ -114,7 +113,6 @@ func NewBatcher( outCh := DefaultOutputTableChan() sendChan := make(chan SendType, 2) b := &Batcher{ - rewriteRules: EmptyRewriteRule(), sendErr: errCh, outCh: outCh, sender: sender, @@ -227,8 +225,10 @@ type DrainResult struct { TablesToSend []CreatedTable // BlankTablesAfterSend are tables that will be full-restored after this batch send. BlankTablesAfterSend []CreatedTable - RewriteRules *RewriteRules - Ranges []rtree.Range + // RewriteRules are the rewrite rules for the tables. + // the key is the table id after rewritten. + RewriteRulesMap map[int64]*RewriteRules + Ranges []rtree.Range // Record which part of ranges belongs to the table TableEndOffsetInRanges []int } @@ -240,14 +240,19 @@ func (result DrainResult) Files() []TableIDWithFiles { for i, endOffset := range result.TableEndOffsetInRanges { tableID := result.TablesToSend[i].Table.ID ranges := result.Ranges[startOffset:endOffset] - files := make([]*backuppb.File, 0, len(result.Ranges)*2) + // each range has at least a default file + a write file + files := make([]*backuppb.File, 0, len(ranges)*2) for _, rg := range ranges { files = append(files, rg.Files...) } - + var rules *RewriteRules + if r, ok := result.RewriteRulesMap[tableID]; ok { + rules = r + } tableIDWithFiles = append(tableIDWithFiles, TableIDWithFiles{ - TableID: tableID, - Files: files, + TableID: tableID, + Files: files, + RewriteRules: rules, }) // update start offset @@ -261,7 +266,7 @@ func newDrainResult() DrainResult { return DrainResult{ TablesToSend: make([]CreatedTable, 0), BlankTablesAfterSend: make([]CreatedTable, 0), - RewriteRules: EmptyRewriteRule(), + RewriteRulesMap: EmptyRewriteRulesMap(), Ranges: make([]rtree.Range, 0), TableEndOffsetInRanges: make([]int, 0), } @@ -329,7 +334,7 @@ func (b *Batcher) drainRanges() DrainResult { thisTableLen := len(thisTable.Range) collected := len(result.Ranges) - result.RewriteRules.Append(*thisTable.RewriteRule) + result.RewriteRulesMap[thisTable.Table.ID] = thisTable.RewriteRule result.TablesToSend = append(result.TablesToSend, thisTable.CreatedTable) // the batch is full, we should stop here! @@ -423,7 +428,6 @@ func (b *Batcher) Add(tbs TableWithRange) { zap.Int("batch size", b.Len()), ) b.cachedTables = append(b.cachedTables, tbs) - b.rewriteRules.Append(*tbs.RewriteRule) atomic.AddInt32(&b.size, int32(len(tbs.Range))) b.cachedTablesMu.Unlock() diff --git a/br/pkg/restore/batcher_test.go b/br/pkg/restore/batcher_test.go index d6d7cab7e90c7..dad4634becf73 100644 --- a/br/pkg/restore/batcher_test.go +++ b/br/pkg/restore/batcher_test.go @@ -39,7 +39,9 @@ func (sender *drySender) RestoreBatch(ranges restore.DrainResult) { defer sender.mu.Unlock() log.Info("fake restore range", rtree.ZapRanges(ranges.Ranges)) sender.nBatch++ - sender.rewriteRules.Append(*ranges.RewriteRules) + for _, r := range ranges.RewriteRulesMap { + sender.rewriteRules.Append(*r) + } sender.ranges = append(sender.ranges, ranges.Ranges...) sender.sink.EmitTables(ranges.BlankTablesAfterSend...) } diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index 1fe97552743f5..74db13d968e15 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -395,7 +395,9 @@ func (rc *Client) InitCheckpointMetadataForLogRestore(ctx context.Context, taskN return gcRatio, nil } -func (rc *Client) allocTableIDs(ctx context.Context, tables []*metautil.Table) error { +// AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in +// the download stage. +func (rc *Client) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error { rc.preallocedTableIDs = tidalloc.New(tables) ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR) err := kv.RunInNewTxn(ctx, rc.GetDomain().Store(), true, func(_ context.Context, txn kv.Transaction) error { @@ -539,7 +541,6 @@ func (rc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBacke } func (rc *Client) InitClients(ctx context.Context, backend *backuppb.StorageBackend, isRawKvMode bool, isTxnKvMode bool) { - storeWorkerPoolMap := make(map[uint64]chan struct{}) stores, err := conn.GetAllTiKVStoresWithRetry(ctx, rc.pdClient, util.SkipTiFlash) if err != nil { log.Fatal("failed to get stores", zap.Error(err)) @@ -552,15 +553,11 @@ func (rc *Client) InitClients(ctx context.Context, backend *backuppb.StorageBack // ToDo remove it when token bucket is stable enough. log.Info("use token bucket to control download and ingest flow") useTokenBucket = true - for _, store := range stores { - ch := utils.BuildWorkerTokenChannel(concurrencyPerStore) - storeWorkerPoolMap[store.Id] = ch - } } metaClient := split.NewSplitClient(rc.pdClient, rc.pdHTTPClient, rc.tlsConf, isRawKvMode) importCli := NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf) - rc.fileImporter = NewFileImporter(metaClient, importCli, backend, isRawKvMode, isTxnKvMode, storeWorkerPoolMap, rc.rewriteMode, concurrencyPerStore, useTokenBucket) + rc.fileImporter = NewFileImporter(metaClient, importCli, backend, isRawKvMode, isTxnKvMode, stores, rc.rewriteMode, concurrencyPerStore, useTokenBucket) } func (rc *Client) SetRawKVClient(c *RawKVBatchClient) { @@ -576,9 +573,10 @@ func (rc *Client) InitBackupMeta( c context.Context, backupMeta *backuppb.BackupMeta, backend *backuppb.StorageBackend, - reader *metautil.MetaReader) error { + reader *metautil.MetaReader, + loadStats bool) error { if rc.needLoadSchemas(backupMeta) { - databases, err := metautil.LoadBackupTables(c, reader) + databases, err := metautil.LoadBackupTables(c, reader, loadStats) if err != nil { return errors.Trace(err) } @@ -1026,11 +1024,6 @@ func (rc *Client) GoCreateTables( } outCh := make(chan CreatedTable, len(tables)) rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter) - if err := rc.allocTableIDs(ctx, tables); err != nil { - errCh <- err - close(outCh) - return outCh - } var err error @@ -1407,10 +1400,9 @@ func getGroupFiles(files []*backuppb.File, supportMulti bool) [][]*backuppb.File // SplitRanges implements TiKVRestorer. func (rc *Client) SplitRanges(ctx context.Context, ranges []rtree.Range, - rewriteRules *RewriteRules, updateCh glue.Progress, isRawKv bool) error { - return SplitRanges(ctx, rc, ranges, rewriteRules, updateCh, isRawKv) + return SplitRanges(ctx, rc, ranges, updateCh, isRawKv) } func (rc *Client) WrapLogFilesIterWithSplitHelper(logIter LogIter, rules map[int64]*RewriteRules, g glue.Glue, store kv.Storage) (LogIter, error) { @@ -1470,7 +1462,6 @@ func (rc *Client) WrapLogFilesIterWithCheckpoint( func (rc *Client) RestoreSSTFiles( ctx context.Context, tableIDWithFiles []TableIDWithFiles, - rewriteRules *RewriteRules, updateCh glue.Progress, ) (err error) { start := time.Now() @@ -1505,6 +1496,7 @@ LOOPFORTABLE: for _, tableIDWithFile := range tableIDWithFiles { tableID := tableIDWithFile.TableID files := tableIDWithFile.Files + rules := tableIDWithFile.RewriteRules fileCount += len(files) for rangeFiles, leftFiles = drainFilesByRange(files); len(rangeFiles) != 0; rangeFiles, leftFiles = drainFilesByRange(leftFiles) { filesReplica := rangeFiles @@ -1529,7 +1521,7 @@ LOOPFORTABLE: updateCh.Inc() } }() - return rc.fileImporter.ImportSSTFiles(ectx, fs, rewriteRules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion()) + return rc.fileImporter.ImportSSTFiles(ectx, fs, rules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion()) }(filesGroup); importErr != nil { return errors.Trace(importErr) } @@ -1547,7 +1539,13 @@ LOOPFORTABLE: return nil } if rc.granularity == string(CoarseGrained) { + rc.fileImporter.cond.L.Lock() + for rc.fileImporter.ShouldBlock() { + // wait for download worker notified + rc.fileImporter.cond.Wait() + } eg.Go(restoreFn) + rc.fileImporter.cond.L.Unlock() } else { // if we are not use coarse granularity which means // we still pipeline split & scatter regions and import sst files @@ -1869,62 +1867,54 @@ func (rc *Client) GoUpdateMetaAndLoadStats( inCh <-chan *CreatedTable, errCh chan<- error, statsConcurrency uint, + loadStats bool, ) chan *CreatedTable { log.Info("Start to update meta then load stats") outCh := DefaultOutputTableChan() workers := utils.NewWorkerPool(statsConcurrency, "UpdateStats") - // The rc.db is not thread safe - var updateMetaLock sync.Mutex go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error { oldTable := tbl.OldTable - // Not need to return err when failed because of update analysis-meta - restoreTS, err := rc.GetTSWithRetry(ctx) - if err != nil { - log.Error("getTS failed", zap.Error(err)) - } else { - updateMetaLock.Lock() - - log.Info("start update metas", - zap.Stringer("table", oldTable.Info.Name), - zap.Stringer("db", oldTable.DB.Name)) - err = rc.db.UpdateStatsMeta(ctx, tbl.Table.ID, restoreTS, oldTable.TotalKvs) - if err != nil { - log.Error("update stats meta failed", zap.Any("table", tbl.Table), zap.Error(err)) - } - - updateMetaLock.Unlock() - } - - if oldTable.Stats != nil { + var statsErr error = nil + if loadStats && oldTable.Stats != nil { log.Info("start loads analyze after validate checksum", zap.Int64("old id", oldTable.Info.ID), zap.Int64("new id", tbl.Table.ID), ) start := time.Now() // NOTICE: skip updating cache after load stats from json - if err := rc.statsHandler.LoadStatsFromJSONNoUpdate(ctx, rc.dom.InfoSchema(), oldTable.Stats, 0); err != nil { - log.Error("analyze table failed", zap.Any("table", oldTable.Stats), zap.Error(err)) + if statsErr = rc.statsHandler.LoadStatsFromJSONNoUpdate(ctx, rc.dom.InfoSchema(), oldTable.Stats, 0); statsErr != nil { + log.Error("analyze table failed", zap.Any("table", oldTable.Stats), zap.Error(statsErr)) } log.Info("restore stat done", zap.Stringer("table", oldTable.Info.Name), zap.Stringer("db", oldTable.DB.Name), zap.Duration("cost", time.Since(start))) - } else if oldTable.StatsFileIndexes != nil { + } else if loadStats && len(oldTable.StatsFileIndexes) > 0 { log.Info("start to load statistic data for each partition", zap.Int64("old id", oldTable.Info.ID), zap.Int64("new id", tbl.Table.ID), ) start := time.Now() rewriteIDMap := getTableIDMap(tbl.Table, tbl.OldTable.Info) - if err := metautil.RestoreStats(ctx, s, cipher, rc.statsHandler, tbl.Table, oldTable.StatsFileIndexes, rewriteIDMap); err != nil { - log.Error("analyze table failed", zap.Any("table", oldTable.StatsFileIndexes), zap.Error(err)) + if statsErr = metautil.RestoreStats(ctx, s, cipher, rc.statsHandler, tbl.Table, oldTable.StatsFileIndexes, rewriteIDMap); statsErr != nil { + log.Error("analyze table failed", zap.Any("table", oldTable.StatsFileIndexes), zap.Error(statsErr)) } log.Info("restore statistic data done", zap.Stringer("table", oldTable.Info.Name), zap.Stringer("db", oldTable.DB.Name), zap.Duration("cost", time.Since(start))) } + + if statsErr != nil || !loadStats || (oldTable.Stats == nil && len(oldTable.StatsFileIndexes) == 0) { + // Not need to return err when failed because of update analysis-meta + log.Info("start update metas", zap.Stringer("table", oldTable.Info.Name), zap.Stringer("db", oldTable.DB.Name)) + // the total kvs contains the index kvs, but the stats meta needs the count of rows + count := int64(oldTable.TotalKvs / uint64(len(oldTable.Info.Indices)+1)) + if statsErr = rc.statsHandler.SaveMetaToStorage(tbl.Table.ID, count, 0, "br restore"); statsErr != nil { + log.Error("update stats meta failed", zap.Any("table", tbl.Table), zap.Error(statsErr)) + } + } return nil }, func() { log.Info("all stats updated") @@ -2159,7 +2149,7 @@ func (rc *Client) ResetRestoreLabels(ctx context.Context) error { if !rc.isOnline { return nil } - log.Info("start reseting store labels") + log.Info("start resetting store labels") return rc.toolClient.SetStoresLabel(ctx, rc.restoreStores, restoreLabelKey, "") } @@ -2258,7 +2248,7 @@ func (rc *Client) ResetPlacementRules(ctx context.Context, tables []*model.Table if !rc.isOnline || len(rc.restoreStores) == 0 { return nil } - log.Info("start reseting placement rules") + log.Info("start resetting placement rules") var failedTables []int64 for _, t := range tables { err := rc.toolClient.DeletePlacementRule(ctx, "pd", rc.getRuleID(t.ID)) @@ -2805,7 +2795,7 @@ func initFullBackupTables( // read full backup databases to get map[table]table.Info reader := metautil.NewMetaReader(backupMeta, s, nil) - databases, err := metautil.LoadBackupTables(ctx, reader) + databases, err := metautil.LoadBackupTables(ctx, reader, false) if err != nil { return nil, errors.Trace(err) } diff --git a/br/pkg/restore/db_test.go b/br/pkg/restore/db_test.go index 01c21892712bd..24abf08dc24b6 100644 --- a/br/pkg/restore/db_test.go +++ b/br/pkg/restore/db_test.go @@ -384,7 +384,7 @@ func TestGetExistedUserDBs(t *testing.T) { }, nil, nil, 1) require.Nil(t, err) - dom.MockInfoCacheAndLoadInfoSchema(builder.Build()) + dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64)) dbs = restore.GetExistedUserDBs(dom) require.Equal(t, 0, len(dbs)) @@ -396,7 +396,7 @@ func TestGetExistedUserDBs(t *testing.T) { }, nil, nil, 1) require.Nil(t, err) - dom.MockInfoCacheAndLoadInfoSchema(builder.Build()) + dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64)) dbs = restore.GetExistedUserDBs(dom) require.Equal(t, 1, len(dbs)) @@ -412,7 +412,7 @@ func TestGetExistedUserDBs(t *testing.T) { }, nil, nil, 1) require.Nil(t, err) - dom.MockInfoCacheAndLoadInfoSchema(builder.Build()) + dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64)) dbs = restore.GetExistedUserDBs(dom) require.Equal(t, 2, len(dbs)) } @@ -425,5 +425,5 @@ func TestGetExistedUserDBs(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { - require.Equal(t, int64(186), session.CurrentBootstrapVersion) + require.Equal(t, int64(195), session.CurrentBootstrapVersion) } diff --git a/br/pkg/restore/import.go b/br/pkg/restore/import.go index b78f98224bc96..8f909cea1fc63 100644 --- a/br/pkg/restore/import.go +++ b/br/pkg/restore/import.go @@ -311,16 +311,72 @@ func (ic *importClient) SupportMultiIngest(ctx context.Context, stores []uint64) return true, nil } +type storeTokenChannelMap struct { + sync.RWMutex + tokens map[uint64]chan struct{} +} + +func (s *storeTokenChannelMap) acquireTokenCh(storeID uint64, bufferSize uint) chan struct{} { + s.RLock() + tokenCh, ok := s.tokens[storeID] + // handle the case that the store is new-scaled in the cluster + if !ok { + s.RUnlock() + s.Lock() + // Notice: worker channel can't replaced, because it is still used after unlock. + if tokenCh, ok = s.tokens[storeID]; !ok { + tokenCh = utils.BuildWorkerTokenChannel(bufferSize) + s.tokens[storeID] = tokenCh + } + s.Unlock() + } else { + s.RUnlock() + } + return tokenCh +} + +func (s *storeTokenChannelMap) ShouldBlock() bool { + s.RLock() + defer s.RUnlock() + if len(s.tokens) == 0 { + // never block if there is no store worker pool + return false + } + for _, pool := range s.tokens { + if len(pool) > 0 { + // At least one store worker pool has available worker + return false + } + } + return true +} + +func newStoreTokenChannelMap(stores []*metapb.Store, bufferSize uint) *storeTokenChannelMap { + storeTokenChannelMap := &storeTokenChannelMap{ + sync.RWMutex{}, + make(map[uint64]chan struct{}), + } + if bufferSize == 0 { + return storeTokenChannelMap + } + for _, store := range stores { + ch := utils.BuildWorkerTokenChannel(bufferSize) + storeTokenChannelMap.tokens[store.Id] = ch + } + return storeTokenChannelMap +} + // FileImporter used to import a file to TiKV. type FileImporter struct { metaClient split.SplitClient importClient ImporterClient backend *backuppb.StorageBackend - storeWorkerPoolRWLock sync.RWMutex - storeWorkerPoolMap map[uint64]chan struct{} - concurrencyPerStore uint - useTokenBucket bool + downloadTokensMap *storeTokenChannelMap + ingestTokensMap *storeTokenChannelMap + + concurrencyPerStore uint + useTokenBucket bool kvMode KvMode rawStartKey []byte @@ -329,6 +385,7 @@ type FileImporter struct { rewriteMode RewriteMode cacheKey string + cond *sync.Cond } // NewFileImporter returns a new file importClient. @@ -338,7 +395,7 @@ func NewFileImporter( backend *backuppb.StorageBackend, isRawKvMode bool, isTxnKvMode bool, - storeWorkerPoolMap map[uint64]chan struct{}, + stores []*metapb.Store, rewriteMode RewriteMode, concurrencyPerStore uint, useTokenBucket bool, @@ -350,17 +407,42 @@ func NewFileImporter( if isTxnKvMode { kvMode = Txn } + + downloadTokensMap := newStoreTokenChannelMap(stores, 0) + ingestTokensMap := newStoreTokenChannelMap(stores, 0) + + if useTokenBucket { + downloadTokensMap = newStoreTokenChannelMap(stores, concurrencyPerStore) + ingestTokensMap = newStoreTokenChannelMap(stores, concurrencyPerStore) + } return FileImporter{ metaClient: metaClient, backend: backend, importClient: importClient, - storeWorkerPoolMap: storeWorkerPoolMap, + downloadTokensMap: downloadTokensMap, + ingestTokensMap: ingestTokensMap, kvMode: kvMode, rewriteMode: rewriteMode, cacheKey: fmt.Sprintf("BR-%s-%d", time.Now().Format("20060102150405"), rand.Int63()), concurrencyPerStore: concurrencyPerStore, useTokenBucket: useTokenBucket, + cond: sync.NewCond(new(sync.Mutex)), + } +} + +func (importer *FileImporter) ShouldBlock() bool { + if importer != nil { + return importer.downloadTokensMap.ShouldBlock() || importer.ingestTokensMap.ShouldBlock() } + return false +} + +func (importer *FileImporter) releaseToken(tokenCh chan struct{}) { + tokenCh <- struct{}{} + // finish the task, notify the main goroutine to continue + importer.cond.L.Lock() + importer.cond.Signal() + importer.cond.L.Unlock() } func (importer *FileImporter) Close() error { @@ -1031,28 +1113,14 @@ func (importer *FileImporter) downloadSSTV2( for _, p := range regionInfo.Region.GetPeers() { peer := p eg.Go(func() error { - importer.storeWorkerPoolRWLock.RLock() - workerCh, ok := importer.storeWorkerPoolMap[peer.GetStoreId()] - // handle the case that the store is new-scaled in the cluster - if !ok { - importer.storeWorkerPoolRWLock.RUnlock() - importer.storeWorkerPoolRWLock.Lock() - // Notice: worker channel can't replaced, because it is still used after unlock. - if workerCh, ok = importer.storeWorkerPoolMap[peer.GetStoreId()]; !ok { - workerCh = utils.BuildWorkerTokenChannel(importer.concurrencyPerStore) - importer.storeWorkerPoolMap[peer.GetStoreId()] = workerCh - } - importer.storeWorkerPoolRWLock.Unlock() - } else { - importer.storeWorkerPoolRWLock.RUnlock() - } + tokenCh := importer.downloadTokensMap.acquireTokenCh(peer.GetStoreId(), importer.concurrencyPerStore) select { case <-ectx.Done(): return ectx.Err() - case <-workerCh: + case <-tokenCh: } defer func() { - workerCh <- struct{}{} + importer.releaseToken(tokenCh) }() for _, file := range files { req, ok := downloadReqsMap[file.Name] @@ -1191,6 +1259,15 @@ func (importer *FileImporter) ingest( info *split.RegionInfo, downloadMetas []*import_sstpb.SSTMeta, ) error { + tokenCh := importer.ingestTokensMap.acquireTokenCh(info.Leader.GetStoreId(), importer.concurrencyPerStore) + select { + case <-ctx.Done(): + return ctx.Err() + case <-tokenCh: + } + defer func() { + importer.releaseToken(tokenCh) + }() for { ingestResp, errIngest := importer.ingestSSTs(ctx, downloadMetas, info) if errIngest != nil { diff --git a/br/pkg/restore/import_retry_test.go b/br/pkg/restore/import_retry_test.go index 6dbc05e2d402e..411af37c46d2a 100644 --- a/br/pkg/restore/import_retry_test.go +++ b/br/pkg/restore/import_retry_test.go @@ -163,7 +163,7 @@ func TestServerIsBusy(t *testing.T) { require.NoError(t, err) assertRegions(t, idEqualsTo2Regions, "aay", "bba") assertRegions(t, meetRegions, "", "aay", "bba", "bbh", "cca", "") - require.Equal(t, rs.RetryTimes(), 1) + require.Equal(t, rs.Attempt(), 1) } func TestServerIsBusyWithMemoryIsLimited(t *testing.T) { @@ -204,7 +204,7 @@ func TestServerIsBusyWithMemoryIsLimited(t *testing.T) { require.NoError(t, err) assertRegions(t, idEqualsTo2Regions, "aay", "bba") assertRegions(t, meetRegions, "", "aay", "bba", "bbh", "cca", "") - require.Equal(t, rs.RetryTimes(), 0) + require.Equal(t, rs.Attempt(), 2) } func printRegion(name string, infos []*split.RegionInfo) { diff --git a/br/pkg/restore/merge.go b/br/pkg/restore/merge.go index deaa1d1c48bd8..7f588d1483804 100644 --- a/br/pkg/restore/merge.go +++ b/br/pkg/restore/merge.go @@ -7,8 +7,13 @@ import ( "github.com/pingcap/errors" backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/log" berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/rtree" + "github.com/pingcap/tidb/pkg/tablecodec" + "go.uber.org/zap" ) const ( @@ -29,13 +34,16 @@ type MergeRangesStat struct { MergedRegionBytesAvg int } -// MergeFileRanges returns ranges of the files are merged based on +// MergeAndRewriteFileRanges returns ranges of the files are merged based on // splitSizeBytes and splitKeyCount. // // By merging small ranges, it speeds up restoring a backup that contains many // small ranges (regions) as it reduces split region and scatter region. -func MergeFileRanges( - files []*backuppb.File, splitSizeBytes, splitKeyCount uint64, +func MergeAndRewriteFileRanges( + files []*backuppb.File, + rewriteRules *RewriteRules, + splitSizeBytes, + splitKeyCount uint64, ) ([]rtree.Range, *MergeRangesStat, error) { if len(files) == 0 { return []rtree.Range{}, &MergeRangesStat{}, nil @@ -78,12 +86,20 @@ func MergeFileRanges( for _, f := range filesMap[key] { rangeSize += f.Size_ } - if out := rangeTree.InsertRange(rtree.Range{ + rg := &rtree.Range{ StartKey: files[0].GetStartKey(), EndKey: files[0].GetEndKey(), Files: files, Size: rangeSize, - }); out != nil { + } + // rewrite Range for split. + // so that splitRanges no need to handle rewrite rules any more. + tmpRng, err := RewriteRange(rg, rewriteRules) + if err != nil { + return nil, nil, errors.Annotatef(berrors.ErrRestoreInvalidRange, + "unable to rewrite range files %+v", files) + } + if out := rangeTree.InsertRange(*tmpRng); out != nil { return nil, nil, errors.Annotatef(berrors.ErrRestoreInvalidRange, "duplicate range %s files %+v", out, files) } @@ -107,3 +123,40 @@ func MergeFileRanges( MergedRegionBytesAvg: int(mergedRegionBytesAvg), }, nil } + +func RewriteRange(rg *rtree.Range, rewriteRules *RewriteRules) (*rtree.Range, error) { + if rewriteRules == nil { + return rg, nil + } + startID := tablecodec.DecodeTableID(rg.StartKey) + endID := tablecodec.DecodeTableID(rg.EndKey) + var rule *import_sstpb.RewriteRule + if startID != endID { + log.Warn("table id does not match", + logutil.Key("startKey", rg.StartKey), + logutil.Key("endKey", rg.EndKey), + zap.Int64("startID", startID), + zap.Int64("endID", endID)) + return nil, errors.Annotate(berrors.ErrRestoreTableIDMismatch, "table id mismatch") + } + rg.StartKey, rule = replacePrefix(rg.StartKey, rewriteRules) + if rule == nil { + log.Warn("cannot find rewrite rule", logutil.Key("key", rg.StartKey)) + } else { + log.Debug( + "rewrite start key", + logutil.Key("key", rg.StartKey), logutil.RewriteRule(rule)) + } + oldKey := rg.EndKey + rg.EndKey, rule = replacePrefix(rg.EndKey, rewriteRules) + if rule == nil { + log.Warn("cannot find rewrite rule", logutil.Key("key", rg.EndKey)) + } else { + log.Debug( + "rewrite end key", + logutil.Key("origin-key", oldKey), + logutil.Key("key", rg.EndKey), + logutil.RewriteRule(rule)) + } + return rg, nil +} diff --git a/br/pkg/restore/merge_test.go b/br/pkg/restore/merge_test.go index 404d187e1a59b..a9c185070e2a4 100644 --- a/br/pkg/restore/merge_test.go +++ b/br/pkg/restore/merge_test.go @@ -12,9 +12,11 @@ import ( "github.com/pingcap/errors" backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/tidb/br/pkg/conn" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/rtree" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" @@ -208,7 +210,7 @@ func TestMergeRanges(t *testing.T) { for _, f := range cs.files { files = append(files, fb.build(f[0], f[1], f[2], f[3], f[4])...) } - rngs, stat, err := restore.MergeFileRanges(files, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) + rngs, stat, err := restore.MergeAndRewriteFileRanges(files, nil, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) require.NoErrorf(t, err, "%+v", cs) require.Equalf(t, cs.stat.TotalRegions, stat.TotalRegions, "%+v", cs) require.Equalf(t, cs.stat.MergedRegions, stat.MergedRegions, "%+v", cs) @@ -230,8 +232,8 @@ func TestMergeRawKVRanges(t *testing.T) { files = append(files, fb.build(1, 0, 2, 1, 1)...) // RawKV does not have write cf files = files[1:] - _, stat, err := restore.MergeFileRanges( - files, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) + _, stat, err := restore.MergeAndRewriteFileRanges( + files, nil, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) require.NoError(t, err) require.Equal(t, 1, stat.TotalRegions) require.Equal(t, 1, stat.MergedRegions) @@ -243,8 +245,8 @@ func TestInvalidRanges(t *testing.T) { files = append(files, fb.build(1, 0, 1, 1, 1)...) files[0].Name = "invalid.sst" files[0].Cf = "invalid" - _, _, err := restore.MergeFileRanges( - files, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) + _, _, err := restore.MergeAndRewriteFileRanges( + files, nil, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) require.Error(t, err) require.Equal(t, berrors.ErrRestoreInvalidBackup, errors.Cause(err)) } @@ -265,7 +267,7 @@ func benchmarkMergeRanges(b *testing.B, filesCount int) { } var err error for i := 0; i < b.N; i++ { - _, _, err = restore.MergeFileRanges(files, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) + _, _, err = restore.MergeAndRewriteFileRanges(files, nil, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) if err != nil { b.Error(err) } @@ -291,3 +293,91 @@ func BenchmarkMergeRanges50k(b *testing.B) { func BenchmarkMergeRanges100k(b *testing.B) { benchmarkMergeRanges(b, 100000) } +func TestRewriteRange(t *testing.T) { + // Define test cases + cases := []struct { + rg *rtree.Range + rewriteRules *restore.RewriteRules + expectedRange *rtree.Range + expectedError error + }{ + // Test case 1: No rewrite rules + { + rg: &rtree.Range{ + StartKey: []byte("startKey"), + EndKey: []byte("endKey"), + }, + rewriteRules: nil, + expectedRange: &rtree.Range{StartKey: []byte("startKey"), EndKey: []byte("endKey")}, + expectedError: nil, + }, + // Test case 2: Rewrite rule found for both start key and end key + { + rg: &rtree.Range{ + StartKey: append(tablecodec.GenTableIndexPrefix(1), []byte("startKey")...), + EndKey: append(tablecodec.GenTableIndexPrefix(1), []byte("endKey")...), + }, + rewriteRules: &restore.RewriteRules{ + Data: []*import_sstpb.RewriteRule{ + { + OldKeyPrefix: tablecodec.GenTableIndexPrefix(1), + NewKeyPrefix: tablecodec.GenTableIndexPrefix(2), + }, + }, + }, + expectedRange: &rtree.Range{ + StartKey: append(tablecodec.GenTableIndexPrefix(2), []byte("startKey")...), + EndKey: append(tablecodec.GenTableIndexPrefix(2), []byte("endKey")...), + }, + expectedError: nil, + }, + // Test case 3: Rewrite rule found for end key + { + rg: &rtree.Range{ + StartKey: append(tablecodec.GenTableIndexPrefix(1), []byte("startKey")...), + EndKey: append(tablecodec.GenTableIndexPrefix(1), []byte("endKey")...), + }, + rewriteRules: &restore.RewriteRules{ + Data: []*import_sstpb.RewriteRule{ + { + OldKeyPrefix: append(tablecodec.GenTableIndexPrefix(1), []byte("endKey")...), + NewKeyPrefix: append(tablecodec.GenTableIndexPrefix(2), []byte("newEndKey")...), + }, + }, + }, + expectedRange: &rtree.Range{ + StartKey: append(tablecodec.GenTableIndexPrefix(1), []byte("startKey")...), + EndKey: append(tablecodec.GenTableIndexPrefix(2), []byte("newEndKey")...), + }, + expectedError: nil, + }, + // Test case 4: Table ID mismatch + { + rg: &rtree.Range{ + StartKey: []byte("t1_startKey"), + EndKey: []byte("t2_endKey"), + }, + rewriteRules: &restore.RewriteRules{ + Data: []*import_sstpb.RewriteRule{ + { + OldKeyPrefix: []byte("t1_startKey"), + NewKeyPrefix: []byte("t2_newStartKey"), + }, + }, + }, + expectedRange: nil, + expectedError: errors.Annotate(berrors.ErrRestoreTableIDMismatch, "table id mismatch"), + }, + } + + // Run test cases + for _, tc := range cases { + actualRange, actualError := restore.RewriteRange(tc.rg, tc.rewriteRules) + if tc.expectedError != nil { + require.EqualError(t, tc.expectedError, actualError.Error()) + } else { + require.NoError(t, actualError) + } + require.Equal(t, tc.expectedRange, actualRange) + } +} diff --git a/br/pkg/restore/pipeline_items.go b/br/pkg/restore/pipeline_items.go index 4eaea8cc98dbf..594e571925de0 100644 --- a/br/pkg/restore/pipeline_items.go +++ b/br/pkg/restore/pipeline_items.go @@ -161,6 +161,7 @@ func DefaultOutputTableChan() chan *CreatedTable { type TableWithRange struct { CreatedTable + // Range has been rewrited by rewrite rules. Range []rtree.Range } @@ -168,6 +169,10 @@ type TableIDWithFiles struct { TableID int64 Files []*backuppb.File + // RewriteRules is the rewrite rules for the specify table. + // because these rules belongs to the *one table*. + // we can hold them here. + RewriteRules *RewriteRules } // Exhaust drains all remaining errors in the channel, into a slice of errors. @@ -203,13 +208,11 @@ type TiKVRestorer interface { // After spliting, it also scatters the fresh regions. SplitRanges(ctx context.Context, ranges []rtree.Range, - rewriteRules *RewriteRules, updateCh glue.Progress, isRawKv bool) error // RestoreSSTFiles import the files to the TiKV. RestoreSSTFiles(ctx context.Context, tableIDWithFiles []TableIDWithFiles, - rewriteRules *RewriteRules, updateCh glue.Progress) error } @@ -351,7 +354,7 @@ func (b *tikvSender) splitWorker(ctx context.Context, // hence the checksum would fail. done := b.registerTableIsRestoring(result.TablesToSend) pool.ApplyOnErrorGroup(eg, func() error { - err := b.client.SplitRanges(ectx, result.Ranges, result.RewriteRules, b.updateCh, false) + err := b.client.SplitRanges(ectx, result.Ranges, b.updateCh, false) if err != nil { log.Error("failed on split range", rtree.ZapRanges(result.Ranges), zap.Error(err)) return err @@ -421,7 +424,7 @@ func (b *tikvSender) restoreWorker(ctx context.Context, ranges <-chan drainResul // There has been a worker in the `RestoreSSTFiles` procedure. // Spawning a raw goroutine won't make too many requests to TiKV. eg.Go(func() error { - e := b.client.RestoreSSTFiles(ectx, files, r.result.RewriteRules, b.updateCh) + e := b.client.RestoreSSTFiles(ectx, files, b.updateCh) if e != nil { log.Error("restore batch meet error", logutil.ShortError(e), zapTableIDWithFiles(files)) r.done() diff --git a/br/pkg/restore/range.go b/br/pkg/restore/range.go index 874398f1174e4..c36b1a82b7536 100644 --- a/br/pkg/restore/range.go +++ b/br/pkg/restore/range.go @@ -9,8 +9,6 @@ import ( berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/rtree" - "github.com/pingcap/tidb/pkg/tablecodec" - "go.uber.org/zap" ) // Range record start and end key for localStoreDir.DB @@ -21,41 +19,9 @@ type Range struct { } // SortRanges checks if the range overlapped and sort them. -func SortRanges(ranges []rtree.Range, rewriteRules *RewriteRules) ([]rtree.Range, error) { +func SortRanges(ranges []rtree.Range) ([]rtree.Range, error) { rangeTree := rtree.NewRangeTree() for _, rg := range ranges { - if rewriteRules != nil { - startID := tablecodec.DecodeTableID(rg.StartKey) - endID := tablecodec.DecodeTableID(rg.EndKey) - var rule *import_sstpb.RewriteRule - if startID != endID { - log.Warn("table id does not match", - logutil.Key("startKey", rg.StartKey), - logutil.Key("endKey", rg.EndKey), - zap.Int64("startID", startID), - zap.Int64("endID", endID)) - return nil, errors.Annotate(berrors.ErrRestoreTableIDMismatch, "table id mismatch") - } - rg.StartKey, rule = replacePrefix(rg.StartKey, rewriteRules) - if rule == nil { - log.Warn("cannot find rewrite rule", logutil.Key("key", rg.StartKey)) - } else { - log.Debug( - "rewrite start key", - logutil.Key("key", rg.StartKey), logutil.RewriteRule(rule)) - } - oldKey := rg.EndKey - rg.EndKey, rule = replacePrefix(rg.EndKey, rewriteRules) - if rule == nil { - log.Warn("cannot find rewrite rule", logutil.Key("key", rg.EndKey)) - } else { - log.Debug( - "rewrite end key", - logutil.Key("origin-key", oldKey), - logutil.Key("key", rg.EndKey), - logutil.RewriteRule(rule)) - } - } if out := rangeTree.InsertRange(rg); out != nil { log.Error("insert ranges overlapped", logutil.Key("startKeyOut", out.StartKey), @@ -81,6 +47,11 @@ func (r *RewriteRules) Append(other RewriteRules) { r.Data = append(r.Data, other.Data...) } +// EmptyRewriteRule make a map of new, empty rewrite rules. +func EmptyRewriteRulesMap() map[int64]*RewriteRules { + return make(map[int64]*RewriteRules) +} + // EmptyRewriteRule make a new, empty rewrite rule. func EmptyRewriteRule() *RewriteRules { return &RewriteRules{ diff --git a/br/pkg/restore/range_test.go b/br/pkg/restore/range_test.go index 322789ec023c1..2c84e2b7f0d72 100644 --- a/br/pkg/restore/range_test.go +++ b/br/pkg/restore/range_test.go @@ -33,7 +33,11 @@ func TestSortRange(t *testing.T) { EndKey: append(tablecodec.GenTableRecordPrefix(1), []byte("bbb")...), Files: nil, }, } - rs1, err := SortRanges(ranges1, rewriteRules) + for i, rg := range ranges1 { + tmp, _ := RewriteRange(&rg, rewriteRules) + ranges1[i] = *tmp + } + rs1, err := SortRanges(ranges1) require.NoErrorf(t, err, "sort range1 failed: %v", err) rangeEquals(t, rs1, []rtree.Range{ { @@ -48,13 +52,19 @@ func TestSortRange(t *testing.T) { EndKey: append(tablecodec.GenTableRecordPrefix(2), []byte("bbb")...), Files: nil, }, } - _, err = SortRanges(ranges2, rewriteRules) - require.Error(t, err) - require.Regexp(t, "table id mismatch.*", err.Error()) + for _, rg := range ranges2 { + _, err := RewriteRange(&rg, rewriteRules) + require.Error(t, err) + require.Regexp(t, "table id mismatch.*", err.Error()) + } ranges3 := initRanges() rewriteRules1 := initRewriteRules() - rs3, err := SortRanges(ranges3, rewriteRules1) + for i, rg := range ranges3 { + tmp, _ := RewriteRange(&rg, rewriteRules1) + ranges3[i] = *tmp + } + rs3, err := SortRanges(ranges3) require.NoErrorf(t, err, "sort range1 failed: %v", err) rangeEquals(t, rs3, []rtree.Range{ {StartKey: []byte("bbd"), EndKey: []byte("bbf"), Files: nil}, diff --git a/br/pkg/restore/rawkv_client.go b/br/pkg/restore/rawkv_client.go index 0b897a3ebca9b..22f0a9a86d62f 100644 --- a/br/pkg/restore/rawkv_client.go +++ b/br/pkg/restore/rawkv_client.go @@ -28,8 +28,7 @@ func NewRawkvClient(ctx context.Context, pdAddrs []string, security config.Secur ctx, pdAddrs, security, - pd.WithCustomTimeoutOption(10*time.Second), - pd.WithMaxErrorRetry(5)) + pd.WithCustomTimeoutOption(10*time.Second)) } type KVPair struct { diff --git a/br/pkg/restore/split.go b/br/pkg/restore/split.go index 43052e8ef6588..cbecddc251f7b 100644 --- a/br/pkg/restore/split.go +++ b/br/pkg/restore/split.go @@ -64,7 +64,6 @@ type OnSplitFunc func(key [][]byte) func (rs *RegionSplitter) ExecuteSplit( ctx context.Context, ranges []rtree.Range, - rewriteRules *RewriteRules, storeCount int, isRawKv bool, onSplit OnSplitFunc, @@ -82,7 +81,7 @@ func (rs *RegionSplitter) ExecuteSplit( // Sort the range for getting the min and max key of the ranges // TODO: this sort may not needed if we sort tables after creatation outside. - sortedRanges, errSplit := SortRanges(ranges, rewriteRules) + sortedRanges, errSplit := SortRanges(ranges) if errSplit != nil { return errors.Trace(errSplit) } @@ -269,41 +268,10 @@ func (rs *RegionSplitter) splitRegions( if err != nil { return nil, errors.Trace(err) } - rs.waitRegionsSplitted(ctx, newRegions) + _ = rs.client.WaitRegionsSplit(ctx, newRegions) return newRegions, nil } -// waitRegionsSplitted check multiple regions have finished the split. -func (rs *RegionSplitter) waitRegionsSplitted(ctx context.Context, splitRegions []*split.RegionInfo) { - // Wait for a while until the regions successfully split. - for _, region := range splitRegions { - rs.waitRegionSplitted(ctx, region.Region.Id) - } -} - -// waitRegionSplitted check single region has finished the split. -func (rs *RegionSplitter) waitRegionSplitted(ctx context.Context, regionID uint64) { - state := utils.InitialRetryState( - split.SplitCheckMaxRetryTimes, - split.SplitCheckInterval, - split.SplitMaxCheckInterval, - ) - err := utils.WithRetry(ctx, func() error { //nolint: errcheck - ok, err := rs.hasHealthyRegion(ctx, regionID) - if err != nil { - log.Warn("wait for split failed", zap.Uint64("regionID", regionID), zap.Error(err)) - return err - } - if ok { - return nil - } - return errors.Annotate(berrors.ErrPDSplitFailed, "wait region splitted failed") - }, &state) - if err != nil { - log.Warn("failed to split regions", logutil.ShortError(err)) - } -} - // waitRegionsScattered try to wait mutilple regions scatterd in 3 minutes. // this could timeout, but if many regions scatterd the restore could continue // so we don't wait long time here. @@ -323,101 +291,11 @@ func (rs *RegionSplitter) waitRegionsScattered(ctx context.Context, scatterRegio } } -// hasHealthyRegion is used to check whether region splitted success -func (rs *RegionSplitter) hasHealthyRegion(ctx context.Context, regionID uint64) (bool, error) { - regionInfo, err := rs.client.GetRegionByID(ctx, regionID) - if err != nil { - return false, errors.Trace(err) - } - // the region hasn't get ready. - if regionInfo == nil { - return false, nil - } - - // check whether the region is healthy and report. - // TODO: the log may be too verbose. we should use Prometheus metrics once it get ready for BR. - for _, peer := range regionInfo.PendingPeers { - log.Debug("unhealthy region detected", logutil.Peer(peer), zap.String("type", "pending")) - } - for _, peer := range regionInfo.DownPeers { - log.Debug("unhealthy region detected", logutil.Peer(peer), zap.String("type", "down")) - } - // we ignore down peers for they are (normally) hard to be fixed in reasonable time. - // (or once there is a peer down, we may get stuck at waiting region get ready.) - return len(regionInfo.PendingPeers) == 0, nil -} - func (rs *RegionSplitter) WaitForScatterRegionsTimeout(ctx context.Context, regionInfos []*split.RegionInfo, timeout time.Duration) int { - var ( - startTime = time.Now() - interval = split.ScatterWaitInterval - leftRegions = mapRegionInfoSlice(regionInfos) - retryCnt = 0 - - reScatterRegions = make([]*split.RegionInfo, 0, len(regionInfos)) - ) - for { - loggedLongRetry := false - reScatterRegions = reScatterRegions[:0] - for regionID, regionInfo := range leftRegions { - if retryCnt > 10 && !loggedLongRetry { - loggedLongRetry = true - resp, err := rs.client.GetOperator(ctx, regionID) - log.Info("retried many times to wait for scattering regions, checking operator", - zap.Int("retryCnt", retryCnt), - zap.Uint64("anyRegionID", regionID), - zap.Stringer("resp", resp), - zap.Error(err)) - } - ok, rescatter, err := rs.client.IsScatterRegionFinished(ctx, regionID) - if err != nil { - log.Warn("scatter region failed: do not have the region", - logutil.Region(regionInfo.Region), zap.Error(err)) - delete(leftRegions, regionID) - continue - } - if ok { - delete(leftRegions, regionID) - continue - } - if rescatter { - reScatterRegions = append(reScatterRegions, regionInfo) - } - // RUNNING_STATUS, just wait and check it in the next loop - } - - if len(leftRegions) == 0 { - return 0 - } - - if len(reScatterRegions) > 0 { - err2 := rs.client.ScatterRegions(ctx, reScatterRegions) - if err2 != nil { - log.Warn("failed to scatter regions", zap.Error(err2)) - } - } - - if time.Since(startTime) > timeout { - break - } - retryCnt += 1 - interval = 2 * interval - if interval > split.ScatterMaxWaitInterval { - interval = split.ScatterMaxWaitInterval - } - time.Sleep(interval) - } - - return len(leftRegions) -} - -func mapRegionInfoSlice(regionInfos []*split.RegionInfo) map[uint64]*split.RegionInfo { - regionInfoMap := make(map[uint64]*split.RegionInfo) - for _, info := range regionInfos { - regionID := info.Region.GetId() - regionInfoMap[regionID] = info - } - return regionInfoMap + ctx2, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + leftRegions, _ := rs.client.WaitRegionsScattered(ctx2, regionInfos) + return leftRegions } // TestGetSplitSortedKeysFromSortedRegionsTest is used only in unit test @@ -625,7 +503,7 @@ func (helper *LogSplitHelper) splitRegionByPoints( startKey = point } - return regionSplitter.ExecuteSplit(ctx, ranges, nil, 3, false, func([][]byte) {}) + return regionSplitter.ExecuteSplit(ctx, ranges, 3, false, func([][]byte) {}) } select { case <-ctx.Done(): diff --git a/br/pkg/restore/split/BUILD.bazel b/br/pkg/restore/split/BUILD.bazel index 2663c89bcd56c..92cfdfa253d98 100644 --- a/br/pkg/restore/split/BUILD.bazel +++ b/br/pkg/restore/split/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//br/pkg/errors", "//br/pkg/lightning/common", "//br/pkg/lightning/config", + "//br/pkg/lightning/log", "//br/pkg/logutil", "//br/pkg/redact", "//br/pkg/utils", @@ -49,10 +50,11 @@ go_test( ], embed = [":split"], flaky = True, - shard_count = 6, + shard_count = 8, deps = [ "//br/pkg/errors", "//br/pkg/utils", + "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/pdpb", diff --git a/br/pkg/restore/split/client.go b/br/pkg/restore/split/client.go index edfd18fd484c4..71a4256696f62 100644 --- a/br/pkg/restore/split/client.go +++ b/br/pkg/restore/split/client.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "crypto/tls" + "slices" "strconv" "strings" "sync" @@ -23,6 +24,7 @@ import ( berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/config" + brlog "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/utils" pd "github.com/tikv/pd/client" @@ -40,55 +42,61 @@ const ( splitRegionMaxRetryTime = 4 ) -// SplitClient is an external client used by RegionSplitter. -type SplitClient interface { - // GetStore gets a store by a store id. - GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) - // GetRegion gets a region which includes a specified key. - GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) - // GetRegionByID gets a region by a region id. - GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) - // SplitRegion splits a region from a key, if key is not included in the region, it will return nil. - // note: the key should not be encoded - SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) - // BatchSplitRegions splits a region from a batch of keys. - // note: the keys should not be encoded - BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error) - // BatchSplitRegionsWithOrigin splits a region from a batch of keys - // and return the original region and split new regions - BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *RegionInfo, - keys [][]byte) (*RegionInfo, []*RegionInfo, error) - // ScatterRegion scatters a specified region. - ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error - // ScatterRegions scatters regions in a batch. - ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error - // GetOperator gets the status of operator of the specified region. - GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) - // ScanRegions gets a list of regions, starts from the region that contains key. - // Limit limits the maximum number of regions returned. - ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) - // GetPlacementRule loads a placement rule from PD. - GetPlacementRule(ctx context.Context, groupID, ruleID string) (*pdhttp.Rule, error) - // SetPlacementRule insert or update a placement rule to PD. - SetPlacementRule(ctx context.Context, rule *pdhttp.Rule) error - // DeletePlacementRule removes a placement rule from PD. - DeletePlacementRule(ctx context.Context, groupID, ruleID string) error - // SetStoresLabel add or update specified label of stores. If labelValue - // is empty, it clears the label. - SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error - // IsScatterRegionFinished check the latest successful operator and return the - // follow status: - // - // return (finished, needRescatter, error) - // - // if the latest operator is not `scatter-operator`, or its status is SUCCESS, - // it's likely that the scatter region operator is finished. - // - // if the latest operator is `scatter-operator` and its status is TIMEOUT or - // CANCEL, the needRescatter is true and the function caller needs to scatter - // this region again. - IsScatterRegionFinished(ctx context.Context, regionID uint64) (scatterDone bool, needRescatter bool, scatterErr error) -} +type ( + // SplitClient is an external client used by RegionSplitter. + SplitClient interface { + // GetStore gets a store by a store id. + GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) + // GetRegion gets a region which includes a specified key. + GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) + // GetRegionByID gets a region by a region id. + GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) + // SplitRegion splits a region from a key, if key is not included in the region, it will return nil. + // note: the key should not be encoded + SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) + // BatchSplitRegions splits a region from a batch of keys. + // note: the keys should not be encoded + BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error) + // BatchSplitRegionsWithOrigin splits a region from a batch of keys + // and return the original region and split new regions + BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *RegionInfo, + keys [][]byte) (*RegionInfo, []*RegionInfo, error) + // WaitRegionsSplit waits for an already started split regions action to finish. + // It will check each region one by one internally, each region has a timeout of + // about 60s. + // + // To keep compatibility, it always checks all new regions even if error happens. + // In that case, it will return the last error. + WaitRegionsSplit(ctx context.Context, newRegions []*RegionInfo) error + // ScatterRegion scatters a specified region. + ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error + // ScatterRegions scatters regions in a batch. + ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error + // GetOperator gets the status of operator of the specified region. + GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) + // ScanRegions gets a list of regions, starts from the region that contains key. + // Limit limits the maximum number of regions returned. + ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) + // GetPlacementRule loads a placement rule from PD. + GetPlacementRule(ctx context.Context, groupID, ruleID string) (*pdhttp.Rule, error) + // SetPlacementRule insert or update a placement rule to PD. + SetPlacementRule(ctx context.Context, rule *pdhttp.Rule) error + // DeletePlacementRule removes a placement rule from PD. + DeletePlacementRule(ctx context.Context, groupID, ruleID string) error + // SetStoresLabel add or update specified label of stores. If labelValue + // is empty, it clears the label. + SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error + // WaitRegionsScattered waits for an already started scatter region action to + // finish. Internally it will backoff and retry at the maximum internal of 2 + // seconds. If the scatter makes progress during the retry, it will not decrease + // the retry counter. If there's always no progress, it will retry for about 1h. + // Caller can set the context timeout to control the max waiting time. + // + // The first return value is always the number of regions that are not finished + // scattering no matter what the error is. + WaitRegionsScattered(ctx context.Context, regionInfos []*RegionInfo) (notFinished int, err error) + } +) // pdClient is a wrapper of pd client, can be used by RegionSplitter. type pdClient struct { @@ -486,6 +494,59 @@ func (c *pdClient) BatchSplitRegions( return newRegions, err } +// WaitRegionsSplit implements SplitClient. +func (c *pdClient) WaitRegionsSplit(ctx context.Context, newRegions []*RegionInfo) error { + var lastErr error + for _, region := range newRegions { + regionID := region.Region.GetId() + state := utils.InitialRetryState( + SplitCheckMaxRetryTimes, + SplitCheckInterval, + SplitMaxCheckInterval, + ) + err := utils.WithRetry(ctx, func() error { //nolint: errcheck + ok, err := c.hasHealthyRegion(ctx, regionID) + if err != nil { + log.Warn("wait for split failed", zap.Uint64("regionID", regionID), zap.Error(err)) + return err + } + if ok { + return nil + } + return errors.Annotate(berrors.ErrPDSplitFailed, "wait region split failed") + }, &state) + // we only care about whether the last region splitted successfully. + // because we are waiting region report status *sequentially*. + if err != nil { + lastErr = err + } + } + return lastErr +} + +func (c *pdClient) hasHealthyRegion(ctx context.Context, regionID uint64) (bool, error) { + regionInfo, err := c.GetRegionByID(ctx, regionID) + if err != nil { + return false, errors.Trace(err) + } + // the region hasn't get ready. + if regionInfo == nil { + return false, nil + } + + // check whether the region is healthy and report. + // TODO: the log may be too verbose. we should use Prometheus metrics once it get ready for BR. + for _, peer := range regionInfo.PendingPeers { + log.Debug("unhealthy region detected", logutil.Peer(peer), zap.String("type", "pending")) + } + for _, peer := range regionInfo.DownPeers { + log.Debug("unhealthy region detected", logutil.Peer(peer), zap.String("type", "down")) + } + // we ignore down peers for they are (normally) hard to be fixed in reasonable time. + // (or once there is a peer down, we may get stuck at waiting region get ready.) + return len(regionInfo.PendingPeers) == 0, nil +} + func (c *pdClient) getStoreCount(ctx context.Context) (int, error) { stores, err := util.GetAllTiKVStores(ctx, c.client, util.SkipTiFlash) if err != nil { @@ -624,7 +685,7 @@ func (c *pdClient) scatterRegionsSequentially(ctx context.Context, newRegions [] } } -func (c *pdClient) IsScatterRegionFinished( +func (c *pdClient) isScatterRegionFinished( ctx context.Context, regionID uint64, ) (scatterDone bool, needRescatter bool, scatterErr error) { @@ -636,13 +697,105 @@ func (c *pdClient) IsScatterRegionFinished( } return false, false, errors.Trace(err) } - return IsScatterRegionFinished(resp) + return isScatterRegionFinished(resp) +} + +func (c *pdClient) WaitRegionsScattered(ctx context.Context, regions []*RegionInfo) (int, error) { + var ( + backoffer = NewBackoffMayNotCountBackoffer() + retryCnt = -1 + needRescatter = make([]*RegionInfo, 0, len(regions)) + needRecheck = make([]*RegionInfo, 0, len(regions)) + ) + + err := utils.WithRetryReturnLastErr(ctx, func() error { + retryCnt++ + loggedInThisRound := false + needRecheck = needRecheck[:0] + needRescatter = needRescatter[:0] + + for i, region := range regions { + regionID := region.Region.GetId() + + if retryCnt > 10 && !loggedInThisRound { + loggedInThisRound = true + resp, err := c.GetOperator(ctx, regionID) + brlog.FromContext(ctx).Info( + "retried many times to wait for scattering regions, checking operator", + zap.Int("retryCnt", retryCnt), + zap.Uint64("firstRegionID", regionID), + zap.Stringer("response", resp), + zap.Error(err), + ) + } + + ok, rescatter, err := c.isScatterRegionFinished(ctx, regionID) + if err != nil { + if !common.IsRetryableError(err) { + brlog.FromContext(ctx).Warn( + "wait for scatter region encountered non-retryable error", + logutil.Region(region.Region), + zap.Error(err), + ) + needRecheck = append(needRecheck, regions[i:]...) + return err + } + // if meet retryable error, recheck this region in next round + brlog.FromContext(ctx).Warn( + "wait for scatter region encountered error, will retry again", + logutil.Region(region.Region), + zap.Error(err), + ) + needRecheck = append(needRecheck, region) + continue + } + + if ok { + continue + } + // not finished scattered, check again in next round + needRecheck = append(needRecheck, region) + + if rescatter { + needRescatter = append(needRescatter, region) + } + } + + if len(needRecheck) == 0 { + return nil + } + + backoffErr := ErrBackoff + // if made progress in this round, don't increase the retryCnt + if len(needRecheck) < len(regions) { + backoffErr = ErrBackoffAndDontCount + } + + regions = slices.Clone(needRecheck) + + if len(needRescatter) > 0 { + scatterErr := c.ScatterRegions(ctx, needRescatter) + if scatterErr != nil { + if !common.IsRetryableError(scatterErr) { + return scatterErr + } + + return errors.Annotate(backoffErr, scatterErr.Error()) + } + } + return errors.Annotatef( + backoffErr, + "scatter region not finished, retryCnt: %d, needRecheck: %d, needRescatter: %d, the first unfinished region: %s", + retryCnt, len(needRecheck), len(needRescatter), needRecheck[0].Region.String(), + ) + }, backoffer) + + return len(needRecheck), err } -// IsScatterRegionFinished checks whether the scatter region operator is -// finished. TODO(lance6716): hide this function after scatter logic is unified -// for BR and lightning. -func IsScatterRegionFinished(resp *pdpb.GetOperatorResponse) ( +// isScatterRegionFinished checks whether the scatter region operator is +// finished. +func isScatterRegionFinished(resp *pdpb.GetOperatorResponse) ( scatterDone bool, needRescatter bool, scatterErr error, diff --git a/br/pkg/restore/split/split.go b/br/pkg/restore/split/split.go index 279fa6c63d68f..bbc79a5c44983 100644 --- a/br/pkg/restore/split/split.go +++ b/br/pkg/restore/split/split.go @@ -97,7 +97,7 @@ func PaginateScanRegion( var ( lastRegions []*RegionInfo err error - backoffer = NewWaitRegionOnlineBackoffer().(*WaitRegionOnlineBackoffer) + backoffer = NewWaitRegionOnlineBackoffer() ) _ = utils.WithRetry(ctx, func() error { regions := make([]*RegionInfo, 0, 16) @@ -208,7 +208,7 @@ type WaitRegionOnlineBackoffer struct { } // NewWaitRegionOnlineBackoffer create a backoff to wait region online. -func NewWaitRegionOnlineBackoffer() utils.Backoffer { +func NewWaitRegionOnlineBackoffer() *WaitRegionOnlineBackoffer { return &WaitRegionOnlineBackoffer{ Stat: utils.InitialRetryState( WaitRegionOnlineAttemptTimes, @@ -220,6 +220,7 @@ func NewWaitRegionOnlineBackoffer() utils.Backoffer { // NextBackoff returns a duration to wait before retrying again func (b *WaitRegionOnlineBackoffer) NextBackoff(err error) time.Duration { + // TODO(lance6716): why we only backoff when the error is ErrPDBatchScanRegion? if berrors.ErrPDBatchScanRegion.Equal(err) { // it needs more time to wait splitting the regions that contains data in PITR. // 2s * 150 @@ -231,7 +232,7 @@ func (b *WaitRegionOnlineBackoffer) NextBackoff(err error) time.Duration { }) return delayTime } - b.Stat.StopRetry() + b.Stat.GiveUp() return 0 } @@ -239,3 +240,48 @@ func (b *WaitRegionOnlineBackoffer) NextBackoff(err error) time.Duration { func (b *WaitRegionOnlineBackoffer) Attempt() int { return b.Stat.Attempt() } + +// BackoffMayNotCountBackoffer is a backoffer but it may not increase the retry +// counter. It should be used with ErrBackoff or ErrBackoffAndDontCount. +type BackoffMayNotCountBackoffer struct { + state utils.RetryState +} + +var ( + ErrBackoff = errors.New("found backoff error") + ErrBackoffAndDontCount = errors.New("found backoff error but don't count") +) + +// NewBackoffMayNotCountBackoffer creates a new backoffer that may backoff or retry. +// +// TODO: currently it has the same usage as NewWaitRegionOnlineBackoffer so we +// don't expose its inner settings. +func NewBackoffMayNotCountBackoffer() *BackoffMayNotCountBackoffer { + return &BackoffMayNotCountBackoffer{ + state: utils.InitialRetryState( + WaitRegionOnlineAttemptTimes, + time.Millisecond*10, + time.Second*2, + ), + } +} + +// NextBackoff implements utils.Backoffer. For BackoffMayNotCountBackoffer, only +// ErrBackoff and ErrBackoffAndDontCount is meaningful. +func (b *BackoffMayNotCountBackoffer) NextBackoff(err error) time.Duration { + if errors.ErrorEqual(err, ErrBackoff) { + return b.state.ExponentialBackoff() + } + if errors.ErrorEqual(err, ErrBackoffAndDontCount) { + delay := b.state.ExponentialBackoff() + b.state.ReduceRetry() + return delay + } + b.state.GiveUp() + return 0 +} + +// Attempt implements utils.Backoffer. +func (b *BackoffMayNotCountBackoffer) Attempt() int { + return b.state.Attempt() +} diff --git a/br/pkg/restore/split/split_test.go b/br/pkg/restore/split/split_test.go index 060f09b688632..fcd2b2ddeaaee 100644 --- a/br/pkg/restore/split/split_test.go +++ b/br/pkg/restore/split/split_test.go @@ -3,9 +3,11 @@ package split import ( "context" + goerrors "errors" "testing" "time" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" @@ -139,14 +141,15 @@ func TestScatterSequentiallyRetryCnt(t *testing.T) { type mockOldPDClient struct { pd.Client - scattered map[uint64]struct{} + scattered map[uint64]int + getOperatorResps map[uint64][]*pdpb.GetOperatorResponse } func (c *mockOldPDClient) ScatterRegion(_ context.Context, regionID uint64) error { if c.scattered == nil { - c.scattered = make(map[uint64]struct{}) + c.scattered = make(map[uint64]int) } - c.scattered[regionID] = struct{}{} + c.scattered[regionID]++ return nil } @@ -154,6 +157,12 @@ func (c *mockOldPDClient) ScatterRegions(context.Context, []uint64, ...pd.Region return nil, status.Error(codes.Unimplemented, "Ah, yep") } +func (c *mockOldPDClient) GetOperator(_ context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + ret := c.getOperatorResps[regionID][0] + c.getOperatorResps[regionID] = c.getOperatorResps[regionID][1:] + return ret, nil +} + func TestScatterBackwardCompatibility(t *testing.T) { client := pdClient{ needScatterVal: true, @@ -176,5 +185,152 @@ func TestScatterBackwardCompatibility(t *testing.T) { } err := client.ScatterRegions(ctx, regions) require.NoError(t, err) - require.Equal(t, map[uint64]struct{}{1: {}, 2: {}}, client.client.(*mockOldPDClient).scattered) + require.Equal(t, map[uint64]int{1: 1, 2: 1}, client.client.(*mockOldPDClient).scattered) +} + +func TestWaitForScatterRegions(t *testing.T) { + mockPDCli := &mockOldPDClient{} + client := pdClient{ + needScatterVal: true, + client: mockPDCli, + } + client.needScatterInit.Do(func() {}) + regionCnt := 6 + checkGetOperatorRespsDrained := func() { + for i := 1; i <= regionCnt; i++ { + require.Len(t, mockPDCli.getOperatorResps[uint64(i)], 0) + } + } + checkNoRetry := func() { + for i := 1; i <= regionCnt; i++ { + require.Equal(t, 0, mockPDCli.scattered[uint64(i)]) + } + } + + ctx := context.Background() + regions := make([]*RegionInfo, 0, regionCnt) + for i := 1; i <= regionCnt; i++ { + regions = append(regions, &RegionInfo{ + Region: &metapb.Region{ + Id: uint64(i), + }, + }) + } + + mockPDCli.scattered = make(map[uint64]int) + mockPDCli.getOperatorResps = make(map[uint64][]*pdpb.GetOperatorResponse) + mockPDCli.getOperatorResps[1] = []*pdpb.GetOperatorResponse{ + {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, + } + mockPDCli.getOperatorResps[2] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("not-scatter-region")}, + } + mockPDCli.getOperatorResps[3] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, + } + mockPDCli.getOperatorResps[4] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_TIMEOUT}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, + } + mockPDCli.getOperatorResps[5] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_CANCEL}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, + {Desc: []byte("not-scatter-region")}, + } + // should trigger a retry + mockPDCli.getOperatorResps[6] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_REPLACE}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, + } + + left, err := client.WaitRegionsScattered(ctx, regions) + require.NoError(t, err) + require.Equal(t, 0, left) + for i := 1; i <= 3; i++ { + require.Equal(t, 0, mockPDCli.scattered[uint64(i)]) + } + // OperatorStatus_TIMEOUT should trigger rescatter once + require.Equal(t, 1, mockPDCli.scattered[uint64(4)]) + // 3 * OperatorStatus_CANCEL should trigger 3 * rescatter + require.Equal(t, 3, mockPDCli.scattered[uint64(5)]) + // OperatorStatus_REPLACE should trigger rescatter once + require.Equal(t, 1, mockPDCli.scattered[uint64(6)]) + checkGetOperatorRespsDrained() + + // test non-retryable error + + mockPDCli.scattered = make(map[uint64]int) + mockPDCli.getOperatorResps = make(map[uint64][]*pdpb.GetOperatorResponse) + mockPDCli.getOperatorResps[1] = []*pdpb.GetOperatorResponse{ + {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, + } + mockPDCli.getOperatorResps[2] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("not-scatter-region")}, + } + // mimic non-retryable error + mockPDCli.getOperatorResps[3] = []*pdpb.GetOperatorResponse{ + {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_DATA_COMPACTED}}}, + } + left, err = client.WaitRegionsScattered(ctx, regions) + require.ErrorContains(t, err, "get operator error: DATA_COMPACTED") + require.Equal(t, 4, left) // region 3,4,5,6 is not scattered + checkGetOperatorRespsDrained() + checkNoRetry() + + // test backoff is timed-out + + backup := WaitRegionOnlineAttemptTimes + WaitRegionOnlineAttemptTimes = 2 + t.Cleanup(func() { + WaitRegionOnlineAttemptTimes = backup + }) + + mockPDCli.scattered = make(map[uint64]int) + mockPDCli.getOperatorResps = make(map[uint64][]*pdpb.GetOperatorResponse) + mockPDCli.getOperatorResps[1] = []*pdpb.GetOperatorResponse{ + {Header: &pdpb.ResponseHeader{Error: &pdpb.Error{Type: pdpb.ErrorType_REGION_NOT_FOUND}}}, + } + mockPDCli.getOperatorResps[2] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("not-scatter-region")}, + } + mockPDCli.getOperatorResps[3] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, + } + mockPDCli.getOperatorResps[4] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, // first retry + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_RUNNING}, // second retry + } + mockPDCli.getOperatorResps[5] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("not-scatter-region")}, + } + mockPDCli.getOperatorResps[6] = []*pdpb.GetOperatorResponse{ + {Desc: []byte("scatter-region"), Status: pdpb.OperatorStatus_SUCCESS}, + } + left, err = client.WaitRegionsScattered(ctx, regions) + require.ErrorContains(t, err, "the first unfinished region: id:4") + require.Equal(t, 1, left) + checkGetOperatorRespsDrained() + checkNoRetry() +} + +func TestBackoffMayNotCountBackoffer(t *testing.T) { + b := NewBackoffMayNotCountBackoffer() + initVal := b.Attempt() + + b.NextBackoff(ErrBackoffAndDontCount) + require.Equal(t, initVal, b.Attempt()) + // test Annotate, which is the real usage in caller + b.NextBackoff(errors.Annotate(ErrBackoffAndDontCount, "caller message")) + require.Equal(t, initVal, b.Attempt()) + + b.NextBackoff(ErrBackoff) + require.Equal(t, initVal-1, b.Attempt()) + + b.NextBackoff(goerrors.New("test")) + require.Equal(t, 0, b.Attempt()) } diff --git a/br/pkg/restore/split_test.go b/br/pkg/restore/split_test.go index a2a266bf51bdc..a6861bd25794d 100644 --- a/br/pkg/restore/split_test.go +++ b/br/pkg/restore/split_test.go @@ -35,13 +35,12 @@ import ( type TestClient struct { split.SplitClient - mu sync.RWMutex - stores map[uint64]*metapb.Store - regions map[uint64]*split.RegionInfo - regionsInfo *pdtypes.RegionTree // For now it's only used in ScanRegions - nextRegionID uint64 - injectInScatter func(*split.RegionInfo) error - injectInOperator func(uint64) (*pdpb.GetOperatorResponse, error) + mu sync.RWMutex + stores map[uint64]*metapb.Store + regions map[uint64]*split.RegionInfo + regionsInfo *pdtypes.RegionTree // For now it's only used in ScanRegions + nextRegionID uint64 + injectInScatter func(*split.RegionInfo) error scattered map[uint64]bool InjectErr bool @@ -203,14 +202,15 @@ func (c *TestClient) BatchSplitRegions( return newRegions, err } +func (c *TestClient) WaitRegionsSplit(context.Context, []*split.RegionInfo) error { + return nil +} + func (c *TestClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error { return c.injectInScatter(regionInfo) } -func (c *TestClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { - if c.injectInOperator != nil { - return c.injectInOperator(regionID) - } +func (c *TestClient) GetOperator(context.Context, uint64) (*pdpb.GetOperatorResponse, error) { return &pdpb.GetOperatorResponse{ Header: new(pdpb.ResponseHeader), }, nil @@ -236,12 +236,8 @@ func (c *TestClient) ScanRegions(ctx context.Context, key, endKey []byte, limit return regions, nil } -func (c *TestClient) IsScatterRegionFinished( - ctx context.Context, - regionID uint64, -) (scatterDone bool, needRescatter bool, scatterErr error) { - resp, _ := c.GetOperator(ctx, regionID) - return split.IsScatterRegionFinished(resp) +func (c *TestClient) WaitRegionsScattered(context.Context, []*split.RegionInfo) (int, error) { + return 0, nil } func TestScanEmptyRegion(t *testing.T) { @@ -249,11 +245,10 @@ func TestScanEmptyRegion(t *testing.T) { ranges := initRanges() // make ranges has only one ranges = ranges[0:1] - rewriteRules := initRewriteRules() regionSplitter := NewRegionSplitter(client) ctx := context.Background() - err := regionSplitter.ExecuteSplit(ctx, ranges, rewriteRules, 1, false, func(key [][]byte) {}) + err := regionSplitter.ExecuteSplit(ctx, ranges, 1, false, func(key [][]byte) {}) // should not return error with only one range entry require.NoError(t, err) } @@ -266,120 +261,18 @@ func TestScanEmptyRegion(t *testing.T) { // [, aay), [aay, bba), [bba, bbf), [bbf, bbh), [bbh, bbj), // [bbj, cca), [cca, xxe), [xxe, xxz), [xxz, ) func TestSplitAndScatter(t *testing.T) { - t.Run("BatchScatter", func(t *testing.T) { - client := initTestClient(false) - runTestSplitAndScatterWith(t, client) - }) - t.Run("WaitScatter", func(t *testing.T) { - client := initTestClient(false) - runWaitScatter(t, client) - }) -} - -// +------------+---------------------------- -// | region | states -// +------------+---------------------------- -// | [ , aay) | SUCCESS -// +------------+---------------------------- -// | [aay, bba) | CANCEL, SUCCESS -// +------------+---------------------------- -// | [bba, bbh) | RUNNING, TIMEOUT, SUCCESS -// +------------+---------------------------- -// | [bbh, cca) | -// +------------+---------------------------- -// | [cca, ) | CANCEL, RUNNING, SUCCESS -// +------------+---------------------------- -// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ) -// states: -func runWaitScatter(t *testing.T, client *TestClient) { - // configuration - type Operatorstates struct { - index int - status []pdpb.OperatorStatus - } - results := map[string]*Operatorstates{ - "": {status: []pdpb.OperatorStatus{pdpb.OperatorStatus_SUCCESS}}, - string(codec.EncodeBytesExt([]byte{}, []byte("aay"), false)): {status: []pdpb.OperatorStatus{pdpb.OperatorStatus_CANCEL, pdpb.OperatorStatus_SUCCESS}}, - string(codec.EncodeBytesExt([]byte{}, []byte("bba"), false)): {status: []pdpb.OperatorStatus{pdpb.OperatorStatus_RUNNING, pdpb.OperatorStatus_TIMEOUT, pdpb.OperatorStatus_SUCCESS}}, - string(codec.EncodeBytesExt([]byte{}, []byte("bbh"), false)): {}, - string(codec.EncodeBytesExt([]byte{}, []byte("cca"), false)): {status: []pdpb.OperatorStatus{pdpb.OperatorStatus_CANCEL, pdpb.OperatorStatus_RUNNING, pdpb.OperatorStatus_SUCCESS}}, - } - // after test done, the `leftScatterCount` should be empty - leftScatterCount := map[string]int{ - string(codec.EncodeBytesExt([]byte{}, []byte("aay"), false)): 1, - string(codec.EncodeBytesExt([]byte{}, []byte("bba"), false)): 1, - string(codec.EncodeBytesExt([]byte{}, []byte("cca"), false)): 1, - } - client.injectInScatter = func(ri *split.RegionInfo) error { - states, ok := results[string(ri.Region.StartKey)] - require.True(t, ok) - require.NotEqual(t, 0, len(states.status)) - require.NotEqual(t, pdpb.OperatorStatus_SUCCESS, states.status[states.index]) - states.index += 1 - cnt, ok := leftScatterCount[string(ri.Region.StartKey)] - require.True(t, ok) - if cnt == 1 { - delete(leftScatterCount, string(ri.Region.StartKey)) - } else { - leftScatterCount[string(ri.Region.StartKey)] = cnt - 1 - } - return nil - } - regionsMap := client.GetAllRegions() - leftOperatorCount := map[string]int{ - "": 1, - string(codec.EncodeBytesExt([]byte{}, []byte("aay"), false)): 2, - string(codec.EncodeBytesExt([]byte{}, []byte("bba"), false)): 3, - string(codec.EncodeBytesExt([]byte{}, []byte("bbh"), false)): 1, - string(codec.EncodeBytesExt([]byte{}, []byte("cca"), false)): 3, - } - client.injectInOperator = func(u uint64) (*pdpb.GetOperatorResponse, error) { - ri := regionsMap[u] - cnt, ok := leftOperatorCount[string(ri.Region.StartKey)] - require.True(t, ok) - if cnt == 1 { - delete(leftOperatorCount, string(ri.Region.StartKey)) - } else { - leftOperatorCount[string(ri.Region.StartKey)] = cnt - 1 - } - states, ok := results[string(ri.Region.StartKey)] - require.True(t, ok) - if len(states.status) == 0 { - return &pdpb.GetOperatorResponse{ - Desc: []byte("other"), - }, nil - } - if states.status[states.index] == pdpb.OperatorStatus_RUNNING { - states.index += 1 - return &pdpb.GetOperatorResponse{ - Desc: []byte("scatter-region"), - Status: states.status[states.index-1], - }, nil - } - return &pdpb.GetOperatorResponse{ - Desc: []byte("scatter-region"), - Status: states.status[states.index], - }, nil - } - - // begin to test - ctx := context.Background() - regions := make([]*split.RegionInfo, 0, len(regionsMap)) - for _, info := range regionsMap { - regions = append(regions, info) - } - regionSplitter := NewRegionSplitter(client) - leftCnt := regionSplitter.WaitForScatterRegionsTimeout(ctx, regions, 2000*time.Second) - require.Equal(t, leftCnt, 0) -} - -func runTestSplitAndScatterWith(t *testing.T, client *TestClient) { + client := initTestClient(false) ranges := initRanges() - rewriteRules := initRewriteRules() regionSplitter := NewRegionSplitter(client) - ctx := context.Background() - err := regionSplitter.ExecuteSplit(ctx, ranges, rewriteRules, 1, false, func(key [][]byte) {}) + + rules := initRewriteRules() + for i, rg := range ranges { + tmp, err := RewriteRange(&rg, rules) + require.NoError(t, err) + ranges[i] = *tmp + } + err := regionSplitter.ExecuteSplit(ctx, ranges, 1, false, func(key [][]byte) {}) require.NoError(t, err) regions := client.GetAllRegions() if !validateRegions(regions) { @@ -426,7 +319,7 @@ func TestRawSplit(t *testing.T) { ctx := context.Background() regionSplitter := NewRegionSplitter(client) - err := regionSplitter.ExecuteSplit(ctx, ranges, nil, 1, true, func(key [][]byte) {}) + err := regionSplitter.ExecuteSplit(ctx, ranges, 1, true, func(key [][]byte) {}) require.NoError(t, err) regions := client.GetAllRegions() expectedKeys := []string{"", "aay", "bba", "bbh", "cca", ""} @@ -627,7 +520,7 @@ type fakeRestorer struct { tableIDIsInsequence bool } -func (f *fakeRestorer) SplitRanges(ctx context.Context, ranges []rtree.Range, rewriteRules *RewriteRules, updateCh glue.Progress, isRawKv bool) error { +func (f *fakeRestorer) SplitRanges(ctx context.Context, ranges []rtree.Range, updateCh glue.Progress, isRawKv bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -644,7 +537,7 @@ func (f *fakeRestorer) SplitRanges(ctx context.Context, ranges []rtree.Range, re return nil } -func (f *fakeRestorer) RestoreSSTFiles(ctx context.Context, tableIDWithFiles []TableIDWithFiles, rewriteRules *RewriteRules, updateCh glue.Progress) error { +func (f *fakeRestorer) RestoreSSTFiles(ctx context.Context, tableIDWithFiles []TableIDWithFiles, updateCh glue.Progress) error { f.mu.Lock() defer f.mu.Unlock() @@ -922,6 +815,10 @@ func (f *fakeSplitClient) ScanRegions(ctx context.Context, startKey, endKey []by return result, nil } +func (f *fakeSplitClient) WaitRegionsScattered(context.Context, []*split.RegionInfo) (int, error) { + return 0, nil +} + func TestGetRewriteTableID(t *testing.T) { var tableID int64 = 76 var oldTableID int64 = 80 diff --git a/br/pkg/restore/util.go b/br/pkg/restore/util.go index d5dfd0ffddb3e..e8d0ad3cee8e4 100644 --- a/br/pkg/restore/util.go +++ b/br/pkg/restore/util.go @@ -366,8 +366,8 @@ func GoValidateFileRanges( } } // Merge small ranges to reduce split and scatter regions. - ranges, stat, err := MergeFileRanges( - files, splitSizeBytes, splitKeyCount) + ranges, stat, err := MergeAndRewriteFileRanges( + files, t.RewriteRule, splitSizeBytes, splitKeyCount) if err != nil { errCh <- err return @@ -507,7 +507,6 @@ func SplitRanges( ctx context.Context, client *Client, ranges []rtree.Range, - rewriteRules *RewriteRules, updateCh glue.Progress, isRawKv bool, ) error { @@ -518,7 +517,7 @@ func SplitRanges( isRawKv, )) - return splitter.ExecuteSplit(ctx, ranges, rewriteRules, client.GetStoreCount(), isRawKv, func(keys [][]byte) { + return splitter.ExecuteSplit(ctx, ranges, client.GetStoreCount(), isRawKv, func(keys [][]byte) { for range keys { updateCh.Inc() } diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index 19a5ebb69bcdd..36a2948e07927 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -68,7 +68,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 22, + shard_count = 25, deps = [ ":streamhelper", "//br/pkg/errors", @@ -89,8 +89,10 @@ go_test( "@com_github_pingcap_kvproto//pkg/logbackuppb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_log//:log", + "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//kv", + "@com_github_tikv_client_go_v2//oracle", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//txnkv/txnlock", diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go index 706035504f63e..0970ddd47de1b 100644 --- a/br/pkg/streamhelper/advancer.go +++ b/br/pkg/streamhelper/advancer.go @@ -71,6 +71,7 @@ type CheckpointAdvancer struct { lastCheckpoint *checkpoint lastCheckpointMu sync.Mutex inResolvingLock atomic.Bool + isPaused atomic.Bool checkpoints *spans.ValueSortedFull checkpointsMu sync.Mutex @@ -446,6 +447,14 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error log.Warn("failed to remove service GC safepoint", logutil.ShortError(err)) } metrics.LastCheckpoint.DeleteLabelValues(e.Name) + case EventPause: + if c.task.GetName() == e.Name { + c.isPaused.CompareAndSwap(false, true) + } + case EventResume: + if c.task.GetName() == e.Name { + c.isPaused.CompareAndSwap(true, false) + } case EventErr: return e.Err } @@ -544,6 +553,25 @@ func (c *CheckpointAdvancer) subscribeTick(ctx context.Context) error { return c.subscriber.PendingErrors() } +func (c *CheckpointAdvancer) isCheckpointLagged(ctx context.Context) (bool, error) { + if c.cfg.CheckPointLagLimit <= 0 { + return false, nil + } + + now, err := c.env.FetchCurrentTS(ctx) + if err != nil { + return false, err + } + + lagDuration := oracle.GetTimeFromTS(now).Sub(oracle.GetTimeFromTS(c.lastCheckpoint.TS)) + if lagDuration > c.cfg.CheckPointLagLimit { + log.Warn("checkpoint lag is too large", zap.String("category", "log backup advancer"), + zap.Stringer("lag", lagDuration)) + return true, nil + } + return false, nil +} + func (c *CheckpointAdvancer) importantTick(ctx context.Context) error { c.checkpointsMu.Lock() c.setCheckpoint(ctx, c.checkpoints.Min()) @@ -551,6 +579,17 @@ func (c *CheckpointAdvancer) importantTick(ctx context.Context) error { if err := c.env.UploadV3GlobalCheckpointForTask(ctx, c.task.Name, c.lastCheckpoint.TS); err != nil { return errors.Annotate(err, "failed to upload global checkpoint") } + isLagged, err := c.isCheckpointLagged(ctx) + if err != nil { + return errors.Annotate(err, "failed to check timestamp") + } + if isLagged { + err := c.env.PauseTask(ctx, c.task.Name) + if err != nil { + return errors.Annotate(err, "failed to pause task") + } + return errors.Annotate(errors.Errorf("check point lagged too large"), "check point lagged too large") + } p, err := c.env.BlockGCUntil(ctx, c.lastCheckpoint.safeTS()) if err != nil { return errors.Annotatef(err, @@ -606,7 +645,7 @@ func (c *CheckpointAdvancer) optionalTick(cx context.Context) error { func (c *CheckpointAdvancer) tick(ctx context.Context) error { c.taskMu.Lock() defer c.taskMu.Unlock() - if c.task == nil { + if c.task == nil || c.isPaused.Load() { log.Debug("No tasks yet, skipping advancing.") return nil } diff --git a/br/pkg/streamhelper/advancer_cliext.go b/br/pkg/streamhelper/advancer_cliext.go index d0593550d537e..3cff741bd9ea8 100644 --- a/br/pkg/streamhelper/advancer_cliext.go +++ b/br/pkg/streamhelper/advancer_cliext.go @@ -29,6 +29,8 @@ const ( EventAdd EventType = iota EventDel EventErr + EventPause + EventResume ) func (t EventType) String() string { @@ -39,6 +41,10 @@ func (t EventType) String() string { return "Del" case EventErr: return "Err" + case EventPause: + return "Pause" + case EventResume: + return "Resume" } return "Unknown" } @@ -70,29 +76,47 @@ func errorEvent(err error) TaskEvent { } func (t AdvancerExt) toTaskEvent(ctx context.Context, event *clientv3.Event) (TaskEvent, error) { - if !bytes.HasPrefix(event.Kv.Key, []byte(PrefixOfTask())) { - return TaskEvent{}, errors.Annotatef(berrors.ErrInvalidArgument, - "the path isn't a task path (%s)", string(event.Kv.Key)) + te := TaskEvent{} + var prefix string + + if bytes.HasPrefix(event.Kv.Key, []byte(PrefixOfTask())) { + prefix = PrefixOfTask() + te.Name = strings.TrimPrefix(string(event.Kv.Key), prefix) + } else if bytes.HasPrefix(event.Kv.Key, []byte(PrefixOfPause())) { + prefix = PrefixOfPause() + te.Name = strings.TrimPrefix(string(event.Kv.Key), prefix) + } else { + return TaskEvent{}, + errors.Annotatef(berrors.ErrInvalidArgument, "the path isn't a task/pause path (%s)", + string(event.Kv.Key)) } - te := TaskEvent{} - te.Name = strings.TrimPrefix(string(event.Kv.Key), PrefixOfTask()) - if event.Type == clientv3.EventTypeDelete { - te.Type = EventDel - } else if event.Type == clientv3.EventTypePut { + switch { + case event.Type == clientv3.EventTypePut && prefix == PrefixOfTask(): te.Type = EventAdd - } else { - return TaskEvent{}, errors.Annotatef(berrors.ErrInvalidArgument, "event type is wrong (%s)", event.Type) + case event.Type == clientv3.EventTypeDelete && prefix == PrefixOfTask(): + te.Type = EventDel + case event.Type == clientv3.EventTypePut && prefix == PrefixOfPause(): + te.Type = EventPause + case event.Type == clientv3.EventTypeDelete && prefix == PrefixOfPause(): + te.Type = EventResume + default: + return TaskEvent{}, + errors.Annotatef(berrors.ErrInvalidArgument, + "invalid event type or prefix: type=%s, prefix=%s", event.Type, prefix) } + te.Info = new(backuppb.StreamBackupTaskInfo) if err := proto.Unmarshal(event.Kv.Value, te.Info); err != nil { return TaskEvent{}, err } + var err error te.Ranges, err = t.MetaDataClient.TaskByInfo(*te.Info).Ranges(ctx) if err != nil { return TaskEvent{}, err } + return te, nil } @@ -113,7 +137,10 @@ func (t AdvancerExt) eventFromWatch(ctx context.Context, resp clientv3.WatchResp } func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskEvent) { - c := t.Client.Watcher.Watch(ctx, PrefixOfTask(), clientv3.WithPrefix(), clientv3.WithRev(rev)) + taskCh := t.Client.Watcher.Watch(ctx, PrefixOfTask(), clientv3.WithPrefix(), clientv3.WithRev(rev)) + pauseCh := t.Client.Watcher.Watch(ctx, PrefixOfPause(), clientv3.WithPrefix(), clientv3.WithRev(rev)) + + // inner function def handleResponse := func(resp clientv3.WatchResponse) bool { events, err := t.eventFromWatch(ctx, resp) if err != nil { @@ -127,21 +154,26 @@ func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskE } return true } + + // inner function def collectRemaining := func() { log.Info("Start collecting remaining events in the channel.", zap.String("category", "log backup advancer"), - zap.Int("remained", len(c))) + zap.Int("remained", len(taskCh))) defer log.Info("Finish collecting remaining events in the channel.", zap.String("category", "log backup advancer")) for { + if taskCh == nil && pauseCh == nil { + return + } + select { - case resp, ok := <-c: - if !ok { - return + case resp, ok := <-taskCh: + if !ok || !handleResponse(resp) { + taskCh = nil } - if !handleResponse(resp) { - return + case resp, ok := <-pauseCh: + if !ok || !handleResponse(resp) { + pauseCh = nil } - default: - return } } } @@ -150,7 +182,7 @@ func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskE defer close(ch) for { select { - case resp, ok := <-c: + case resp, ok := <-taskCh: failpoint.Inject("advancer_close_channel", func() { // We cannot really close the channel, just simulating it. ok = false @@ -162,6 +194,18 @@ func (t AdvancerExt) startListen(ctx context.Context, rev int64, ch chan<- TaskE if !handleResponse(resp) { return } + case resp, ok := <-pauseCh: + failpoint.Inject("advancer_close_pause_channel", func() { + // We cannot really close the channel, just simulating it. + ok = false + }) + if !ok { + ch <- errorEvent(io.EOF) + return + } + if !handleResponse(resp) { + return + } case <-ctx.Done(): collectRemaining() ch <- errorEvent(ctx.Err()) diff --git a/br/pkg/streamhelper/advancer_env.go b/br/pkg/streamhelper/advancer_env.go index 10aa14f2ee0aa..7092edb3686ea 100644 --- a/br/pkg/streamhelper/advancer_env.go +++ b/br/pkg/streamhelper/advancer_env.go @@ -10,6 +10,7 @@ import ( "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/util/engine" + "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/txnkv/txnlock" pd "github.com/tikv/pd/client" @@ -48,6 +49,11 @@ func (c PDRegionScanner) BlockGCUntil(ctx context.Context, at uint64) (uint64, e return c.UpdateServiceGCSafePoint(ctx, logBackupServiceID, int64(logBackupSafePointTTL.Seconds()), at) } +// TODO: It should be able to synchoronize the current TS with the PD. +func (c PDRegionScanner) FetchCurrentTS(ctx context.Context) (uint64, error) { + return oracle.ComposeTS(time.Now().UnixMilli(), 0), nil +} + // RegionScan gets a list of regions, starts from the region that contains key. // Limit limits the maximum number of regions returned. func (c PDRegionScanner) RegionScan(ctx context.Context, key, endKey []byte, limit int) ([]RegionWithLeader, error) { @@ -152,6 +158,7 @@ type StreamMeta interface { UploadV3GlobalCheckpointForTask(ctx context.Context, taskName string, checkpoint uint64) error // ClearV3GlobalCheckpointForTask clears the global checkpoint to the meta store. ClearV3GlobalCheckpointForTask(ctx context.Context, taskName string) error + PauseTask(ctx context.Context, taskName string) error } var _ tikv.RegionLockResolver = &AdvancerLockResolver{} diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go index a3beb6b034ff9..a98373bbcd88c 100644 --- a/br/pkg/streamhelper/advancer_test.go +++ b/br/pkg/streamhelper/advancer_test.go @@ -17,6 +17,7 @@ import ( "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/streamhelper/spans" "github.com/pingcap/tidb/pkg/kv" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/txnkv/txnlock" @@ -463,3 +464,80 @@ func TestRemoveTaskAndFlush(t *testing.T) { return !adv.HasSubscribion() }, 10*time.Second, 100*time.Millisecond) } + +func TestEnableCheckPointLimit(t *testing.T) { + c := createFakeCluster(t, 4, false) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.UpdateConfigWith(func(c *config.Config) { + c.CheckPointLagLimit = 1 * time.Minute + }) + adv.StartTaskListener(ctx) + for i := 0; i < 5; i++ { + c.advanceClusterTimeBy(30 * time.Second) + c.advanceCheckpointBy(20 * time.Second) + require.NoError(t, adv.OnTick(ctx)) + } +} + +func TestCheckPointLagged(t *testing.T) { + c := createFakeCluster(t, 4, false) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.UpdateConfigWith(func(c *config.Config) { + c.CheckPointLagLimit = 1 * time.Minute + }) + adv.StartTaskListener(ctx) + c.advanceClusterTimeBy(1 * time.Minute) + require.NoError(t, adv.OnTick(ctx)) + c.advanceClusterTimeBy(1 * time.Minute) + require.ErrorContains(t, adv.OnTick(ctx), "lagged too large") + // after some times, the isPaused will be set and ticks are skipped + require.Eventually(t, func() bool { + return assert.NoError(t, adv.OnTick(ctx)) + }, 5*time.Second, 100*time.Millisecond) +} + +func TestCheckPointResume(t *testing.T) { + c := createFakeCluster(t, 4, false) + defer func() { + fmt.Println(c) + }() + c.splitAndScatter("01", "02", "022", "023", "033", "04", "043") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := &testEnv{fakeCluster: c, testCtx: t} + adv := streamhelper.NewCheckpointAdvancer(env) + adv.UpdateConfigWith(func(c *config.Config) { + c.CheckPointLagLimit = 1 * time.Minute + }) + adv.StartTaskListener(ctx) + c.advanceClusterTimeBy(1 * time.Minute) + require.NoError(t, adv.OnTick(ctx)) + c.advanceClusterTimeBy(1 * time.Minute) + require.ErrorContains(t, adv.OnTick(ctx), "lagged too large") + require.Eventually(t, func() bool { + return assert.NoError(t, adv.OnTick(ctx)) + }, 5*time.Second, 100*time.Millisecond) + //now the checkpoint issue is fixed and resumed + c.advanceCheckpointBy(1 * time.Minute) + env.ResumeTask(ctx) + require.Eventually(t, func() bool { + return assert.NoError(t, adv.OnTick(ctx)) + }, 5*time.Second, 100*time.Millisecond) + //with time passed, the checkpoint will exceed the limit again + c.advanceClusterTimeBy(2 * time.Minute) + require.ErrorContains(t, adv.OnTick(ctx), "lagged too large") +} diff --git a/br/pkg/streamhelper/basic_lib_for_test.go b/br/pkg/streamhelper/basic_lib_for_test.go index ac5a5009b57e0..4d26ebdb14b62 100644 --- a/br/pkg/streamhelper/basic_lib_for_test.go +++ b/br/pkg/streamhelper/basic_lib_for_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/util/codec" + "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/tikvrpc" "github.com/tikv/client-go/v2/txnkv/txnlock" @@ -102,6 +103,7 @@ type fakeCluster struct { onGetClient func(uint64) error onClearCache func(uint64) error serviceGCSafePoint uint64 + currentTS uint64 } func (r *region) splitAt(newID uint64, k string) *region { @@ -273,6 +275,10 @@ func (f *fakeCluster) BlockGCUntil(ctx context.Context, at uint64) (uint64, erro return at, nil } +func (f *fakeCluster) FetchCurrentTS(ctx context.Context) (uint64, error) { + return f.currentTS, nil +} + // RegionScan gets a list of regions, starts from the region that contains key. // Limit limits the maximum number of regions returned. func (f *fakeCluster) RegionScan(ctx context.Context, key []byte, endKey []byte, limit int) ([]streamhelper.RegionWithLeader, error) { @@ -490,6 +496,29 @@ func (f *fakeCluster) advanceCheckpoints() uint64 { return minCheckpoint } +func (f *fakeCluster) advanceCheckpointBy(duration time.Duration) uint64 { + minCheckpoint := uint64(math.MaxUint64) + for _, r := range f.regions { + f.updateRegion(r.id, func(r *region) { + newCheckpointTime := oracle.GetTimeFromTS(r.checkpoint.Load()).Add(duration) + newCheckpoint := oracle.GoTimeToTS(newCheckpointTime) + r.checkpoint.Store(newCheckpoint) + if newCheckpoint < minCheckpoint { + minCheckpoint = newCheckpoint + } + r.fsim.flushedEpoch.Store(0) + }) + } + log.Info("checkpoint updated", zap.Uint64("to", minCheckpoint)) + return minCheckpoint +} + +func (f *fakeCluster) advanceClusterTimeBy(duration time.Duration) uint64 { + newTime := oracle.GoTimeToTS(oracle.GetTimeFromTS(f.currentTS).Add(duration)) + f.currentTS = newTime + return newTime +} + func createFakeCluster(t *testing.T, n int, simEnabled bool) *fakeCluster { c := &fakeCluster{ stores: map[uint64]*fakeStore{}, @@ -654,6 +683,22 @@ func (t *testEnv) ClearV3GlobalCheckpointForTask(ctx context.Context, taskName s return nil } +func (t *testEnv) PauseTask(ctx context.Context, taskName string) error { + t.taskCh <- streamhelper.TaskEvent{ + Type: streamhelper.EventPause, + Name: taskName, + } + return nil +} + +func (t *testEnv) ResumeTask(ctx context.Context) error { + t.taskCh <- streamhelper.TaskEvent{ + Type: streamhelper.EventResume, + Name: "whole", + } + return nil +} + func (t *testEnv) getCheckpoint() uint64 { t.mu.Lock() defer t.mu.Unlock() @@ -752,6 +797,10 @@ func (p *mockPDClient) GetStore(_ context.Context, storeID uint64) (*metapb.Stor }, nil } +func (p *mockPDClient) GetClusterID(ctx context.Context) uint64 { + return 1 +} + func newMockRegion(regionID uint64, startKey []byte, endKey []byte) *pd.Region { leader := &metapb.Peer{ Id: regionID, diff --git a/br/pkg/streamhelper/config/advancer_conf.go b/br/pkg/streamhelper/config/advancer_conf.go index 35fbb14f354f3..b8fd1a03569a5 100644 --- a/br/pkg/streamhelper/config/advancer_conf.go +++ b/br/pkg/streamhelper/config/advancer_conf.go @@ -17,6 +17,7 @@ const ( DefaultConsistencyCheckTick = 5 DefaultTryAdvanceThreshold = 4 * time.Minute + DefaultCheckPointLagLimit = 0 DefaultBackOffTime = 5 * time.Second DefaultTickInterval = 12 * time.Second DefaultFullScanTick = 4 @@ -34,6 +35,8 @@ type Config struct { TickDuration time.Duration `toml:"tick-interval" json:"tick-interval"` // The threshold for polling TiKV for checkpoint of some range. TryAdvanceThreshold time.Duration `toml:"try-advance-threshold" json:"try-advance-threshold"` + // The maximum lag could be tolerated for the checkpoint lag. + CheckPointLagLimit time.Duration `toml:"check-point-lag-limit" json:"check-point-lag-limit"` } func DefineFlagsForCheckpointAdvancerConfig(f *pflag.FlagSet) { @@ -50,6 +53,7 @@ func Default() Config { BackoffTime: DefaultBackOffTime, TickDuration: DefaultTickInterval, TryAdvanceThreshold: DefaultTryAdvanceThreshold, + CheckPointLagLimit: DefaultCheckPointLagLimit, } } @@ -76,6 +80,11 @@ func (conf Config) GetDefaultStartPollThreshold() time.Duration { return conf.TryAdvanceThreshold } +// GetCheckPointLagLimit returns the maximum lag could be tolerated for the checkpoint lag. +func (conf Config) GetCheckPointLagLimit() time.Duration { + return conf.CheckPointLagLimit +} + // GetSubscriberErrorStartPollThreshold returns the threshold of begin polling the checkpoint // when the subscriber meets error. func (conf Config) GetSubscriberErrorStartPollThreshold() time.Duration { diff --git a/br/pkg/streamhelper/models.go b/br/pkg/streamhelper/models.go index 60f2164afe2f3..63308ee6a6bfc 100644 --- a/br/pkg/streamhelper/models.go +++ b/br/pkg/streamhelper/models.go @@ -94,6 +94,12 @@ func Pause(task string) string { return path.Join(streamKeyPrefix, taskPausePath, task) } +// PrefixOfPause returns the prefix for pausing the task. +// Normally it would be /pause/ +func PrefixOfPause() string { + return path.Join(streamKeyPrefix, taskPausePath) + "/" +} + // LastErrorPrefixOf make the prefix for searching last error by some task. func LastErrorPrefixOf(task string) string { return strings.TrimSuffix(path.Join(streamKeyPrefix, taskLastErrorPath, task), "/") + "/" diff --git a/br/pkg/streamhelper/regioniter.go b/br/pkg/streamhelper/regioniter.go index 45079902032b7..af545fd3a36a4 100644 --- a/br/pkg/streamhelper/regioniter.go +++ b/br/pkg/streamhelper/regioniter.go @@ -42,6 +42,8 @@ type TiKVClusterMeta interface { // NOTE: once we support multi tasks, perhaps we need to allow the caller to provide a namespace. // For now, all tasks (exactly one task in fact) use the same checkpoint. BlockGCUntil(ctx context.Context, at uint64) (uint64, error) + + FetchCurrentTS(ctx context.Context) (uint64, error) } type Store struct { diff --git a/br/pkg/streamhelper/regioniter_test.go b/br/pkg/streamhelper/regioniter_test.go index f51e145664b6a..b205cd00b3de8 100644 --- a/br/pkg/streamhelper/regioniter_test.go +++ b/br/pkg/streamhelper/regioniter_test.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/br/pkg/logutil" @@ -16,6 +17,7 @@ import ( "github.com/pingcap/tidb/br/pkg/streamhelper/spans" "github.com/pingcap/tidb/pkg/kv" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -81,6 +83,11 @@ func (c constantRegions) BlockGCUntil(ctx context.Context, at uint64) (uint64, e return 0, status.Error(codes.Unimplemented, "Unsupported operation") } +// TODO: It should be able to synchoronize the current TS with the PD. +func (c constantRegions) FetchCurrentTS(ctx context.Context) (uint64, error) { + return oracle.ComposeTS(time.Now().UnixMilli(), 0), nil +} + func makeSubrangeRegions(keys ...string) constantRegions { if len(keys) == 0 { return nil diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index bc5ce4a860a34..b103562e427c4 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -131,7 +131,7 @@ func (cfg *RestoreCommonConfig) adjust() { if len(cfg.Granularity) == 0 { cfg.Granularity = string(restore.FineGrained) } - if cfg.ConcurrencyPerStore.Modified { + if !cfg.ConcurrencyPerStore.Modified { cfg.ConcurrencyPerStore.Value = conn.DefaultImportNumGoroutines } } @@ -770,7 +770,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo) - if err = client.InitBackupMeta(c, backupMeta, u, reader); err != nil { + if err = client.InitBackupMeta(c, backupMeta, u, reader, cfg.LoadStats); err != nil { return errors.Trace(err) } @@ -940,6 +940,12 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } } + // preallocate the table id, because any ddl job or database creation also allocates the global ID + err = client.AllocTableIDs(ctx, tables) + if err != nil { + return errors.Trace(err) + } + // execute DDL first err = client.ExecDDLs(ctx, ddlJobs) if err != nil { @@ -1070,10 +1076,8 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf ctx, postHandleCh, mgr.GetStorage().GetClient(), errCh, updateCh, cfg.ChecksumConcurrency) } - // pipeline load stats - if cfg.LoadStats { - postHandleCh = client.GoUpdateMetaAndLoadStats(ctx, s, &cfg.CipherInfo, postHandleCh, errCh, cfg.StatsConcurrency) - } + // pipeline update meta and load stats + postHandleCh = client.GoUpdateMetaAndLoadStats(ctx, s, &cfg.CipherInfo, postHandleCh, errCh, cfg.StatsConcurrency, cfg.LoadStats) // pipeline wait Tiflash synced if cfg.WaitTiflashReady { diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go index 0e4a89a99fe5e..5b9b009853a02 100644 --- a/br/pkg/task/restore_raw.go +++ b/br/pkg/task/restore_raw.go @@ -116,7 +116,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR return errors.Trace(err) } reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo) - if err = client.InitBackupMeta(c, backupMeta, u, reader); err != nil { + if err = client.InitBackupMeta(c, backupMeta, u, reader, true); err != nil { return errors.Trace(err) } @@ -137,8 +137,8 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR } summary.CollectInt("restore files", len(files)) - ranges, _, err := restore.MergeFileRanges( - files, kvConfigs.MergeRegionSize.Value, kvConfigs.MergeRegionKeyCount.Value) + ranges, _, err := restore.MergeAndRewriteFileRanges( + files, nil, kvConfigs.MergeRegionSize.Value, kvConfigs.MergeRegionKeyCount.Value) if err != nil { return errors.Trace(err) } @@ -153,7 +153,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR !cfg.LogProgress) // RawKV restore does not need to rewrite keys. - err = restore.SplitRanges(ctx, client, ranges, nil, updateCh, true) + err = restore.SplitRanges(ctx, client, ranges, updateCh, true) if err != nil { return errors.Trace(err) } diff --git a/br/pkg/task/restore_test.go b/br/pkg/task/restore_test.go index 5d48bcf0b832f..9301c1ea88f39 100644 --- a/br/pkg/task/restore_test.go +++ b/br/pkg/task/restore_test.go @@ -39,6 +39,10 @@ type mockPDClient struct { pd.Client } +func (m mockPDClient) GetClusterID(_ context.Context) uint64 { + return 1 +} + func (m mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) { return []*metapb.Store{}, nil } @@ -256,6 +260,7 @@ func mockReadSchemasFromBackupMeta(t *testing.T, db2Tables map[string][]string) &backuppb.CipherInfo{ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, }), + true, ) require.NoError(t, err) return dbs diff --git a/br/pkg/task/restore_txn.go b/br/pkg/task/restore_txn.go index 15086e7ca72b5..596b1d29d714e 100644 --- a/br/pkg/task/restore_txn.go +++ b/br/pkg/task/restore_txn.go @@ -60,7 +60,7 @@ func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config) return errors.Trace(err) } reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo) - if err = client.InitBackupMeta(c, backupMeta, u, reader); err != nil { + if err = client.InitBackupMeta(c, backupMeta, u, reader, true); err != nil { return errors.Trace(err) } @@ -78,8 +78,8 @@ func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config) } summary.CollectInt("restore files", len(files)) - ranges, _, err := restore.MergeFileRanges( - files, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) + ranges, _, err := restore.MergeAndRewriteFileRanges( + files, nil, conn.DefaultMergeRegionSizeBytes, conn.DefaultMergeRegionKeyCount) if err != nil { return errors.Trace(err) } @@ -93,7 +93,7 @@ func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config) !cfg.LogProgress) // RawKV restore does not need to rewrite keys. - err = restore.SplitRanges(ctx, client, ranges, nil, updateCh, false) + err = restore.SplitRanges(ctx, client, ranges, updateCh, false) if err != nil { return errors.Trace(err) } diff --git a/br/pkg/task/show/cmd.go b/br/pkg/task/show/cmd.go index 9358b64d8636b..88ff61843a4b3 100644 --- a/br/pkg/task/show/cmd.go +++ b/br/pkg/task/show/cmd.go @@ -93,7 +93,7 @@ func (exec *CmdExecutor) Read(ctx context.Context) (ShowResult, error) { out := make(chan *metautil.Table, 16) errc := make(chan error, 1) go func() { - errc <- exec.meta.ReadSchemasFiles(ctx, out, metautil.SkipFiles) + errc <- exec.meta.ReadSchemasFiles(ctx, out, metautil.SkipFiles, metautil.SkipStats) close(out) }() ts, err := collectResult(ctx, out, errc, convertTable) diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index b9db66c4e18cd..1294deed838d2 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -87,7 +87,7 @@ go_test( ], embed = [":utils"], flaky = True, - shard_count = 34, + shard_count = 35, deps = [ "//br/pkg/errors", "//pkg/kv", diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go index 658a4d965d886..02dc521deddbc 100644 --- a/br/pkg/utils/backoff.go +++ b/br/pkg/utils/backoff.go @@ -74,6 +74,15 @@ type RetryState struct { nextBackoff time.Duration } +// InitialRetryState make the initial state for retrying. +func InitialRetryState(maxRetryTimes int, initialBackoff, maxBackoff time.Duration) RetryState { + return RetryState{ + maxRetry: maxRetryTimes, + maxBackoff: maxBackoff, + nextBackoff: initialBackoff, + } +} + // Whether in the current state we can retry. func (rs *RetryState) ShouldRetry() bool { return rs.retryTimes < rs.maxRetry @@ -94,41 +103,17 @@ func (rs *RetryState) GiveUp() { rs.retryTimes = rs.maxRetry } -// InitialRetryState make the initial state for retrying. -func InitialRetryState(maxRetryTimes int, initialBackoff, maxBackoff time.Duration) RetryState { - return RetryState{ - maxRetry: maxRetryTimes, - maxBackoff: maxBackoff, - nextBackoff: initialBackoff, - } -} - -// RecordRetry simply record retry times, and no backoff -func (rs *RetryState) RecordRetry() { - rs.retryTimes++ -} - // ReduceRetry reduces retry times for 1. func (rs *RetryState) ReduceRetry() { rs.retryTimes-- } -// RetryTimes returns the retry times. -// usage: unit test. -func (rs *RetryState) RetryTimes() int { - return rs.retryTimes -} - // Attempt implements the `Backoffer`. // TODO: Maybe use this to replace the `exponentialBackoffer` (which is nearly homomorphic to this)? func (rs *RetryState) Attempt() int { return rs.maxRetry - rs.retryTimes } -func (rs *RetryState) StopRetry() { - rs.retryTimes = rs.maxRetry -} - // NextBackoff implements the `Backoffer`. func (rs *RetryState) NextBackoff(error) time.Duration { return rs.ExponentialBackoff() diff --git a/br/pkg/utils/backoff_test.go b/br/pkg/utils/backoff_test.go index c2a9247f4269d..8cf52435fdc9a 100644 --- a/br/pkg/utils/backoff_test.go +++ b/br/pkg/utils/backoff_test.go @@ -80,6 +80,28 @@ func TestBackoffWithFatalError(t *testing.T) { }, multierr.Errors(err)) } +func TestWithRetryReturnLastErr(t *testing.T) { + var counter int + backoffer := utils.NewBackoffer(10, time.Nanosecond, time.Nanosecond, utils.NewDefaultContext()) + gRPCError := status.Error(codes.Unavailable, "transport is closing") + err := utils.WithRetryReturnLastErr(context.Background(), func() error { + defer func() { counter++ }() + switch counter { + case 0: + return gRPCError // nolint:wrapcheck + case 1: + return berrors.ErrKVEpochNotMatch + case 2: + return berrors.ErrKVDownloadFailed + case 3: + return berrors.ErrKVRangeIsEmpty + } + return nil + }, backoffer) + require.Equal(t, 4, counter) + require.ErrorIs(t, berrors.ErrKVRangeIsEmpty, err) +} + func TestBackoffWithFatalRawGRPCError(t *testing.T) { var counter int canceledError := status.Error(codes.Canceled, "context canceled") diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go index d754070ff80d1..3f86d776762e6 100644 --- a/br/pkg/utils/retry.go +++ b/br/pkg/utils/retry.go @@ -245,6 +245,29 @@ func WithRetryV2[T any]( return *new(T), allErrors // nolint:wrapcheck } +// WithRetryReturnLastErr is like WithRetry but the returned error is the last +// error during retry rather than a multierr. +func WithRetryReturnLastErr( + ctx context.Context, + retryableFunc RetryableFunc, + backoffer Backoffer, +) error { + var lastErr error + for backoffer.Attempt() > 0 { + lastErr = retryableFunc() + if lastErr == nil { + return nil + } + select { + case <-ctx.Done(): + return lastErr + case <-time.After(backoffer.NextBackoff(lastErr)): + } + } + + return lastErr +} + // MessageIsRetryableStorageError checks whether the message returning from TiKV is retryable ExternalStorageError. func MessageIsRetryableStorageError(msg string) bool { msgLower := strings.ToLower(msg) diff --git a/br/pkg/version/version_test.go b/br/pkg/version/version_test.go index 657891f6b21a7..f84ffdc3ec804 100644 --- a/br/pkg/version/version_test.go +++ b/br/pkg/version/version_test.go @@ -23,6 +23,10 @@ type mockPDClient struct { getAllStores func() []*metapb.Store } +func (m *mockPDClient) GetClusterID(_ context.Context) uint64 { + return 1 +} + func (m *mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) { if m.getAllStores != nil { return m.getAllStores(), nil diff --git a/br/tests/lightning_character_sets/run.sh b/br/tests/lightning_character_sets/run.sh index 3049c53502b22..88059c6ed2495 100755 --- a/br/tests/lightning_character_sets/run.sh +++ b/br/tests/lightning_character_sets/run.sh @@ -80,6 +80,8 @@ check_contains 's: 5291' # test about unsupported charset in UTF-8 encoding dump files # test local backend run_lightning --config "$CUR/greek.toml" -d "$CUR/greek" 2>&1 | grep -q "Unknown character set: 'greek'" +# check TiDB does not receive the DDL +check_not_contains "greek" $TEST_DIR/tidb.log run_sql 'DROP DATABASE IF EXISTS charsets;' run_sql 'CREATE DATABASE charsets;' run_sql 'CREATE TABLE charsets.greek (c VARCHAR(20) PRIMARY KEY);' diff --git a/br/tests/lightning_checksum_mismatch/config1.toml b/br/tests/lightning_checksum_mismatch/config1.toml new file mode 100644 index 0000000000000..f1e1875a1d5ba --- /dev/null +++ b/br/tests/lightning_checksum_mismatch/config1.toml @@ -0,0 +1,9 @@ +[tikv-importer] +backend = "local" +duplicate-resolution = "" + +[post-restore] +checksum = "required" + +[mydumper.csv] +header = false diff --git a/br/tests/lightning_checksum_mismatch/config.toml b/br/tests/lightning_checksum_mismatch/config2.toml similarity index 100% rename from br/tests/lightning_checksum_mismatch/config.toml rename to br/tests/lightning_checksum_mismatch/config2.toml diff --git a/br/tests/lightning_checksum_mismatch/config3.toml b/br/tests/lightning_checksum_mismatch/config3.toml new file mode 100644 index 0000000000000..cdc1eea8f48da --- /dev/null +++ b/br/tests/lightning_checksum_mismatch/config3.toml @@ -0,0 +1,11 @@ +[tikv-importer] +backend = "local" + +[conflict] +strategy = "none" + +[post-restore] +checksum = "required" + +[mydumper.csv] +header = false diff --git a/br/tests/lightning_checksum_mismatch/run.sh b/br/tests/lightning_checksum_mismatch/run.sh index 184585cf77116..82a2500c398d4 100755 --- a/br/tests/lightning_checksum_mismatch/run.sh +++ b/br/tests/lightning_checksum_mismatch/run.sh @@ -2,4 +2,12 @@ set -eux -run_lightning 2>&1 | grep -q "checksum mismatched remote vs local" +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +run_lightning --config "$CUR/config1.toml" 2>&1 | grep -q "checksum mismatched remote vs local" + +run_sql 'DROP DATABASE IF EXISTS cm' +run_lightning --config "$CUR/config2.toml" 2>&1 | grep -q "checksum mismatched remote vs local" + +run_sql 'DROP DATABASE IF EXISTS cm' +run_lightning --config "$CUR/config3.toml" 2>&1 | grep -q "checksum mismatched remote vs local" diff --git a/br/tests/lightning_compress/local-config.toml b/br/tests/lightning_compress/local-config.toml new file mode 100644 index 0000000000000..2d7b557f25287 --- /dev/null +++ b/br/tests/lightning_compress/local-config.toml @@ -0,0 +1,18 @@ +[mydumper.csv] +separator = ',' +delimiter = '"' +header = true +not-null = false +null = '\N' +backslash-escape = true +trim-last-separator = false + +[conflict] +strategy = 'replace' +precheck-conflict-before-import = true + +[checkpoint] +enable = true +schema = "tidb_lightning_checkpoint_test" +driver = "mysql" +keep-after-success = true diff --git a/br/tests/lightning_compress/run.sh b/br/tests/lightning_compress/run.sh index 34e13578200c6..a8c9448b2ea9d 100755 --- a/br/tests/lightning_compress/run.sh +++ b/br/tests/lightning_compress/run.sh @@ -19,13 +19,13 @@ for BACKEND in tidb local; do run_sql 'DROP DATABASE IF EXISTS compress' run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test' set +e - run_lightning --backend $BACKEND -d "$CUR/data.$compress" --enable-checkpoint=1 2> /dev/null + run_lightning --backend $BACKEND --config "$CUR/$BACKEND-config.toml" -d "$CUR/data.$compress" --enable-checkpoint=1 2> /dev/null set -e # restart lightning from checkpoint, the second line should be written successfully export GO_FAILPOINTS= set +e - run_lightning --backend $BACKEND -d "$CUR/data.$compress" --enable-checkpoint=1 2> /dev/null + run_lightning --backend $BACKEND --config "$CUR/$BACKEND-config.toml" -d "$CUR/data.$compress" --enable-checkpoint=1 2> /dev/null set -e run_sql 'SELECT count(*), sum(PROCESSLIST_TIME), sum(THREAD_OS_ID), count(PROCESSLIST_STATE) FROM compress.threads' diff --git a/br/tests/lightning_compress/config.toml b/br/tests/lightning_compress/tidb-config.toml similarity index 100% rename from br/tests/lightning_compress/config.toml rename to br/tests/lightning_compress/tidb-config.toml diff --git a/br/tests/lightning_config_max_error/err_config.toml b/br/tests/lightning_config_max_error/err_config.toml index 4467bc582429f..a423b1e57e8f2 100644 --- a/br/tests/lightning_config_max_error/err_config.toml +++ b/br/tests/lightning_config_max_error/err_config.toml @@ -1,8 +1,6 @@ [conflict] threshold = 4 +strategy = "replace" [mydumper.csv] header = true - -[tikv-importer] -duplicate-resolution = 'replace' diff --git a/br/tests/lightning_config_max_error/ignore_config.toml b/br/tests/lightning_config_max_error/ignore_config.toml new file mode 100644 index 0000000000000..8cc3282bf9dea --- /dev/null +++ b/br/tests/lightning_config_max_error/ignore_config.toml @@ -0,0 +1,5 @@ +[conflict] +strategy = "ignore" + +[mydumper.csv] +header = true diff --git a/br/tests/lightning_config_max_error/normal_config.toml b/br/tests/lightning_config_max_error/normal_config.toml index 1092efdf0c733..8494704cafc77 100644 --- a/br/tests/lightning_config_max_error/normal_config.toml +++ b/br/tests/lightning_config_max_error/normal_config.toml @@ -1,8 +1,6 @@ [conflict] threshold = 20 +strategy = "replace" [mydumper.csv] header = true - -[tikv-importer] -duplicate-resolution = 'replace' diff --git a/br/tests/lightning_config_max_error/normal_config_old_style.toml b/br/tests/lightning_config_max_error/normal_config_old_style.toml index c3408f00a2144..051046f66dd42 100644 --- a/br/tests/lightning_config_max_error/normal_config_old_style.toml +++ b/br/tests/lightning_config_max_error/normal_config_old_style.toml @@ -4,5 +4,5 @@ max-error = 0 # this actually sets the type error [mydumper.csv] header = true -[tikv-importer] -duplicate-resolution = 'replace' +[conflict] +strategy = "replace" diff --git a/br/tests/lightning_config_max_error/run.sh b/br/tests/lightning_config_max_error/run.sh index 897dda57a95df..2adeaf93e08ff 100755 --- a/br/tests/lightning_config_max_error/run.sh +++ b/br/tests/lightning_config_max_error/run.sh @@ -81,6 +81,12 @@ check_contains "COUNT(*): ${duplicated_row_count}" run_sql 'SELECT COUNT(*) FROM mytest.testtbl' check_contains "COUNT(*): ${remaining_row_count}" +# import a fourth time +run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records' +! run_lightning --backend local --config "${mydir}/ignore_config.toml" +[ $? -eq 0 ] +tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Config:ErrInvalidConfig]conflict.strategy cannot be set to \\\"ignore\\\" when use tikv-importer.backend = \\\"local\\\"" + # Check tidb backend record duplicate entry in conflict_records table run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records' run_lightning --backend tidb --config "${mydir}/tidb.toml" diff --git a/br/tests/lightning_config_skip_csv_header/err_config.toml b/br/tests/lightning_config_skip_csv_header/err_config.toml index 0fb86faf9643c..9a37f4172a00f 100644 --- a/br/tests/lightning_config_skip_csv_header/err_config.toml +++ b/br/tests/lightning_config_skip_csv_header/err_config.toml @@ -5,5 +5,5 @@ check-requirements=true header = true header-schema-match = true -[tikv-importer] -duplicate-resolution = 'replace' +[conflict] +strategy = "replace" diff --git a/br/tests/lightning_config_skip_csv_header/err_default_config.toml b/br/tests/lightning_config_skip_csv_header/err_default_config.toml index 6df483fb9e152..9cf73857d7b8b 100644 --- a/br/tests/lightning_config_skip_csv_header/err_default_config.toml +++ b/br/tests/lightning_config_skip_csv_header/err_default_config.toml @@ -4,5 +4,5 @@ check-requirements=true [mydumper.csv] header = true -[tikv-importer] -duplicate-resolution = 'replace' +[conflict] +strategy = "replace" diff --git a/br/tests/lightning_config_skip_csv_header/normal_config.toml b/br/tests/lightning_config_skip_csv_header/normal_config.toml index 4572ebe0b2fd5..b5f1aab40c78b 100644 --- a/br/tests/lightning_config_skip_csv_header/normal_config.toml +++ b/br/tests/lightning_config_skip_csv_header/normal_config.toml @@ -5,5 +5,5 @@ check-requirements=true header = true header-schema-match = false -[tikv-importer] -duplicate-resolution = 'replace' +[conflict] +strategy = "replace" diff --git a/br/tests/lightning_distributed_import/config.toml b/br/tests/lightning_distributed_import/config.toml index 947b16037dd5d..7d48feeaec2a1 100644 --- a/br/tests/lightning_distributed_import/config.toml +++ b/br/tests/lightning_distributed_import/config.toml @@ -1,8 +1,10 @@ [tikv-importer] backend = 'local' -duplicate-resolution = 'none' incremental-import = true +[conflict] +strategy = "" + [post-restore] checksum = "required" diff --git a/br/tests/lightning_duplicate_detection/config1.toml b/br/tests/lightning_duplicate_detection/config1.toml index 09f3ed0528ea9..e433e92446644 100644 --- a/br/tests/lightning_duplicate_detection/config1.toml +++ b/br/tests/lightning_duplicate_detection/config1.toml @@ -5,9 +5,11 @@ table-concurrency = 10 [tikv-importer] backend = "local" -duplicate-resolution = 'replace' incremental-import = true +[conflict] +strategy = "replace" + [checkpoint] enable = true schema = "tidb_lightning_checkpoint1" diff --git a/br/tests/lightning_duplicate_detection/config2.toml b/br/tests/lightning_duplicate_detection/config2.toml index 17e840fd45250..bce104c01b73c 100644 --- a/br/tests/lightning_duplicate_detection/config2.toml +++ b/br/tests/lightning_duplicate_detection/config2.toml @@ -5,9 +5,11 @@ table-concurrency = 10 [tikv-importer] backend = "local" -duplicate-resolution = 'replace' incremental-import = true +[conflict] +strategy = "replace" + [checkpoint] enable = true schema = "tidb_lightning_checkpoint2" diff --git a/br/tests/lightning_duplicate_detection_new/local-error.toml b/br/tests/lightning_duplicate_detection_new/local-error.toml index 7bdfadf58bf81..41466925f592a 100644 --- a/br/tests/lightning_duplicate_detection_new/local-error.toml +++ b/br/tests/lightning_duplicate_detection_new/local-error.toml @@ -3,3 +3,6 @@ driver = "mysql" [tikv-importer] on-duplicate = "error" + +[conflict] +precheck-conflict-before-import = true diff --git a/br/tests/lightning_duplicate_detection_new/local-ignore.toml b/br/tests/lightning_duplicate_detection_new/local-ignore.toml deleted file mode 100644 index e15958aeb247e..0000000000000 --- a/br/tests/lightning_duplicate_detection_new/local-ignore.toml +++ /dev/null @@ -1,5 +0,0 @@ -[conflict] -max-record-rows = 1000 - -[tikv-importer] -on-duplicate = "ignore" diff --git a/br/tests/lightning_duplicate_detection_new/local-limit-error-records.toml b/br/tests/lightning_duplicate_detection_new/local-limit-error-records.toml index 02e491b568c0a..f91079069eb8b 100644 --- a/br/tests/lightning_duplicate_detection_new/local-limit-error-records.toml +++ b/br/tests/lightning_duplicate_detection_new/local-limit-error-records.toml @@ -1,4 +1,5 @@ [conflict] +precheck-conflict-before-import = true max-record-rows = 50 [tikv-importer] diff --git a/br/tests/lightning_duplicate_detection_new/local-replace.toml b/br/tests/lightning_duplicate_detection_new/local-replace.toml index 490e80158d014..45175ab79e7c6 100644 --- a/br/tests/lightning_duplicate_detection_new/local-replace.toml +++ b/br/tests/lightning_duplicate_detection_new/local-replace.toml @@ -1,5 +1,6 @@ [conflict] max-record-rows = 1000 +precheck-conflict-before-import = true [checkpoint] driver = "mysql" diff --git a/br/tests/lightning_duplicate_detection_new/run.sh b/br/tests/lightning_duplicate_detection_new/run.sh index 247dfd871c4c8..838b90d850953 100755 --- a/br/tests/lightning_duplicate_detection_new/run.sh +++ b/br/tests/lightning_duplicate_detection_new/run.sh @@ -57,17 +57,6 @@ run_lightning --backend tidb --config "$CUR/tidb-ignore.toml" --log-file "$LOG_F expected_rows=$(run_sql "SELECT count(*) AS total_count FROM test.dup_detect" | grep "total_count" | awk '{print $2}') expected_pks=$(run_sql "SELECT group_concat(col1 ORDER BY col1) AS pks FROM test.dup_detect" | grep "pks" | awk '{print $2}') -cleanup -run_lightning --backend local --config "$CUR/local-ignore.toml" --log-file "$LOG_FILE" -actual_rows=$(run_sql "SELECT count(*) AS total_count FROM test.dup_detect" | grep "total_count" | awk '{print $2}') -actual_pks=$(run_sql "SELECT group_concat(col1 ORDER BY col1) AS pks FROM test.dup_detect" | grep "pks" | awk '{print $2}') -if [ "$expected_rows" != "$actual_rows" ] || [ "$expected_pks" != "$actual_pks" ]; then - echo "local backend ignore strategy result is not equal to tidb backend" - exit 1 -fi -run_sql "SELECT count(*) FROM lightning_task_info.conflict_records" -check_contains "count(*): 228" - # 3. Test error strategy. cleanup run_lightning --backend local --config "$CUR/local-error.toml" --log-file "$LOG_FILE" 2>&1 | grep -q "duplicate key in table \`test\`.\`dup_detect\` caused by index .*, but because checkpoint is off we can't have more details" diff --git a/br/tests/lightning_duplicate_resolution_error/config.toml b/br/tests/lightning_duplicate_resolution_error/config.toml index b64c2df3e8618..c0a8e019ae69d 100644 --- a/br/tests/lightning_duplicate_resolution_error/config.toml +++ b/br/tests/lightning_duplicate_resolution_error/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'error' add-index-by-sql = false +[conflict] +strategy = "error" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_error/run.sh b/br/tests/lightning_duplicate_resolution_error/run.sh index 6584eb48ee70a..d6ae96f6b6b72 100644 --- a/br/tests/lightning_duplicate_resolution_error/run.sh +++ b/br/tests/lightning_duplicate_resolution_error/run.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2021 PingCAP, Inc. +# Copyright 2024 PingCAP, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,6 @@ run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v2' ! run_lightning --backend local --config "${mydir}/config.toml" [ $? -eq 0 ] -tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundDuplicateKey]found duplicate key" +tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a, primary key is '3', row data is '(3, 3, \\\"3.csv\\\")'" check_not_contains "the whole procedure completed" $TEST_DIR/lightning.log diff --git a/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/config.toml b/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/config.toml index 41ab032d9d6fa..7e4cc884b4b32 100644 --- a/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/config.toml +++ b/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'error' add-index-by-sql = false +[conflict] +strategy = "error" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh b/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh index 6584eb48ee70a..2eab9d2bbe77b 100644 --- a/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh +++ b/br/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2021 PingCAP, Inc. +# Copyright 2024 PingCAP, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,6 @@ run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v2' ! run_lightning --backend local --config "${mydir}/config.toml" [ $? -eq 0 ] -tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundDuplicateKey]found duplicate key" +tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a" check_not_contains "the whole procedure completed" $TEST_DIR/lightning.log diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/config.toml b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/config.toml index 41ab032d9d6fa..7e4cc884b4b32 100644 --- a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/config.toml +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'error' add-index-by-sql = false +[conflict] +strategy = "error" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.1.csv b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.1.csv index c46afaa19de4e..6e7a35d1a685f 100644 --- a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.1.csv +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.1.csv @@ -1,5 +1,5 @@ a,b,c -1,1,1.csv +1,101,1.csv 2,2,2.csv 3,3,3.csv 4,4,4.csv diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.2.csv b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.2.csv index 16fa5cf896a6a..786ce23c44791 100644 --- a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.2.csv +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/data/dup_resolve.a.2.csv @@ -1,5 +1,5 @@ a,b,c -6,1,1.csv +6,101,1.csv 7,7,7.csv 8,8,8.csv 9,9,9.csv diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh index 6584eb48ee70a..73cb0c301c7c6 100644 --- a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2021 PingCAP, Inc. +# Copyright 2024 PingCAP, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,6 @@ run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v2' ! run_lightning --backend local --config "${mydir}/config.toml" [ $? -eq 0 ] -tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundDuplicateKey]found duplicate key" +tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_b', unique key is '[101]', primary key is '1'" check_not_contains "the whole procedure completed" $TEST_DIR/lightning.log diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/config.toml b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/config.toml new file mode 100644 index 0000000000000..7e4cc884b4b32 --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/config.toml @@ -0,0 +1,19 @@ +[lightning] +task-info-schema-name = 'lightning_task_info' + +[tikv-importer] +backend = 'local' +add-index-by-sql = false + +[conflict] +strategy = "error" + +[checkpoint] +enable = false + +[mydumper] +batch-size = 2 +# ensure each file is its own engine to facilitate cross-engine detection. + +[mydumper.csv] +header = true diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve-schema-create.sql b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve-schema-create.sql new file mode 100644 index 0000000000000..f8d42367a3d4c --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve-schema-create.sql @@ -0,0 +1 @@ +create schema dup_resolve; diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a-schema.sql b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a-schema.sql new file mode 100644 index 0000000000000..c3cc966e2d793 --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a-schema.sql @@ -0,0 +1,7 @@ +create table a ( + a int primary key, + b int not null, + c text, + d int, + unique key key_bd(b,d) +); diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.1.csv b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.1.csv new file mode 100644 index 0000000000000..89fc791fff938 --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.1.csv @@ -0,0 +1,6 @@ +a,b,c,d +1,101,1.csv,9 +2,2,2.csv,0 +3,3,3.csv,0 +4,4,4.csv,0 +5,5,5.csv,0 diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.2.csv b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.2.csv new file mode 100644 index 0000000000000..1dea5aa0f5d36 --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/data/dup_resolve.a.2.csv @@ -0,0 +1,5 @@ +a,b,c,d +6,101,6.csv,9 +7,7,7.csv,0 +8,8,8.csv,0 +9,9,9.csv,0 diff --git a/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh new file mode 100644 index 0000000000000..ca9fe272272ee --- /dev/null +++ b/br/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright 2024 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +check_cluster_version 5 2 0 'duplicate detection' || exit 0 + +mydir=$(dirname "${BASH_SOURCE[0]}") + +run_sql 'DROP TABLE IF EXISTS dup_resolve.a' +run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v2' + +! run_lightning --backend local --config "${mydir}/config.toml" +[ $? -eq 0 ] + +tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_bd', unique key is '[101 9]', primary key is '1'" + +check_not_contains "the whole procedure completed" $TEST_DIR/lightning.log diff --git a/br/tests/lightning_duplicate_resolution_incremental/config1.toml b/br/tests/lightning_duplicate_resolution_incremental/config1.toml index ea6fdd300f320..7646319f6cc21 100644 --- a/br/tests/lightning_duplicate_resolution_incremental/config1.toml +++ b/br/tests/lightning_duplicate_resolution_incremental/config1.toml @@ -5,9 +5,11 @@ table-concurrency = 10 [tikv-importer] backend = "local" -duplicate-resolution = "replace" incremental-import = true +[conflict] +strategy = "replace" + [checkpoint] enable = true schema = "tidb_lightning_checkpoint_dupe_resolve_incremental1" diff --git a/br/tests/lightning_duplicate_resolution_replace_multiple_keys_clustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_multiple_keys_clustered_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_multiple_keys_clustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_multiple_keys_clustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_multiple_keys_nonclustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_multiple_keys_nonclustered_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_multiple_keys_nonclustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_multiple_keys_nonclustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_clustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_clustered_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_clustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_clustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_nonclustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_nonclustered_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_nonclustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_multiple_unique_keys_nonclustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_key/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_key/config.toml index 1617a992eb0d0..5bb5b00a4d240 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_key/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_key/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_clustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_clustered_pk/config.toml index babdb61ffe1ae..2707c4522be6c 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_clustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_clustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_nonclustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_nonclustered_pk/config.toml index babdb61ffe1ae..2707c4522be6c 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_nonclustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_key_multiple_conflicts_nonclustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_clustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_clustered_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_clustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_clustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_clustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_clustered_pk/config.toml index 92ff01703a15b..421ed184fb033 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_clustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_clustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_nonclustered_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_nonclustered_pk/config.toml index 92ff01703a15b..421ed184fb033 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_nonclustered_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_nonclustered_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_nonclustered_varchar_pk/config.toml b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_nonclustered_varchar_pk/config.toml index d49b2583e944c..1196c2686f47f 100644 --- a/br/tests/lightning_duplicate_resolution_replace_one_unique_key_nonclustered_varchar_pk/config.toml +++ b/br/tests/lightning_duplicate_resolution_replace_one_unique_key_nonclustered_varchar_pk/config.toml @@ -3,9 +3,11 @@ task-info-schema-name = 'lightning_task_info' [tikv-importer] backend = 'local' -duplicate-resolution = 'replace' add-index-by-sql = false +[conflict] +strategy = "replace" + [checkpoint] enable = false diff --git a/br/tests/lightning_foreign_key/local-config.toml b/br/tests/lightning_foreign_key/local-config.toml new file mode 100644 index 0000000000000..ca64ed04cad65 --- /dev/null +++ b/br/tests/lightning_foreign_key/local-config.toml @@ -0,0 +1,5 @@ +[tikv-importer] +on-duplicate = "error" + +[conflict] +precheck-conflict-before-import = true diff --git a/br/tests/lightning_foreign_key/run.sh b/br/tests/lightning_foreign_key/run.sh index 1c045b61f43be..70280d01f71ce 100755 --- a/br/tests/lightning_foreign_key/run.sh +++ b/br/tests/lightning_foreign_key/run.sh @@ -16,6 +16,8 @@ set -eu +mydir=$(dirname "${BASH_SOURCE[0]}") + run_sql 'DROP DATABASE IF EXISTS fk;' run_sql 'CREATE DATABASE IF NOT EXISTS fk;' # Create existing tables that import data will reference. @@ -24,7 +26,7 @@ run_sql 'CREATE TABLE fk.t2 (a BIGINT PRIMARY KEY);' for BACKEND in tidb local; do run_sql 'DROP TABLE IF EXISTS fk.t, fk.parent, fk.child;' - run_lightning --backend $BACKEND + run_lightning --backend $BACKEND --config "${mydir}/$BACKEND-config.toml" run_sql 'SELECT GROUP_CONCAT(a) FROM fk.t ORDER BY a;' check_contains '1,2,3,4,5' diff --git a/br/tests/lightning_foreign_key/config.toml b/br/tests/lightning_foreign_key/tidb-config.toml similarity index 100% rename from br/tests/lightning_foreign_key/config.toml rename to br/tests/lightning_foreign_key/tidb-config.toml diff --git a/br/tests/lightning_issue_40657/config.toml b/br/tests/lightning_issue_40657/config.toml index e5d268f5cd3ab..91075f53ade29 100644 --- a/br/tests/lightning_issue_40657/config.toml +++ b/br/tests/lightning_issue_40657/config.toml @@ -1,7 +1,9 @@ [tikv-importer] backend = "local" -duplicate-resolution = "replace" add-index-by-sql = false +[conflict] +strategy = "replace" + [mydumper.csv] header = true diff --git a/br/tests/run.sh b/br/tests/run.sh index f3ff0c5dcbd33..a70cd28ffa6f1 100755 --- a/br/tests/run.sh +++ b/br/tests/run.sh @@ -19,6 +19,7 @@ CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) export PATH="$PATH:$CUR/../../bin:$CUR/../bin:$CUR/_utils" export TEST_DIR=/tmp/backup_restore_test export COV_DIR="/tmp/group_cover" +mkdir -p $COV_DIR || true source $CUR/_utils/run_services # Create COV_DIR if not exists diff --git a/br/tests/run_group_lightning_tests.sh b/br/tests/run_group_lightning_tests.sh index abee364fe2d74..eea5b542b6f1a 100755 --- a/br/tests/run_group_lightning_tests.sh +++ b/br/tests/run_group_lightning_tests.sh @@ -22,7 +22,7 @@ declare -A groups groups=( ["G00"]='lightning_auto_random_default lightning_bom_file lightning_character_sets lightning_check_partial_imported lightning_checkpoint lightning_checkpoint_chunks lightning_checkpoint_columns lightning_checkpoint_dirty_tableid' ["G01"]='lightning_checkpoint_engines lightning_checkpoint_engines_order lightning_checkpoint_error_destroy lightning_checkpoint_parquet lightning_checkpoint_timestamp lightning_checksum_mismatch lightning_cmdline_override lightning_column_permutation lightning_common_handle lightning_compress lightning_concurrent-restore' - ["G02"]='lightning_config_max_error lightning_config_skip_csv_header lightning_csv lightning_default-columns lightning_disable_scheduler_by_key_range lightning_disk_quota lightning_distributed_import lightning_drop_other_tables_halfway lightning_duplicate_detection lightning_duplicate_detection_new lightning_duplicate_resolution_error lightning_duplicate_resolution_error_pk_multiple_files lightning_duplicate_resolution_error_uk_multiple_files lightning_duplicate_resolution_incremental' + ["G02"]='lightning_config_max_error lightning_config_skip_csv_header lightning_csv lightning_default-columns lightning_disable_scheduler_by_key_range lightning_disk_quota lightning_distributed_import lightning_drop_other_tables_halfway lightning_duplicate_detection lightning_duplicate_detection_new lightning_duplicate_resolution_error lightning_duplicate_resolution_error_pk_multiple_files lightning_duplicate_resolution_error_uk_multiple_files lightning_duplicate_resolution_error_uk_multiple_files_multicol_index lightning_duplicate_resolution_incremental' ["G03"]='lightning_duplicate_resolution_replace_multiple_keys_clustered_pk lightning_duplicate_resolution_replace_multiple_keys_nonclustered_pk lightning_duplicate_resolution_replace_multiple_unique_keys_clustered_pk lightning_duplicate_resolution_replace_multiple_unique_keys_nonclustered_pk lightning_duplicate_resolution_replace_one_key lightning_duplicate_resolution_replace_one_key_multiple_conflicts_clustered_pk lightning_duplicate_resolution_replace_one_key_multiple_conflicts_nonclustered_pk' ["G04"]='lightning_duplicate_resolution_replace_one_unique_key_clustered_pk lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_clustered_pk lightning_duplicate_resolution_replace_one_unique_key_multiple_conflicts_nonclustered_pk lightning_duplicate_resolution_replace_one_unique_key_nonclustered_varchar_pk lightning_error_summary lightning_examples lightning_exotic_filenames lightning_extend_routes' ["G05"]='lightning_fail_fast lightning_fail_fast_on_nonretry_err lightning_file_routing lightning_foreign_key lightning_gcs lightning_generated_columns lightning_ignore_columns lightning_import_compress lightning_incremental lightning_issue_282 lightning_issue_40657 lightning_issue_410 lightning_issue_519 lightning_local_backend lightning_max_incr' diff --git a/br/tidb-lightning.toml b/br/tidb-lightning.toml index 1f85163c6ae53..f23416a5f1928 100644 --- a/br/tidb-lightning.toml +++ b/br/tidb-lightning.toml @@ -89,27 +89,29 @@ driver = "file" # - origin. keep the checkpoints data unchanged. #keep-after-success = "remove" + +[conflict] +# Starting from v7.3.0, a new version of strategy is introduced to handle conflicting data. The default value is "". +# - "": TiDB Lightning does not detect or handle conflicting data. If the source file contains conflicting primary or unique key records, the subsequent step reports an error. In the logical import mode, the strategy is converted to "error" directly. +# - "error": When detecting conflicting primary or unique key records in the imported data, TiDB Lightning terminates the import and reports an error. +# - "replace": When encountering conflicting primary or unique key records, TiDB Lightning retains the latest data and overwrites the old data. +# The conflict data are recorded in the `lightning_task_info.conflict_error_v2` table and the `conflict_records` table of the target TiDB cluster. +# You can manually insert the correct records into the target table based on your business requirements. +# Note that the target TiKV must be v5.2.0 or later versions. +# - "ignore": When encountering conflicting primary or unique key records, TiDB Lightning retains the old data and ignores the new data. This option can only be used in the logical import mode. +strategy = "" +# Controls whether TiDB Lightning checks conflicts before the import. The default value is false, which means that TiDB Lightning only checks conflicts after the import. If it is set to true, TiDB Lightning checks conflicts both before and after the import. This parameter can be used only in the physical import mode (`tikv-importer.backend = "local"`). +# precheck-conflict-before-import = false +# Controls the maximum number of conflict errors that can be handled when strategy is "replace" or "ignore". You can set it only when strategy is "replace" or "ignore". The default value is 9223372036854775807, which means that almost all errors are tolerant. In physical import mode, this parameter takes effect only when `precheck-conflict-before-import` is enabled. +# threshold = 9223372036854775807 +# Controls the maximum number of records in the conflict_records table. The default value is 100. If the strategy is "ignore", the conflict records that are ignored are recorded; if the strategy is "replace", the conflict records that are overwritten are recorded. However, the "replace" strategy cannot record the conflict records in the logical import mode. In physical import mode, this parameter takes effect only when `precheck-conflict-before-import` is enabled. +# max-record-rows = 100 + [tikv-importer] # Delivery backend, can be "importer", "local" or "tidb". backend = "importer" # Address of tikv-importer when the backend is 'importer' addr = "127.0.0.1:8287" -# What to do on duplicated record (unique key conflict) when the backend is 'tidb'. Possible values are: -# - replace: replace the old record by the new record (i.e. insert rows using "REPLACE INTO") -# - ignore: keep the old record and ignore the new record (i.e. insert rows using "INSERT IGNORE INTO") -# - error: produce an error (i.e. insert rows using "INSERT INTO"), which will count towards the max-error limit. -#on-duplicate = "replace" -# Whether to detect and resolve duplicate records (unique key conflict) when the backend is 'local'. -# Current supports three resolution algorithms: -# - none: doesn't detect duplicate records, which has the best performance of the three algorithms, but probably leads to -# inconsistent data in the target TiDB. -# - record: only records duplicate records to `lightning_task_info.conflict_error_v2` table on the target TiDB. Note that this -# required the version of target TiKV version is no less than v5.2.0, otherwise it will fallback to 'none'. -# - remove: records all duplicate records like the 'record' algorithm and remove all duplicate records to ensure a consistent -# state in the target TiDB. -# - replace: records all duplicate records like the 'record' algorithm and remove some duplicate records and reserve the others -# that will not cause conflict to ensure a consistent state in the target TiDB. -#duplicate-resolution = 'none' # Maximum KV size of SST files produced in the 'local' backend. This should be the same as # the TiKV region size to avoid further region splitting. The default value is 96 MiB. #region-split-size = '96MiB' diff --git a/cmd/benchdb/main.go b/cmd/benchdb/main.go index 3c3bff47d4658..10438ac5726a8 100644 --- a/cmd/benchdb/main.go +++ b/cmd/benchdb/main.go @@ -56,7 +56,7 @@ var ( func main() { flag.Parse() flag.PrintDefaults() - err := logutil.InitLogger(logutil.NewLogConfig(*logLevel, logutil.DefaultLogFormat, "", logutil.EmptyFileLogConfig, false)) + err := logutil.InitLogger(logutil.NewLogConfig(*logLevel, logutil.DefaultLogFormat, "", "", logutil.EmptyFileLogConfig, false)) terror.MustNil(err) err = store.Register("tikv", driver.TiKVDriver{}) terror.MustNil(err) diff --git a/cmd/tidb-server/BUILD.bazel b/cmd/tidb-server/BUILD.bazel index 064df9b6b0117..778b62f58f789 100644 --- a/cmd/tidb-server/BUILD.bazel +++ b/cmd/tidb-server/BUILD.bazel @@ -48,6 +48,7 @@ go_library( "//pkg/util/memory", "//pkg/util/metricsutil", "//pkg/util/printer", + "//pkg/util/redact", "//pkg/util/sem", "//pkg/util/signal", "//pkg/util/stmtsummary/v2:stmtsummary", diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index b9b73976ecf35..5ce8c03bc2c3e 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -72,6 +72,7 @@ import ( "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/metricsutil" "github.com/pingcap/tidb/pkg/util/printer" + "github.com/pingcap/tidb/pkg/util/redact" "github.com/pingcap/tidb/pkg/util/sem" "github.com/pingcap/tidb/pkg/util/signal" stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" @@ -108,6 +109,7 @@ const ( nmLogLevel = "L" nmLogFile = "log-file" nmLogSlowQuery = "log-slow-query" + nmLogGeneral = "log-general" nmReportStatus = "report-status" nmStatusHost = "status-host" nmStatusPort = "status" @@ -121,6 +123,8 @@ const ( nmRepairList = "repair-list" nmTempDir = "temp-dir" + nmRedact = "redact" + nmProxyProtocolNetworks = "proxy-protocol-networks" nmProxyProtocolHeaderTimeout = "proxy-protocol-header-timeout" nmProxyProtocolFallbackable = "proxy-protocol-fallbackable" @@ -163,6 +167,7 @@ var ( logLevel *string logFile *string logSlowQuery *string + logGeneral *string // Status reportStatus *bool @@ -171,6 +176,9 @@ var ( metricsAddr *string metricsInterval *uint + // subcommand collect-log + redactFlag *bool + // PROXY Protocol proxyProtocolNetworks *string proxyProtocolHeaderTimeout *uint @@ -216,6 +224,7 @@ func initFlagSet() *flag.FlagSet { logLevel = fset.String(nmLogLevel, "info", "log level: info, debug, warn, error, fatal") logFile = fset.String(nmLogFile, "", "log file path") logSlowQuery = fset.String(nmLogSlowQuery, "", "slow query file path") + logGeneral = fset.String(nmLogGeneral, "", "general log file path") // Status reportStatus = flagBoolean(fset, nmReportStatus, true, "If enable status report HTTP service.") @@ -224,6 +233,9 @@ func initFlagSet() *flag.FlagSet { metricsAddr = fset.String(nmMetricsAddr, "", "prometheus pushgateway address, leaves it empty will disable prometheus push.") metricsInterval = fset.Uint(nmMetricsInterval, 15, "prometheus client push interval in second, set \"0\" to disable prometheus push.") + // subcommand collect-log + redactFlag = flagBoolean(fset, nmRedact, false, "remove sensitive words from marked tidb logs, if `./tidb-server --redact=xxx collect-log ` subcommand is used") + // PROXY Protocol proxyProtocolNetworks = fset.String(nmProxyProtocolNetworks, "", "proxy protocol networks allowed IP or *, empty mean disable proxy protocol support") proxyProtocolHeaderTimeout = fset.Uint(nmProxyProtocolHeaderTimeout, 5, "proxy protocol header read timeout, unit is second. (Deprecated: as proxy protocol using lazy mode, header read timeout no longer used)") @@ -250,6 +262,16 @@ func initFlagSet() *flag.FlagSet { func main() { fset := initFlagSet() + if args := fset.Args(); len(args) != 0 { + if args[0] == "collect-logs" && len(args) > 1 { + output := "-" + if len(args) > 2 { + output = args[2] + } + terror.MustNil(redact.DeRedactFile(*redactFlag, args[1], output)) + return + } + } config.InitializeConfig(*configPath, *configCheck, *configStrict, overrideConfig, fset) if *version { setVersions() @@ -580,6 +602,9 @@ func overrideConfig(cfg *config.Config, fset *flag.FlagSet) { if actualFlags[nmLogSlowQuery] { cfg.Log.SlowQueryFile = *logSlowQuery } + if actualFlags[nmLogGeneral] { + cfg.Log.GeneralLogFile = *logGeneral + } // Status if actualFlags[nmReportStatus] { diff --git a/errors.toml b/errors.toml index c7ddce74e0275..956a28a5dfe6b 100644 --- a/errors.toml +++ b/errors.toml @@ -566,11 +566,21 @@ error = ''' encode kv error in file %s at offset %d ''' +["Lightning:Restore:ErrFoundDataConflictRecords"] +error = ''' +found data conflict records in table %s, primary key is '%s', row data is '%s' +''' + ["Lightning:Restore:ErrFoundDuplicateKey"] error = ''' found duplicate key '%s', value '%s' ''' +["Lightning:Restore:ErrFoundIndexConflictRecords"] +error = ''' +found index conflict records in table %s, index name is '%s', unique key is '%s', primary key is '%s' +''' + ["Lightning:Restore:ErrInvalidMetaStatus"] error = ''' invalid meta status: '%s' diff --git a/go.mod b/go.mod index d3e5679749026..96a9b1b9668d8 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.21 require ( cloud.google.com/go/storage v1.36.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/BurntSushi/toml v1.3.2 github.com/DATA-DOG/go-sqlmock v1.5.0 @@ -15,7 +15,7 @@ require ( github.com/apache/skywalking-eyes v0.4.0 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/ashanbrown/makezero v1.1.1 - github.com/aws/aws-sdk-go v1.48.14 + github.com/aws/aws-sdk-go v1.50.0 github.com/bazelbuild/buildtools v0.0.0-20230926111657-7d855c59baeb github.com/bazelbuild/rules_go v0.42.1-0.20231101215950-df20c987afcb github.com/blacktear23/go-proxyprotocol v1.0.6 @@ -45,7 +45,7 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/go-sql-driver/mysql v1.7.1 github.com/gogo/protobuf v1.3.2 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 github.com/golang/snappy v0.0.4 github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa github.com/golangci/golangci-lint v1.56.2 @@ -81,18 +81,18 @@ require ( github.com/otiai10/copy v1.2.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d - github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb + github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c github.com/pingcap/fn v1.0.0 - github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f - github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 - github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 + github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 + github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d + github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc + github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.0 github.com/prometheus/common v0.48.0 - github.com/prometheus/prometheus v0.49.1 + github.com/prometheus/prometheus v0.50.1 github.com/robfig/cron/v3 v3.0.1 github.com/sasha-s/go-deadlock v0.3.1 github.com/scalalang2/golang-fifo v0.1.5 @@ -107,7 +107,7 @@ require ( github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tidwall/btree v1.7.0 - github.com/tikv/client-go/v2 v2.0.8-0.20240308052415-af4f9a9b6e41 + github.com/tikv/client-go/v2 v2.0.8-0.20240318065517-a9128e8200ab github.com/tikv/pd/client v0.0.0-20240229065730-92a31c12238e github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a github.com/twmb/murmur3 v1.1.6 @@ -130,7 +130,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 golang.org/x/net v0.22.0 - golang.org/x/oauth2 v0.17.0 + golang.org/x/oauth2 v0.18.0 golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/term v0.18.0 @@ -138,10 +138,10 @@ require ( golang.org/x/time v0.5.0 golang.org/x/tools v0.18.0 google.golang.org/api v0.162.0 - google.golang.org/grpc v1.62.0 + google.golang.org/grpc v1.62.1 gopkg.in/yaml.v2 v2.4.0 honnef.co/go/tools v0.4.7 - k8s.io/api v0.28.4 + k8s.io/api v0.28.6 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) @@ -153,7 +153,7 @@ require ( github.com/go-kit/log v0.2.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect @@ -167,9 +167,9 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect go.opentelemetry.io/otel/metric v1.22.0 // indirect gonum.org/v1/gonum v0.8.2 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect - k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect ) require ( @@ -178,9 +178,9 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/pubsub v1.36.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -262,11 +262,11 @@ require ( github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/procfs v0.13.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.6 // indirect @@ -296,18 +296,18 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect - golang.org/x/mod v0.15.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apimachinery v0.28.4 // indirect - k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/apimachinery v0.28.6 // indirect + k8s.io/klog/v2 v2.120.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 6d9c66b155935..601186f492a99 100644 --- a/go.sum +++ b/go.sum @@ -46,18 +46,18 @@ cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyT cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -115,13 +115,15 @@ github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvx github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s= -github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.50.0 h1:HBtrLeO+QyDKnc3t1+5DR1RxodOHCGr8ZcrHudpv7jI= +github.com/aws/aws-sdk-go v1.50.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bazelbuild/buildtools v0.0.0-20230926111657-7d855c59baeb h1:4k69c5E7Sa7jmNtv9itBHYA4Z5pfurInuRrtgohxZeA= github.com/bazelbuild/buildtools v0.0.0-20230926111657-7d855c59baeb/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= github.com/bazelbuild/rules_go v0.42.1-0.20231101215950-df20c987afcb h1:CPn7VHaV3czTgk4LdEO+Od5DyYb6HXLL5CUIPignRLE= github.com/bazelbuild/rules_go v0.42.1-0.20231101215950-df20c987afcb/go.mod h1:TFLfii8e49kTgn329knh1lsJFKdxyp/hKlWObY66xwY= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -291,7 +293,6 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -324,8 +325,8 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= @@ -361,8 +362,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -706,8 +707,8 @@ github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d/go.mod h1:p8QnkZn github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk= -github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f h1:FxA+NgsdHNOv+/hZGxUh8Gb3WuZqgqmxDwztEOiA1v4= +github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/fn v1.0.0 h1:CyA6AxcOZkQh52wIqYlAmaVmF6EvrcqFywP463pjA8g= @@ -715,18 +716,18 @@ github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f h1:2xvTjl4OrQY+XF38p8H7qVCXpaUYc5rLiYQhSd07aTI= -github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 h1:nU9wDeMsID8EWawRQVdmRYcNhUrlI4TKogZhXleG4QQ= +github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY= -github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc h1:sEp4lbExDfnMX8HXQyhZrhqo2/SgeFY5KOdo5akc8FM= -github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= +github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 h1:fVNBE06Rjec+EIHaYAKAHa/bIt5lnu3Zh9O6kV7ZAdg= +github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -755,10 +756,10 @@ github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/prometheus v0.49.1 h1:90mDvjrFnca2m+0qPSIDr3y7iHPTAagOAElz7j+HtGk= -github.com/prometheus/prometheus v0.49.1/go.mod h1:aDogiyqmv3aBIWDb5z5Sdcxuuf2BOfiJwOIm9JGpMnI= +github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= +github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= +github.com/prometheus/prometheus v0.50.1 h1:N2L+DYrxqPh4WZStU+o1p/gQlBaqFbcLBTjlp3vpdXw= +github.com/prometheus/prometheus v0.50.1/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -868,8 +869,8 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tikv/client-go/v2 v2.0.8-0.20240308052415-af4f9a9b6e41 h1:ZFmKEJLxY2rOzX6zCGydAMxIsGqjtcmim7vzHROxg38= -github.com/tikv/client-go/v2 v2.0.8-0.20240308052415-af4f9a9b6e41/go.mod h1:a7O+2b8r8V5fyR+eNzUIsCDkX8YC3clhNUpsI1kYrKM= +github.com/tikv/client-go/v2 v2.0.8-0.20240318065517-a9128e8200ab h1:uzd6N2FtE/9d9g0x6QS2O48+Waxy4qjlyMbwk/vSZRQ= +github.com/tikv/client-go/v2 v2.0.8-0.20240318065517-a9128e8200ab/go.mod h1:9s6+YbGt0kW+9qTFDXuc5TkIpwpEf038S1UCa3utsSQ= github.com/tikv/pd/client v0.0.0-20240229065730-92a31c12238e h1:kHXMmskVCNyH53u43I73Y5cmZ6yqqder/jGOiI7ylxs= github.com/tikv/pd/client v0.0.0-20240229065730-92a31c12238e/go.mod h1:Z/QAgOt29zvwBTd0H6pdx45VO6KRNc/O/DzGkVmSyZg= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= @@ -1071,8 +1072,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1133,8 +1134,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= -golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1199,7 +1200,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1402,10 +1402,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 h1:8eadJkXbwDEMNwcB5O0s5Y5eCfyuCLdvaiOIaGTrWmQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 h1:em/y72n4XlYRtayY/cVj6pnVzHa//BDA1BdoO+z9mdE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1423,8 +1423,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1440,8 +1440,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1474,6 +1474,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= @@ -1487,22 +1488,24 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.4.7 h1:9MDAWxMoSnB6QoSqiVr7P5mtkT9pOc1kSxchzPCnqJs= honnef.co/go/tools v0.4.7/go.mod h1:+rnGS1THNh8zMwnd2oVOTL9QF6vmfyG6ZXBULae2uc0= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc= -k8s.io/utils v0.0.0-20230711102312-30195339c3c7/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.28.6 h1:yy6u9CuIhmg55YvF/BavPBBXB+5QicB64njJXxVnzLo= +k8s.io/api v0.28.6/go.mod h1:AM6Ys6g9MY3dl/XNaNfg/GePI0FT7WBGu8efU/lirAo= +k8s.io/apimachinery v0.28.6 h1:RsTeR4z6S07srPg6XYrwXpTJVMXsjPXn0ODakMytSW0= +k8s.io/apimachinery v0.28.6/go.mod h1:QFNX/kCl/EMT2WTSz8k4WLCv2XnkOLMaL8GAVRMdpsA= +k8s.io/client-go v0.28.6 h1:Gge6ziyIdafRchfoBKcpaARuz7jfrK1R1azuwORIsQI= +k8s.io/client-go v0.28.6/go.mod h1:+nu0Yp21Oeo/cBCsprNVXB2BfJTV51lFfe5tXl2rUL8= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= diff --git a/pkg/bindinfo/BUILD.bazel b/pkg/bindinfo/BUILD.bazel index 488c4980439f3..0f56829622c43 100644 --- a/pkg/bindinfo/BUILD.bazel +++ b/pkg/bindinfo/BUILD.bazel @@ -14,6 +14,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/bindinfo", visibility = ["//visibility:public"], deps = [ + "//pkg/bindinfo/internal/logutil", "//pkg/bindinfo/norm", "//pkg/kv", "//pkg/metrics", @@ -31,13 +32,14 @@ go_library( "//pkg/util/chunk", "//pkg/util/hack", "//pkg/util/hint", + "//pkg/util/intest", "//pkg/util/kvcache", - "//pkg/util/logutil", "//pkg/util/mathutil", "//pkg/util/memory", "//pkg/util/parser", "//pkg/util/sqlexec", "//pkg/util/stmtsummary/v2:stmtsummary", + "//pkg/util/stringutil", "//pkg/util/table-filter", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", @@ -69,7 +71,6 @@ go_test( "//pkg/bindinfo/norm", "//pkg/config", "//pkg/domain", - "//pkg/metrics", "//pkg/parser", "//pkg/parser/ast", "//pkg/parser/auth", @@ -88,7 +89,6 @@ go_test( "//pkg/util/stmtsummary", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_failpoint//:failpoint", - "@com_github_prometheus_client_model//go", "@com_github_stretchr_testify//require", "@io_opencensus_go//stats/view", "@org_uber_go_goleak//:goleak", diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index abd345ca2b273..7e3211665daa5 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -17,21 +17,37 @@ package bindinfo import ( "errors" "sync" + "sync/atomic" + "time" - "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil" "github.com/pingcap/tidb/pkg/bindinfo/norm" + "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/hack" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/kvcache" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" + "github.com/pingcap/tidb/pkg/util/stringutil" "go.uber.org/zap" ) +// GetBindingReturnNil is only for test +var GetBindingReturnNil = stringutil.StringerStr("GetBindingReturnNil") + +// GetBindingReturnNilBool is only for test +var GetBindingReturnNilBool atomic.Bool + +// GetBindingReturnNilAlways is only for test +var GetBindingReturnNilAlways = stringutil.StringerStr("getBindingReturnNilAlways") + +// LoadBindingNothing is only for test +var LoadBindingNothing = stringutil.StringerStr("LoadBindingNothing") + // FuzzyBindingCache is based on BindingCache, and provide some more advanced features, like // fuzzy matching, loading binding if cache miss automatically (TODO). type FuzzyBindingCache interface { @@ -57,10 +73,10 @@ type fuzzyBindingCache struct { sql2FuzzyDigest map[string]string // sqlDigest --> fuzzyDigest // loadBindingFromStorageFunc is used to load binding from storage if cache miss. - loadBindingFromStorageFunc func(sqlDigest string) (Bindings, error) + loadBindingFromStorageFunc func(sctx sessionctx.Context, sqlDigest string) (Bindings, error) } -func newFuzzyBindingCache(loadBindingFromStorageFunc func(string) (Bindings, error)) FuzzyBindingCache { +func newFuzzyBindingCache(loadBindingFromStorageFunc func(sessionctx.Context, string) (Bindings, error)) FuzzyBindingCache { return &fuzzyBindingCache{ BindingCache: newBindCache(), fuzzy2SQLDigests: make(map[string][]string), @@ -69,15 +85,25 @@ func newFuzzyBindingCache(loadBindingFromStorageFunc func(string) (Bindings, err } } +func (fbc *fuzzyBindingCache) shouldMetric() bool { + return fbc.loadBindingFromStorageFunc != nil // only metric for GlobalBindingCache, whose loadBindingFromStorageFunc is not nil. +} + func (fbc *fuzzyBindingCache) FuzzyMatchingBinding(sctx sessionctx.Context, fuzzyDigest string, tableNames []*ast.TableName) (matchedBinding Binding, isMatched bool) { matchedBinding, isMatched, missingSQLDigest := fbc.getFromMemory(sctx, fuzzyDigest, tableNames) if len(missingSQLDigest) == 0 { + if fbc.shouldMetric() && isMatched { + metrics.BindingCacheHitCounter.Inc() + } return } + if fbc.shouldMetric() { + metrics.BindingCacheMissCounter.Inc() + } if fbc.loadBindingFromStorageFunc == nil { return } - fbc.loadFromStore(missingSQLDigest) // loadFromStore's SetBinding has a Mutex inside, so it's safe to call it without lock + fbc.loadFromStore(sctx, missingSQLDigest) // loadFromStore's SetBinding has a Mutex inside, so it's safe to call it without lock matchedBinding, isMatched, _ = fbc.getFromMemory(sctx, fuzzyDigest, tableNames) return } @@ -92,7 +118,18 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest leastWildcards := len(tableNames) + 1 enableFuzzyBinding := sctx.GetSessionVars().EnableFuzzyBinding for _, sqlDigest := range fbc.fuzzy2SQLDigests[fuzzyDigest] { - if bindings := bindingCache.GetBinding(sqlDigest); bindings != nil { + bindings := bindingCache.GetBinding(sqlDigest) + if intest.InTest { + if sctx.Value(GetBindingReturnNil) != nil { + if GetBindingReturnNilBool.CompareAndSwap(false, true) { + bindings = nil + } + } + if sctx.Value(GetBindingReturnNilAlways) != nil { + bindings = nil + } + } + if bindings != nil { for _, binding := range bindings { numWildcards, matched := fuzzyMatchBindingTableName(sctx.GetSessionVars().CurrentDB, tableNames, binding.TableNames) if matched && numWildcards > 0 && sctx != nil && !enableFuzzyBinding { @@ -112,11 +149,23 @@ func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest return matchedBinding, isMatched, missingSQLDigest } -func (fbc *fuzzyBindingCache) loadFromStore(missingSQLDigest []string) { +func (fbc *fuzzyBindingCache) loadFromStore(sctx sessionctx.Context, missingSQLDigest []string) { + if intest.InTest && sctx.Value(LoadBindingNothing) != nil { + return + } + defer func(start time.Time) { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("loading binding from storage takes " + time.Since(start).String())) + }(time.Now()) + for _, sqlDigest := range missingSQLDigest { - bindings, err := fbc.loadBindingFromStorageFunc(sqlDigest) + start := time.Now() + bindings, err := fbc.loadBindingFromStorageFunc(sctx, sqlDigest) if err != nil { - logutil.BgLogger().Warn("loadBindingFromStorageFunc binding failed", zap.String("sqlDigest", sqlDigest), zap.Error(err)) + logutil.BindLogger().Warn("failed to load binding from storage", + zap.String("sqlDigest", sqlDigest), + zap.Error(err), + zap.Duration("duration", time.Since(start)), + ) continue } // put binding into the cache @@ -128,7 +177,7 @@ func (fbc *fuzzyBindingCache) loadFromStore(missingSQLDigest []string) { // When the memory capacity of bing_cache is not enough, // there will be some memory-related errors in multiple places. // Only needs to be handled once. - logutil.BgLogger().Warn("BindHandle.Update", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("update binding cache error", zap.Error(err)) } } } @@ -317,9 +366,6 @@ func (c *bindingCache) delete(key bindingCacheKey) bool { // The return value is not read-only, but it shouldn't be changed in the caller functions. // The function is thread-safe. func (c *bindingCache) GetBinding(sqlDigest string) Bindings { - failpoint.Inject("get_binding_return_nil", func(_ failpoint.Value) { - failpoint.Return(nil) - }) c.lock.Lock() defer c.lock.Unlock() return c.get(bindingCacheKey(sqlDigest)) diff --git a/pkg/bindinfo/binding_match.go b/pkg/bindinfo/binding_match.go index 0ac7ae3ab0d44..82a2a7efe6183 100644 --- a/pkg/bindinfo/binding_match.go +++ b/pkg/bindinfo/binding_match.go @@ -86,9 +86,11 @@ func matchSQLBinding(sctx sessionctx.Context, stmtNode ast.StmtNode, info *Bindi if globalHandle == nil { return } - if binding, matched := globalHandle.MatchGlobalBinding(sctx, fuzzyDigest, tableNames); matched { + binding, matched = globalHandle.MatchGlobalBinding(sctx, fuzzyDigest, tableNames) + if matched { return binding, matched, metrics.ScopeGlobal } + return } diff --git a/pkg/bindinfo/capture.go b/pkg/bindinfo/capture.go index 8e30a295848c4..e1f7fc8dc3681 100644 --- a/pkg/bindinfo/capture.go +++ b/pkg/bindinfo/capture.go @@ -15,15 +15,14 @@ package bindinfo import ( - "context" "strconv" "strings" + "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" - "github.com/pingcap/tidb/pkg/util/logutil" utilparser "github.com/pingcap/tidb/pkg/util/parser" stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" tablefilter "github.com/pingcap/tidb/pkg/util/table-filter" @@ -95,7 +94,7 @@ func (h *globalBindingHandle) extractCaptureFilterFromStorage() (filter *capture // uses another background session. rows, _, err := execRows(sctx, `SELECT filter_type, filter_value FROM mysql.capture_plan_baselines_blacklist order by filter_type`) if err != nil { - logutil.BgLogger().Warn("failed to load mysql.capture_plan_baselines_blacklist", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("failed to load mysql.capture_plan_baselines_blacklist", zap.Error(err)) return err } for _, row := range rows { @@ -105,7 +104,7 @@ func (h *globalBindingHandle) extractCaptureFilterFromStorage() (filter *capture case "table": tfilter, valid := ParseCaptureTableFilter(valStr) if !valid { - logutil.BgLogger().Warn("capture table filter is invalid, ignore it", zap.String("category", "sql-bind"), zap.String("filter_value", valStr)) + logutil.BindLogger().Warn("capture table filter is invalid, ignore it", zap.String("filter_value", valStr)) continue } filter.tables = append(filter.tables, tfilter) @@ -114,18 +113,18 @@ func (h *globalBindingHandle) extractCaptureFilterFromStorage() (filter *capture case "frequency": f, err := strconv.ParseInt(valStr, 10, 64) if err != nil { - logutil.BgLogger().Warn("failed to parse frequency type value, ignore it", zap.String("category", "sql-bind"), zap.String("filter_value", valStr), zap.Error(err)) + logutil.BindLogger().Warn("failed to parse frequency type value, ignore it", zap.String("filter_value", valStr), zap.Error(err)) continue } if f < 1 { - logutil.BgLogger().Warn("frequency threshold is less than 1, ignore it", zap.String("category", "sql-bind"), zap.Int64("frequency", f)) + logutil.BindLogger().Warn("frequency threshold is less than 1, ignore it", zap.Int64("frequency", f)) continue } if f > filter.frequency { filter.frequency = f } default: - logutil.BgLogger().Warn("unknown capture filter type, ignore it", zap.String("category", "sql-bind"), zap.String("filter_type", filterTp)) + logutil.BindLogger().Warn("unknown capture filter type, ignore it", zap.String("filter_type", filterTp)) } } return nil @@ -142,7 +141,7 @@ func (h *globalBindingHandle) CaptureBaselines() { for _, bindableStmt := range bindableStmts { stmt, err := parser4Capture.ParseOneStmt(bindableStmt.Query, bindableStmt.Charset, bindableStmt.Collation) if err != nil { - logutil.BgLogger().Debug("parse SQL failed in baseline capture", zap.String("category", "sql-bind"), zap.String("SQL", bindableStmt.Query), zap.Error(err)) + logutil.BindLogger().Debug("parse SQL failed in baseline capture", zap.String("SQL", bindableStmt.Query), zap.Error(err)) continue } if insertStmt, ok := stmt.(*ast.InsertStmt); ok && insertStmt.Select == nil { @@ -174,7 +173,7 @@ func (h *globalBindingHandle) CaptureBaselines() { if r := h.getCache().GetBinding(digest.String()); HasAvailableBinding(r) { continue } - bindSQL := GenerateBindingSQL(context.TODO(), stmt, bindableStmt.PlanHint, true, dbName) + bindSQL := GenerateBindingSQL(stmt, bindableStmt.PlanHint, true, dbName) if bindSQL == "" { continue } @@ -197,7 +196,7 @@ func (h *globalBindingHandle) CaptureBaselines() { // We don't need to pass the `sctx` because the BindSQL has been validated already. err = h.CreateGlobalBinding(nil, binding) if err != nil { - logutil.BgLogger().Debug("create bind record failed in baseline capture", zap.String("category", "sql-bind"), zap.String("SQL", bindableStmt.Query), zap.Error(err)) + logutil.BindLogger().Debug("create bind record failed in baseline capture", zap.String("SQL", bindableStmt.Query), zap.Error(err)) } } } diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index 9441426af3e1d..ec3b39c2def3e 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -23,6 +23,9 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil" + "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/format" @@ -34,7 +37,6 @@ import ( driver "github.com/pingcap/tidb/pkg/types/parser_driver" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/hint" - "github.com/pingcap/tidb/pkg/util/logutil" utilparser "github.com/pingcap/tidb/pkg/util/parser" "go.uber.org/zap" "golang.org/x/sync/singleflight" @@ -209,6 +211,10 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) defer func() { h.setLastUpdateTime(lastUpdateTime) h.setCache(newCache) + + metrics.BindingCacheMemUsage.Set(float64(h.GetMemUsage())) + metrics.BindingCacheMemLimit.Set(float64(h.GetMemCapacity())) + metrics.BindingCacheNumBindings.Set(float64(h.Size())) }() for _, row := range rows { @@ -225,7 +231,7 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) } if err != nil { - logutil.BgLogger().Warn("failed to generate bind record from data row", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("failed to generate bind record from data row", zap.Error(err)) continue } @@ -237,7 +243,7 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) // When the memory capacity of bing_cache is not enough, // there will be some memory-related errors in multiple places. // Only needs to be handled once. - logutil.BgLogger().Warn("BindHandle.Update", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("BindHandle.Update", zap.Error(err)) } } else { newCache.RemoveBinding(sqlDigest) @@ -438,7 +444,7 @@ func (c *invalidBindingCache) reset() { func (h *globalBindingHandle) DropInvalidGlobalBinding() { defer func() { if err := h.LoadFromStorageToCache(false); err != nil { - logutil.BgLogger().Warn("drop invalid global binding error", zap.Error(err)) + logutil.BindLogger().Warn("drop invalid global binding error", zap.Error(err)) } }() @@ -446,7 +452,7 @@ func (h *globalBindingHandle) DropInvalidGlobalBinding() { h.invalidBindings.reset() for _, invalidBinding := range invalidBindings { if _, err := h.dropGlobalBinding(invalidBinding.SQLDigest); err != nil { - logutil.BgLogger().Debug("flush bind record failed", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Debug("flush bind record failed", zap.Error(err)) } } } @@ -544,7 +550,7 @@ func getHintsForSQL(sctx sessionctx.Context, sql string) (string, error) { } // GenerateBindingSQL generates binding sqls from stmt node and plan hints. -func GenerateBindingSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string, skipCheckIfHasParam bool, defaultDB string) string { +func GenerateBindingSQL(stmtNode ast.StmtNode, planHint string, skipCheckIfHasParam bool, defaultDB string) string { // If would be nil for very simple cases such as point get, we do not need to evolve for them. if planHint == "" { return "" @@ -584,7 +590,7 @@ func GenerateBindingSQL(ctx context.Context, stmtNode ast.StmtNode, planHint str restoreCtx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutCharset|format.RestoreNameBackQuotes, &withSb) restoreCtx.DefaultDB = defaultDB if err := n.With.Restore(restoreCtx); err != nil { - logutil.BgLogger().Debug("restore SQL failed", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Debug("restore SQL failed", zap.Error(err)) return "" } withEnd := withIdx + len(withSb.String()) @@ -606,7 +612,7 @@ func GenerateBindingSQL(ctx context.Context, stmtNode ast.StmtNode, planHint str bindSQL = bindSQL[insertIdx:] return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) } - logutil.Logger(ctx).Debug("unexpected statement type when generating bind SQL", zap.String("category", "sql-bind"), zap.Any("statement", stmtNode)) + logutil.BindLogger().Debug("unexpected statement type when generating bind SQL", zap.Any("statement", stmtNode)) return "" } @@ -685,27 +691,37 @@ func (h *globalBindingHandle) Stats(_ *variable.SessionVars) (map[string]any, er } // LoadBindingsFromStorageToCache loads global bindings from storage to the memory cache. -func (h *globalBindingHandle) LoadBindingsFromStorage(sqlDigest string) (Bindings, error) { +func (h *globalBindingHandle) LoadBindingsFromStorage(sctx sessionctx.Context, sqlDigest string) (Bindings, error) { if sqlDigest == "" { return nil, nil } - bindings, err, _ := h.syncBindingSingleflight.Do(sqlDigest, func() (any, error) { + timeout := time.Duration(sctx.GetSessionVars().LoadBindingTimeout) * time.Millisecond + resultChan := h.syncBindingSingleflight.DoChan(sqlDigest, func() (any, error) { return h.loadBindingsFromStorageInternal(sqlDigest) }) - if err != nil { - logutil.BgLogger().Warn("fail to LoadBindingsFromStorageToCache", zap.Error(err)) - return nil, err - } - if bindings == nil { - return nil, nil + select { + case result := <-resultChan: + if result.Err != nil { + return nil, result.Err + } + bindings := result.Val + if bindings == nil { + return nil, nil + } + return bindings.(Bindings), nil + case <-time.After(timeout): + return nil, errors.New("load bindings from storage timeout") } - return bindings.(Bindings), err } func (h *globalBindingHandle) loadBindingsFromStorageInternal(sqlDigest string) (any, error) { + failpoint.Inject("load_bindings_from_storage_internal_timeout", func() { + time.Sleep(time.Second) + }) var bindings Bindings + selectStmt := fmt.Sprintf("SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest FROM mysql.bind_info where sql_digest = '%s'", sqlDigest) err := h.callWithSCtx(false, func(sctx sessionctx.Context) error { - rows, _, err := execRows(sctx, "SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest FROM mysql.bind_info where sql_digest = ?", sqlDigest) + rows, _, err := execRows(sctx, selectStmt) if err != nil { return err } @@ -717,7 +733,7 @@ func (h *globalBindingHandle) loadBindingsFromStorageInternal(sqlDigest string) } _, binding, err := newBinding(sctx, row) if err != nil { - logutil.BgLogger().Warn("failed to generate bind record from data row", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("failed to generate bind record from data row", zap.Error(err)) continue } bindings = append(bindings, binding) diff --git a/pkg/bindinfo/global_handle_test.go b/pkg/bindinfo/global_handle_test.go index a7fa40acb5285..8eb4113ba0b5e 100644 --- a/pkg/bindinfo/global_handle_test.go +++ b/pkg/bindinfo/global_handle_test.go @@ -485,7 +485,6 @@ func TestGlobalBinding(t *testing.T) { _, fuzzyDigest = norm.NormalizeStmtForBinding(stmt, norm.WithFuzz(true)) _, matched = dom.BindHandle().MatchGlobalBinding(tk.Session(), fuzzyDigest, bindinfo.CollectTableNames(stmt)) require.False(t, matched) // dropped - bindHandle = bindinfo.NewGlobalBindingHandle(&mockSessionPool{tk.Session()}) err = bindHandle.LoadFromStorageToCache(true) require.NoError(t, err) diff --git a/pkg/bindinfo/internal/logutil/BUILD.bazel b/pkg/bindinfo/internal/logutil/BUILD.bazel new file mode 100644 index 0000000000000..dfbdf07486ed5 --- /dev/null +++ b/pkg/bindinfo/internal/logutil/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "logutil", + srcs = ["logutil.go"], + importpath = "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil", + visibility = ["//pkg/bindinfo:__subpackages__"], + deps = [ + "//pkg/util/logutil", + "@org_uber_go_zap//:zap", + ], +) diff --git a/pkg/bindinfo/internal/logutil/logutil.go b/pkg/bindinfo/internal/logutil/logutil.go new file mode 100644 index 0000000000000..fca251c9b3c55 --- /dev/null +++ b/pkg/bindinfo/internal/logutil/logutil.go @@ -0,0 +1,25 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "github.com/pingcap/tidb/pkg/util/logutil" + "go.uber.org/zap" +) + +// BindLogger with category "sql-bind" is used to log statistic related messages. +func BindLogger() *zap.Logger { + return logutil.BgLogger().With(zap.String("category", "sql-bind")) +} diff --git a/pkg/bindinfo/session_handle.go b/pkg/bindinfo/session_handle.go index c235b2d13fd63..45e19adc64173 100644 --- a/pkg/bindinfo/session_handle.go +++ b/pkg/bindinfo/session_handle.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/bindinfo/internal/logutil" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" @@ -28,7 +29,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/sessionstates" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/hack" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -69,7 +69,7 @@ func NewSessionBindingHandle() SessionBindingHandle { func (h *sessionBindingHandle) appendSessionBinding(sqlDigest string, meta Bindings) { err := h.ch.SetBinding(sqlDigest, meta) if err != nil { - logutil.BgLogger().Warn("SessionHandle.appendSessionBinding", zap.String("category", "sql-bind"), zap.Error(err)) + logutil.BindLogger().Warn("SessionHandle.appendSessionBinding", zap.Error(err)) } } @@ -100,7 +100,8 @@ func (h *sessionBindingHandle) DropSessionBinding(sqlDigest string) error { // MatchSessionBinding returns the matched binding for this statement. func (h *sessionBindingHandle) MatchSessionBinding(sctx sessionctx.Context, fuzzyDigest string, tableNames []*ast.TableName) (matchedBinding Binding, isMatched bool) { - return h.ch.FuzzyMatchingBinding(sctx, fuzzyDigest, tableNames) + matchedBinding, isMatched = h.ch.FuzzyMatchingBinding(sctx, fuzzyDigest, tableNames) + return } // GetAllSessionBindings return all session bind info. @@ -127,18 +128,58 @@ func (h *sessionBindingHandle) DecodeSessionStates(_ context.Context, sctx sessi if len(sessionStates.Bindings) == 0 { return nil } + + var m []map[string]any + var err error + bindingBytes := hack.Slice(sessionStates.Bindings) + if err = json.Unmarshal(bindingBytes, &m); err != nil { + return err + } + if len(m) == 0 { + return nil + } + var records []Binding - if err := json.Unmarshal(hack.Slice(sessionStates.Bindings), &records); err != nil { + // Key "Bindings" only exists in old versions. + if _, ok := m[0]["Bindings"]; ok { + err = h.decodeOldStyleSessionStates(bindingBytes, &records) + } else { + err = json.Unmarshal(bindingBytes, &records) + } + if err != nil { return err } + for _, record := range records { // Restore hints and ID because hints are hard to encode. - if err := prepareHints(sctx, &record); err != nil { + if err = prepareHints(sctx, &record); err != nil { return err } h.appendSessionBinding(parser.DigestNormalized(record.OriginalSQL).String(), []Binding{record}) } + return nil +} +// Before v8.0.0, the data structure is different. We need to adapt to the old structure so that the sessions +// can be migrated from an old version to a new version. +func (*sessionBindingHandle) decodeOldStyleSessionStates(bindingBytes []byte, bindings *[]Binding) error { + type bindRecord struct { + OriginalSQL string + Db string + Bindings []Binding + } + var records []bindRecord + if err := json.Unmarshal(bindingBytes, &records); err != nil { + return err + } + *bindings = make([]Binding, 0, len(records)) + for _, record := range records { + for _, binding := range record.Bindings { + binding.OriginalSQL = record.OriginalSQL + binding.Db = record.Db + *bindings = append(*bindings, binding) + } + } return nil } diff --git a/pkg/bindinfo/session_handle_test.go b/pkg/bindinfo/session_handle_test.go index 75826afc25a31..ebbe4bd6471fc 100644 --- a/pkg/bindinfo/session_handle_test.go +++ b/pkg/bindinfo/session_handle_test.go @@ -23,13 +23,11 @@ import ( "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/bindinfo/internal" "github.com/pingcap/tidb/pkg/bindinfo/norm" - "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/server" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/util/stmtsummary" - dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -48,14 +46,6 @@ func TestGlobalAndSessionBindingBothExist(t *testing.T) { tk.MustExec("create global binding for SELECT * from t1,t2 where t1.id = t2.id using SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1,t2 where t1.id = t2.id") - // Test bindingUsage, which indicates how many times the binding is used. - metrics.BindUsageCounter.Reset() - tk.MustHavePlan("SELECT * from t1,t2 where t1.id = t2.id", "MergeJoin") - pb := &dto.Metric{} - err := metrics.BindUsageCounter.WithLabelValues(metrics.ScopeGlobal).Write(pb) - require.NoError(t, err) - require.Equal(t, float64(1), pb.GetCounter().GetValue()) - // Test 'tidb_use_plan_baselines' tk.MustExec("set @@tidb_use_plan_baselines = 0") tk.MustHavePlan("SELECT * from t1,t2 where t1.id = t2.id", "HashJoin") diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index d64cd29e36d0e..e28e02aceb44d 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -23,7 +23,6 @@ go_test( "//pkg/util", "//pkg/util/parser", "//pkg/util/stmtsummary", - "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], diff --git a/pkg/bindinfo/tests/bind_test.go b/pkg/bindinfo/tests/bind_test.go index d3b9c4d2bc1f5..71eaf0434184e 100644 --- a/pkg/bindinfo/tests/bind_test.go +++ b/pkg/bindinfo/tests/bind_test.go @@ -20,7 +20,6 @@ import ( "strconv" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/bindinfo/internal" "github.com/pingcap/tidb/pkg/bindinfo/norm" @@ -1099,13 +1098,15 @@ func TestFuzzyBindingHintsWithSourceReturning(t *testing.T) { for _, currentDB := range []string{"db1", "db2", "db3"} { tk.MustExec(`use ` + currentDB) for _, db := range []string{"db1.", "db2.", "db3.", ""} { - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/bindinfo/get_binding_return_nil", `return()`)) query := fmt.Sprintf(c.qTemplate, db) tk.MustExec(query) - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/bindinfo/get_binding_return_nil")) tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + sctx := tk.Session() + sctx.SetValue(bindinfo.GetBindingReturnNil, true) tk.MustExec(query) + sctx.ClearValue(bindinfo.GetBindingReturnNil) tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1")) + bindinfo.GetBindingReturnNilBool.Store(false) } } } diff --git a/pkg/bindinfo/tests/timeout/BUILD.bazel b/pkg/bindinfo/tests/timeout/BUILD.bazel new file mode 100644 index 0000000000000..b297862dca25e --- /dev/null +++ b/pkg/bindinfo/tests/timeout/BUILD.bazel @@ -0,0 +1,18 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "timeout_test", + timeout = "short", + srcs = [ + "main_test.go", + "timeout_test.go", + ], + flaky = True, + deps = [ + "//pkg/bindinfo", + "//pkg/testkit", + "//pkg/testkit/testsetup", + "@com_github_stretchr_testify//require", + "@org_uber_go_goleak//:goleak", + ], +) diff --git a/pkg/bindinfo/tests/timeout/main_test.go b/pkg/bindinfo/tests/timeout/main_test.go new file mode 100644 index 0000000000000..2556d85622f9f --- /dev/null +++ b/pkg/bindinfo/tests/timeout/main_test.go @@ -0,0 +1,35 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package timeout + +import ( + "testing" + + "github.com/pingcap/tidb/pkg/testkit/testsetup" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + testsetup.SetupForCommonTest() + opts := []goleak.Option{ + goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"), + goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), + goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), + goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreTopFunction("time.Sleep"), + } + goleak.VerifyTestMain(m, opts...) +} diff --git a/pkg/bindinfo/tests/timeout/timeout_test.go b/pkg/bindinfo/tests/timeout/timeout_test.go new file mode 100644 index 0000000000000..5c390e5518790 --- /dev/null +++ b/pkg/bindinfo/tests/timeout/timeout_test.go @@ -0,0 +1,85 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package timeout + +import ( + "fmt" + "testing" + + "github.com/pingcap/tidb/pkg/bindinfo" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" +) + +func TestLoadBindingWarn(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + tk.MustExec(`create table t1 (a int)`) + tk.MustExec(`create global binding using select * from t1`) + tk.MustExec(`select * from t1`) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + + sctx := tk.Session() + sctx.SetValue(bindinfo.GetBindingReturnNilAlways, true) + tk.MustExec(`select * from t1`) + warnings := tk.MustQuery(`show warnings`) + require.True(t, len(warnings.Rows()) == 1) + sctx.ClearValue(bindinfo.GetBindingReturnNilAlways) +} + +func TestFuzzyBindingHintsWithSourceReturningTimeout(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + + for _, db := range []string{"db1", "db2", "db3"} { + tk.MustExec(`create database ` + db) + tk.MustExec(`use ` + db) + tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + } + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) + + for _, c := range []struct { + binding string + qTemplate string + }{ + // use index + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where a=1`, + `select * from %st1 where a=1000`}, + } { + tk.MustExec(c.binding) + for _, currentDB := range []string{"db1", "db2", "db3"} { + tk.MustExec(`use ` + currentDB) + for _, db := range []string{"db1.", "db2.", "db3.", ""} { + query := fmt.Sprintf(c.qTemplate, db) + tk.MustExec(query) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + sctx := tk.Session() + sctx.SetValue(bindinfo.GetBindingReturnNil, true) + sctx.SetValue(bindinfo.GetBindingReturnNilAlways, true) + sctx.SetValue(bindinfo.LoadBindingNothing, true) + tk.MustExec(query) + sctx.ClearValue(bindinfo.GetBindingReturnNil) + sctx.ClearValue(bindinfo.GetBindingReturnNilAlways) + sctx.ClearValue(bindinfo.LoadBindingNothing) + tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("0")) + bindinfo.GetBindingReturnNilBool.Store(false) + } + } + } +} diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index d56e963b0ff62..11b8d63b2156a 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -37,7 +37,7 @@ go_test( data = glob(["**"]), embed = [":config"], flaky = True, - shard_count = 23, + shard_count = 24, deps = [ "//pkg/testkit/testsetup", "//pkg/util/logutil", diff --git a/pkg/config/config.go b/pkg/config/config.go index 716204baf623d..366f47e6884fe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -490,6 +490,8 @@ type Log struct { // ExpensiveThreshold is deprecated. ExpensiveThreshold uint `toml:"expensive-threshold" json:"expensive-threshold"` + GeneralLogFile string `toml:"general-log-file" json:"general-log-file"` + // The following items are deprecated. We need to keep them here temporarily // to support the upgrade process. They can be removed in future. @@ -1264,13 +1266,16 @@ func (c *Config) RemovedVariableCheck(confFile string) error { // Load loads config options from a toml file. func (c *Config) Load(confFile string) error { metaData, err := toml.DecodeFile(confFile, c) + if err != nil { + return err + } if c.TokenLimit == 0 { c.TokenLimit = 1000 } // If any items in confFile file are not mapped into the Config struct, issue // an error and stop the server from starting. undecoded := metaData.Undecoded() - if len(undecoded) > 0 && err == nil { + if len(undecoded) > 0 { var undecodedItems []string for _, item := range undecoded { undecodedItems = append(undecodedItems, item.String()) @@ -1441,7 +1446,7 @@ var TableLockDelayClean = func() uint64 { // ToLogConfig converts *Log to *logutil.LogConfig. func (l *Log) ToLogConfig() *logutil.LogConfig { - return logutil.NewLogConfig(l.Level, l.Format, l.SlowQueryFile, l.File, l.getDisableTimestamp(), + return logutil.NewLogConfig(l.Level, l.Format, l.SlowQueryFile, l.GeneralLogFile, l.File, l.getDisableTimestamp(), func(config *zaplog.Config) { config.DisableErrorVerbose = l.getDisableErrorStack() }, func(config *zaplog.Config) { config.Timeout = l.Timeout }, ) diff --git a/pkg/config/config.toml.example b/pkg/config/config.toml.example index 47096de93e4af..c1438a9e8535a 100644 --- a/pkg/config/config.toml.example +++ b/pkg/config/config.toml.example @@ -154,6 +154,10 @@ format = "text" # Stores slow query log into separated files. slow-query-file = "tidb-slow.log" +# Stores general log into separated files. +# If it equals to empty, the general log will be written to the server log. +general-log-file = "" + # Make tidb panic if the write log operation hang for 15s # timeout = 15 @@ -171,6 +175,10 @@ max-days = 0 # Maximum number of old log files to retain. No clean up by default. max-backups = 0 +# Compression function for rotated files. +# It can be empty or "gzip", empty means compression disabled. +compression = "" + [security] # Path of file that contains list of trusted SSL CAs for connection with mysql client. ssl-ca = "" diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index caaa755d77294..f222a419ea532 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -126,7 +126,7 @@ func TestLogConfig(t *testing.T) { require.Equal(t, expectedDisableErrorStack, conf.Log.DisableErrorStack) require.Equal(t, expectedEnableTimestamp, conf.Log.EnableTimestamp) require.Equal(t, expectedDisableTimestamp, conf.Log.DisableTimestamp) - require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, resultedDisableTimestamp, func(config *zaplog.Config) { config.DisableErrorVerbose = resultedDisableErrorVerbose }), conf.Log.ToLogConfig()) + require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", "", conf.Log.File, resultedDisableTimestamp, func(config *zaplog.Config) { config.DisableErrorVerbose = resultedDisableErrorVerbose }), conf.Log.ToLogConfig()) err := f.Truncate(0) require.NoError(t, err) _, err = f.Seek(0, 0) @@ -754,6 +754,7 @@ store-limit=0 ttl-refreshed-txn-size=8192 resolve-lock-lite-threshold = 16 copr-req-timeout = "120s" +enable-replica-selector-v2 = false [tikv-client.async-commit] keys-limit=123 total-key-size-limit=1024 @@ -804,6 +805,8 @@ max_connections = 200 require.Equal(t, uint(6000), conf.TiKVClient.RegionCacheTTL) require.Equal(t, int64(0), conf.TiKVClient.StoreLimit) require.Equal(t, int64(8192), conf.TiKVClient.TTLRefreshedTxnSize) + require.Equal(t, false, conf.TiKVClient.EnableReplicaSelectorV2) + require.Equal(t, true, defaultConf.TiKVClient.EnableReplicaSelectorV2) require.Equal(t, uint(1000), conf.TokenLimit) require.True(t, conf.EnableTableLock) require.Equal(t, uint64(5), conf.DelayCleanTableLock) @@ -910,7 +913,7 @@ spilled-file-encryption-method = "aes128-ctr" require.Equal(t, GetGlobalConfig(), conf) // Test for log config. - require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() }), conf.Log.ToLogConfig()) + require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", "", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() }), conf.Log.ToLogConfig()) // Test for tracing config. tracingConf := &tracing.Configuration{ @@ -1336,3 +1339,25 @@ func TestAutoScalerConfig(t *testing.T) { conf.UseAutoScaler = false }) } + +func TestInvalidConfigWithDeprecatedConfig(t *testing.T) { + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config.toml") + + f, err := os.Create(configFile) + require.NoError(t, err) + + _, err = f.WriteString(` +[log] +slow-threshold = 1000 +[performance] +enforce-mpp = 1 + `) + require.NoError(t, err) + require.NoError(t, f.Sync()) + + var conf Config + err = conf.Load(configFile) + require.Error(t, err) + require.Equal(t, err.Error(), "toml: line 5 (last key \"performance.enforce-mpp\"): incompatible types: TOML value has type int64; destination has type boolean") +} diff --git a/pkg/ddl/backfilling_dist_executor.go b/pkg/ddl/backfilling_dist_executor.go index 79ace852029f8..a35e9323c66ed 100644 --- a/pkg/ddl/backfilling_dist_executor.go +++ b/pkg/ddl/backfilling_dist_executor.go @@ -44,9 +44,6 @@ type BackfillTaskMeta struct { EleTypeKey []byte `json:"ele_type_key"` CloudStorageURI string `json:"cloud_storage_uri"` - // UseMergeSort indicate whether the backfilling task use merge sort step for global sort. - // Merge Sort step aims to support more data. - UseMergeSort bool `json:"use_merge_sort"` } // BackfillSubTaskMeta is the sub-task meta for backfilling index. diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go index 900a3f4f04535..a027dd12b20a1 100644 --- a/pkg/ddl/backfilling_dist_scheduler.go +++ b/pkg/ddl/backfilling_dist_scheduler.go @@ -101,24 +101,9 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch( } return generateNonPartitionPlan(sch.d, tblInfo, job, sch.GlobalSort, len(execIDs)) case proto.BackfillStepMergeSort: - res, err := generateMergePlan(taskHandle, task, logger) - if err != nil { - return nil, err - } - if len(res) > 0 { - backfillMeta.UseMergeSort = true - if err := updateMeta(task, &backfillMeta); err != nil { - return nil, err - } - } - return res, nil + return generateMergePlan(taskHandle, task, logger) case proto.BackfillStepWriteAndIngest: if sch.GlobalSort { - prevStep := proto.BackfillStepReadIndex - if backfillMeta.UseMergeSort { - prevStep = proto.BackfillStepMergeSort - } - failpoint.Inject("mockWriteIngest", func() { m := &BackfillSubTaskMeta{ MetaGroups: []*external.SortedKVMeta{}, @@ -134,7 +119,6 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch( taskHandle, task, backfillMeta.CloudStorageURI, - prevStep, logger) } return nil, nil @@ -143,15 +127,6 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch( } } -func updateMeta(task *proto.Task, taskMeta *BackfillTaskMeta) error { - bs, err := json.Marshal(taskMeta) - if err != nil { - return errors.Trace(err) - } - task.Meta = bs - return nil -} - // GetNextStep implements scheduler.Extension interface. func (sch *BackfillingSchedulerExt) GetNextStep(task *proto.TaskBase) proto.Step { switch task.Step { @@ -369,24 +344,33 @@ func generateGlobalSortIngestPlan( taskHandle diststorage.TaskHandle, task *proto.Task, cloudStorageURI string, - step proto.Step, logger *zap.Logger, ) ([][]byte, error) { var kvMetaGroups []*external.SortedKVMeta - err := forEachBackfillSubtaskMeta(taskHandle, task.ID, step, func(subtask *BackfillSubTaskMeta) { - if kvMetaGroups == nil { - kvMetaGroups = make([]*external.SortedKVMeta, len(subtask.MetaGroups)) - } - for i, cur := range subtask.MetaGroups { - if kvMetaGroups[i] == nil { - kvMetaGroups[i] = &external.SortedKVMeta{} + for _, step := range []proto.Step{proto.BackfillStepMergeSort, proto.BackfillStepReadIndex} { + hasSubtasks := false + err := forEachBackfillSubtaskMeta(taskHandle, task.ID, step, func(subtask *BackfillSubTaskMeta) { + hasSubtasks = true + if kvMetaGroups == nil { + kvMetaGroups = make([]*external.SortedKVMeta, len(subtask.MetaGroups)) + } + for i, cur := range subtask.MetaGroups { + if kvMetaGroups[i] == nil { + kvMetaGroups[i] = &external.SortedKVMeta{} + } + kvMetaGroups[i].Merge(cur) } - kvMetaGroups[i].Merge(cur) + }) + if err != nil { + return nil, err } - }) - if err != nil { - return nil, err + if hasSubtasks { + break + } + // If there is no subtask for merge sort step, + // it means the merge sort step is skipped. } + instanceIDs, err := scheduler.GetLiveExecIDs(ctx) if err != nil { return nil, err @@ -581,7 +565,7 @@ func getRangeSplitter( } return external.NewRangeSplitter(ctx, multiFileStat, extStore, - rangeGroupSize, rangeGroupKeys, maxSizePerRange, maxKeysPerRange, true) + rangeGroupSize, rangeGroupKeys, maxSizePerRange, maxKeysPerRange) } func forEachBackfillSubtaskMeta( diff --git a/pkg/ddl/bdr.go b/pkg/ddl/bdr.go index 8271a79446e7b..86c71473097d8 100644 --- a/pkg/ddl/bdr.go +++ b/pkg/ddl/bdr.go @@ -27,27 +27,26 @@ func deniedByBDRWhenAddColumn(options []*ast.ColumnOption) bool { notNull bool defaultValue bool comment int + generated int ) for _, opt := range options { - if opt.Tp == ast.ColumnOptionNull { - nullable = true - } - if opt.Tp == ast.ColumnOptionNotNull { - notNull = true - } - if opt.Tp == ast.ColumnOptionDefaultValue { + switch opt.Tp { + case ast.ColumnOptionDefaultValue: defaultValue = true - } - if opt.Tp == ast.ColumnOptionComment { + case ast.ColumnOptionComment: comment = 1 + case ast.ColumnOptionGenerated: + generated = 1 + case ast.ColumnOptionNotNull: + notNull = true + case ast.ColumnOptionNull: + nullable = true } } - tpLen := len(options) - comment + tpLen := len(options) - comment - generated - if tpLen == 0 || (tpLen == 1 && nullable) { - return false - } - if tpLen == 2 && notNull && defaultValue { + if tpLen == 0 || (tpLen == 1 && nullable) || (tpLen == 1 && !notNull && defaultValue) || + (tpLen == 2 && notNull && defaultValue) { return false } diff --git a/pkg/ddl/bdr_test.go b/pkg/ddl/bdr_test.go index 5d2a6d3adab11..baecc65a88e0e 100644 --- a/pkg/ddl/bdr_test.go +++ b/pkg/ddl/bdr_test.go @@ -30,7 +30,7 @@ func TestDeniedByBDRWhenAddColumn(t *testing.T) { expected bool }{ { - name: "Test with no options", + name: "Test with no options(implicit nullable)", options: []*ast.ColumnOption{}, expected: false, }, @@ -40,13 +40,33 @@ func TestDeniedByBDRWhenAddColumn(t *testing.T) { expected: false, }, { - name: "Test with notNull and defaultValue options", + name: "Test with implicit nullable and defaultValue options", + options: []*ast.ColumnOption{{Tp: ast.ColumnOptionDefaultValue}}, + expected: false, + }, + { + name: "Test with nullable and defaultValue options", options: []*ast.ColumnOption{{Tp: ast.ColumnOptionNotNull}, {Tp: ast.ColumnOptionDefaultValue}}, expected: false, }, + { + name: "Test with comment options", + options: []*ast.ColumnOption{{Tp: ast.ColumnOptionComment}}, + expected: false, + }, + { + name: "Test with generated options", + options: []*ast.ColumnOption{{Tp: ast.ColumnOptionGenerated}}, + expected: false, + }, + { + name: "Test with comment and generated options", + options: []*ast.ColumnOption{{Tp: ast.ColumnOptionComment}, {Tp: ast.ColumnOptionGenerated}}, + expected: false, + }, { name: "Test with other options", - options: []*ast.ColumnOption{{Tp: ast.ColumnOptionPrimaryKey}}, + options: []*ast.ColumnOption{{Tp: ast.ColumnOptionCheck}}, expected: true, }, } diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go index 0ce3042037c69..bf3b0a1ffad03 100644 --- a/pkg/ddl/column.go +++ b/pkg/ddl/column.go @@ -631,7 +631,7 @@ func SetIdxColNameOffset(idxCol *model.IndexColumn, changingCol *model.ColumnInf idxCol.Name = changingCol.Name idxCol.Offset = changingCol.Offset canPrefix := types.IsTypePrefixable(changingCol.GetType()) - if !canPrefix || (changingCol.GetFlen() < idxCol.Length) { + if !canPrefix || (changingCol.GetFlen() <= idxCol.Length) { idxCol.Length = types.UnspecifiedLength } } diff --git a/pkg/ddl/ddl_api.go b/pkg/ddl/ddl_api.go index 38a5d3b7703a1..f404c18984f08 100644 --- a/pkg/ddl/ddl_api.go +++ b/pkg/ddl/ddl_api.go @@ -1341,7 +1341,8 @@ func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr * col.DefaultIsExpr = true return str, false, nil } - return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), nowFunc.FnName.String()) + return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), + fmt.Sprintf("%s with disallowed args", expr.FnName.String())) case ast.Replace: if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil { return nil, false, errors.Trace(err) @@ -1371,7 +1372,8 @@ func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr * return str, false, nil } } - return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), expr.FnName.String()) + return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), + fmt.Sprintf("%s with disallowed args", expr.FnName.String())) case ast.Upper: if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil { return nil, false, errors.Trace(err) @@ -1397,7 +1399,8 @@ func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr * return str, false, nil } } - return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), expr.FnName.String()) + return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), + fmt.Sprintf("%s with disallowed args", expr.FnName.String())) case ast.StrToDate: // STR_TO_DATE() if err := expression.VerifyArgsWrapper(expr.FnName.L, len(expr.Args)); err != nil { return nil, false, errors.Trace(err) @@ -1413,7 +1416,8 @@ func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr * return str, false, nil } } - return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), fmt.Sprintf("%s with these args", expr.FnName.String())) + return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), + fmt.Sprintf("%s with disallowed args", expr.FnName.String())) default: return nil, false, dbterror.ErrDefValGeneratedNamedFunctionIsNotAllowed.GenWithStackByArgs(col.Name.String(), expr.FnName.String()) } @@ -4603,6 +4607,38 @@ func (d *ddl) AlterTablePartitioning(ctx sessionctx.Context, ident ast.Ident, sp } newPartInfo := newMeta.Partition + for _, index := range newMeta.Indices { + if index.Unique { + ck, err := checkPartitionKeysConstraint(newMeta.GetPartitionInfo(), index.Columns, newMeta) + if err != nil { + return err + } + if !ck { + if ctx.GetSessionVars().EnableGlobalIndex { + return dbterror.ErrCancelledDDLJob.GenWithStack("global index is not supported yet for alter table partitioning") + } + indexTp := "UNIQUE INDEX" + if index.Primary { + indexTp = "PRIMARY" + } + return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs(indexTp) + } + } + } + if newMeta.PKIsHandle { + indexCols := []*model.IndexColumn{{ + Name: newMeta.GetPkName(), + Length: types.UnspecifiedLength, + }} + ck, err := checkPartitionKeysConstraint(newMeta.GetPartitionInfo(), indexCols, newMeta) + if err != nil { + return err + } + if !ck { + return dbterror.ErrUniqueKeyNeedAllFieldsInPf.GenWithStackByArgs("PRIMARY") + } + } + if err = handlePartitionPlacement(ctx, newPartInfo); err != nil { return errors.Trace(err) } @@ -5221,7 +5257,7 @@ func checkExchangePartition(pt *model.TableInfo, nt *model.TableInfo) error { return errors.Trace(dbterror.ErrPartitionExchangePartTable.GenWithStackByArgs(nt.Name)) } - if nt.ForeignKeys != nil { + if len(nt.ForeignKeys) > 0 { return errors.Trace(dbterror.ErrPartitionExchangeForeignKey.GenWithStackByArgs(nt.Name)) } diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index ee068bf657f9e..5445a14cd2574 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -113,7 +113,12 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde mvIndex = true } indexColLen := ip.Length - indexColumnLength, err := getIndexColumnLength(col, ip.Length) + if indexColLen != types.UnspecifiedLength && + types.IsTypeChar(col.FieldType.GetType()) && + indexColLen == col.FieldType.GetFlen() { + indexColLen = types.UnspecifiedLength + } + indexColumnLength, err := getIndexColumnLength(col, indexColLen) if err != nil { return nil, false, err } @@ -1594,31 +1599,12 @@ func (w *addIndexTxnWorker) checkHandleExists(idxInfo *model.IndexInfo, key kv.K } func genKeyExistsErr(key, value []byte, idxInfo *model.IndexInfo, tblInfo *model.TableInfo) error { - idxColLen := len(idxInfo.Columns) indexName := fmt.Sprintf("%s.%s", tblInfo.Name.String(), idxInfo.Name.String()) - colInfos := tables.BuildRowcodecColInfoForIndexColumns(idxInfo, tblInfo) - values, err := tablecodec.DecodeIndexKV(key, value, idxColLen, tablecodec.HandleNotNeeded, colInfos) + valueStr, err := tables.GenIndexValueFromIndex(key, value, tblInfo, idxInfo) if err != nil { - logutil.BgLogger().Warn("decode index key value failed", zap.String("index", indexName), + logutil.BgLogger().Warn("decode index key value / column value failed", zap.String("index", indexName), zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err)) - return kv.ErrKeyExists.FastGenByArgs(key, indexName) - } - valueStr := make([]string, 0, idxColLen) - for i, val := range values[:idxColLen] { - d, err := tablecodec.DecodeColumnValue(val, colInfos[i].Ft, time.Local) - if err != nil { - logutil.BgLogger().Warn("decode column value failed", zap.String("index", indexName), - zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err)) - return kv.ErrKeyExists.FastGenByArgs(key, indexName) - } - str, err := d.ToString() - if err != nil { - str = string(val) - } - if types.IsBinaryStr(colInfos[i].Ft) || types.IsTypeBit(colInfos[i].Ft) { - str = util.FmtNonASCIIPrintableCharToHex(str) - } - valueStr = append(valueStr, str) + return errors.Trace(kv.ErrKeyExists.FastGenByArgs(key, indexName)) } return kv.ErrKeyExists.FastGenByArgs(strings.Join(valueStr, "-"), indexName) } diff --git a/pkg/ddl/index_merge_tmp.go b/pkg/ddl/index_merge_tmp.go index ffdf0abcb4441..8b39fdf0e2dc4 100644 --- a/pkg/ddl/index_merge_tmp.go +++ b/pkg/ddl/index_merge_tmp.go @@ -35,10 +35,9 @@ import ( func (w *mergeIndexWorker) batchCheckTemporaryUniqueKey( txn kv.Transaction, - idxInfo *model.IndexInfo, idxRecords []*temporaryIndexRecord, ) error { - if !idxInfo.Unique { + if !w.currentIndex.Unique { // non-unique key need no check, just overwrite it, // because in most case, backfilling indices is not exists. return nil @@ -55,7 +54,7 @@ func (w *mergeIndexWorker) batchCheckTemporaryUniqueKey( err := checkTempIndexKey(txn, idxRecords[i], val, w.table) if err != nil { if kv.ErrKeyExists.Equal(err) { - return driver.ExtractKeyExistsErrFromIndex(key, val, w.table.Meta(), idxInfo.ID) + return driver.ExtractKeyExistsErrFromIndex(key, val, w.table.Meta(), w.currentIndex.ID) } return errors.Trace(err) } @@ -128,6 +127,10 @@ type mergeIndexWorker struct { tmpIdxRecords []*temporaryIndexRecord originIdxKeys []kv.Key tmpIdxKeys []kv.Key + + needValidateKey bool + currentTempIndexPrefix []byte + currentIndex *model.IndexInfo } func newMergeTempIndexWorker(bfCtx *backfillCtx, t table.PhysicalTable, elements []*meta.Element) *mergeIndexWorker { @@ -144,68 +147,99 @@ func newMergeTempIndexWorker(bfCtx *backfillCtx, t table.PhysicalTable, elements } } +func (w *mergeIndexWorker) validateTaskRange(taskRange *reorgBackfillTask) (skip bool, err error) { + tmpID, err := tablecodec.DecodeIndexID(taskRange.startKey) + if err != nil { + return false, err + } + startIndexID := tmpID & tablecodec.IndexIDMask + tmpID, err = tablecodec.DecodeIndexID(taskRange.endKey) + if err != nil { + return false, err + } + endIndexID := tmpID & tablecodec.IndexIDMask + + w.needValidateKey = startIndexID != endIndexID + containsTargetID := false + for _, idx := range w.indexes { + idxInfo := idx.Meta() + if idxInfo.ID == startIndexID { + containsTargetID = true + w.currentIndex = idxInfo + break + } + if idxInfo.ID == endIndexID { + containsTargetID = true + } + } + return !containsTargetID, nil +} + // BackfillData merge temp index data in txn. func (w *mergeIndexWorker) BackfillData(taskRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error) { + skip, err := w.validateTaskRange(&taskRange) + if skip || err != nil { + return taskCtx, err + } + oprStartTime := time.Now() ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL) - for _, idx := range w.indexes { - idx := idx // Make linter noloopclosure happy. - errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { - taskCtx.addedCount = 0 - taskCtx.scanCount = 0 - updateTxnEntrySizeLimitIfNeeded(txn) - txn.SetOption(kv.Priority, taskRange.priority) - if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(taskRange.getJobID()); tagger != nil { - txn.SetOption(kv.ResourceGroupTagger, tagger) - } - txn.SetOption(kv.ResourceGroupName, w.jobContext.resourceGroupName) - tmpIdxRecords, nextKey, taskDone, err := w.fetchTempIndexVals(txn, idx.Meta(), taskRange) - if err != nil { - return errors.Trace(err) + errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error { + taskCtx.addedCount = 0 + taskCtx.scanCount = 0 + updateTxnEntrySizeLimitIfNeeded(txn) + txn.SetOption(kv.Priority, taskRange.priority) + if tagger := w.GetCtx().getResourceGroupTaggerForTopSQL(taskRange.getJobID()); tagger != nil { + txn.SetOption(kv.ResourceGroupTagger, tagger) + } + txn.SetOption(kv.ResourceGroupName, w.jobContext.resourceGroupName) + + tmpIdxRecords, nextKey, taskDone, err := w.fetchTempIndexVals(txn, taskRange) + if err != nil { + return errors.Trace(err) + } + taskCtx.nextKey = nextKey + taskCtx.done = taskDone + + err = w.batchCheckTemporaryUniqueKey(txn, tmpIdxRecords) + if err != nil { + return errors.Trace(err) + } + + for i, idxRecord := range tmpIdxRecords { + taskCtx.scanCount++ + // The index is already exists, we skip it, no needs to backfill it. + // The following update, delete, insert on these rows, TiDB can handle it correctly. + // If all batch are skipped, update first index key to make txn commit to release lock. + if idxRecord.skip { + continue } - taskCtx.nextKey = nextKey - taskCtx.done = taskDone - err = w.batchCheckTemporaryUniqueKey(txn, idx.Meta(), tmpIdxRecords) + // Lock the corresponding row keys so that it doesn't modify the index KVs + // that are changing by a pessimistic transaction. + rowKey := tablecodec.EncodeRecordKey(w.table.RecordPrefix(), idxRecord.handle) + err := txn.LockKeys(context.Background(), new(kv.LockCtx), rowKey) if err != nil { return errors.Trace(err) } - for i, idxRecord := range tmpIdxRecords { - taskCtx.scanCount++ - // The index is already exists, we skip it, no needs to backfill it. - // The following update, delete, insert on these rows, TiDB can handle it correctly. - // If all batch are skipped, update first index key to make txn commit to release lock. - if idxRecord.skip { - continue - } - - // Lock the corresponding row keys so that it doesn't modify the index KVs - // that are changing by a pessimistic transaction. - rowKey := tablecodec.EncodeRecordKey(w.table.RecordPrefix(), idxRecord.handle) - err := txn.LockKeys(context.Background(), new(kv.LockCtx), rowKey) - if err != nil { - return errors.Trace(err) - } - - if idxRecord.delete { - if idxRecord.unique { - err = txn.GetMemBuffer().DeleteWithFlags(w.originIdxKeys[i], kv.SetNeedLocked) - } else { - err = txn.GetMemBuffer().Delete(w.originIdxKeys[i]) - } + if idxRecord.delete { + if idxRecord.unique { + err = txn.GetMemBuffer().DeleteWithFlags(w.originIdxKeys[i], kv.SetNeedLocked) } else { - err = txn.GetMemBuffer().Set(w.originIdxKeys[i], idxRecord.vals) - } - if err != nil { - return err + err = txn.GetMemBuffer().Delete(w.originIdxKeys[i]) } - taskCtx.addedCount++ + } else { + err = txn.GetMemBuffer().Set(w.originIdxKeys[i], idxRecord.vals) } - return nil - }) - } + if err != nil { + return err + } + taskCtx.addedCount++ + } + return nil + }) failpoint.Inject("mockDMLExecutionMerging", func(val failpoint.Value) { //nolint:forcetypeassert @@ -228,9 +262,41 @@ func (w *mergeIndexWorker) GetCtx() *backfillCtx { return w.backfillCtx } +func (w *mergeIndexWorker) prefixIsChanged(newKey kv.Key) bool { + return len(w.currentTempIndexPrefix) == 0 || !bytes.HasPrefix(newKey, w.currentTempIndexPrefix) +} + +func (w *mergeIndexWorker) updateCurrentIndexInfo(newIndexKey kv.Key) (skip bool, err error) { + tempIdxID, err := tablecodec.DecodeIndexID(newIndexKey) + if err != nil { + return false, err + } + idxID := tablecodec.IndexIDMask & tempIdxID + var curIdx *model.IndexInfo + for _, idx := range w.indexes { + if idx.Meta().ID == idxID { + curIdx = idx.Meta() + } + } + if curIdx == nil { + // Index IDs are always increasing, but not always continuous: + // if DDL adds another index between these indexes, it is possible that: + // multi-schema add index IDs = [1, 2, 4, 5] + // another index ID = [3] + // If the new index get rollback, temp index 0xFFxxx03 may have dirty records. + // We should skip these dirty records. + return true, nil + } + pfx := tablecodec.CutIndexPrefix(newIndexKey) + + w.currentTempIndexPrefix = kv.Key(pfx).Clone() + w.currentIndex = curIdx + + return false, nil +} + func (w *mergeIndexWorker) fetchTempIndexVals( txn kv.Transaction, - indexInfo *model.IndexInfo, taskRange reorgBackfillTask, ) ([]*temporaryIndexRecord, kv.Key, bool, error) { startTime := time.Now() @@ -254,11 +320,18 @@ func (w *mergeIndexWorker) fetchTempIndexVals( return false, nil } + if w.needValidateKey && w.prefixIsChanged(indexKey) { + skip, err := w.updateCurrentIndexInfo(indexKey) + if err != nil || skip { + return skip, err + } + } + tempIdxVal, err := tablecodec.DecodeTempIndexValue(rawValue) if err != nil { return false, err } - tempIdxVal, err = decodeTempIndexHandleFromIndexKV(indexKey, tempIdxVal, len(indexInfo.Columns)) + tempIdxVal, err = decodeTempIndexHandleFromIndexKV(indexKey, tempIdxVal, len(w.currentIndex.Columns)) if err != nil { return false, err } diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel index fca91d77303f6..09c2273707a8c 100644 --- a/pkg/ddl/ingest/BUILD.bazel +++ b/pkg/ddl/ingest/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//pkg/kv", "//pkg/meta", "//pkg/parser/mysql", + "//pkg/parser/terror", "//pkg/sessionctx", "//pkg/sessionctx/variable", "//pkg/table", @@ -67,7 +68,7 @@ go_test( embed = [":ingest"], flaky = True, race = "on", - shard_count = 16, + shard_count = 17, deps = [ "//pkg/config", "//pkg/ddl", diff --git a/pkg/ddl/ingest/backend.go b/pkg/ddl/ingest/backend.go index 7e79d5d50bfba..f8a3ea295b6b3 100644 --- a/pkg/ddl/ingest/backend.go +++ b/pkg/ddl/ingest/backend.go @@ -24,11 +24,13 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/backend" "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" "github.com/pingcap/tidb/br/pkg/lightning/backend/local" + "github.com/pingcap/tidb/br/pkg/lightning/common" lightning "github.com/pingcap/tidb/br/pkg/lightning/config" "github.com/pingcap/tidb/br/pkg/lightning/errormanager" "github.com/pingcap/tidb/br/pkg/lightning/log" tikv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/generic" @@ -90,6 +92,33 @@ type litBackendCtx struct { etcdClient *clientv3.Client } +func (bc *litBackendCtx) handleErrorAfterCollectRemoteDuplicateRows(err error, indexID int64, tbl table.Table, hasDupe bool) error { + if err != nil && !common.ErrFoundIndexConflictRecords.Equal(err) { + logutil.Logger(bc.ctx).Error(LitInfoRemoteDupCheck, zap.Error(err), + zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) + return errors.Trace(err) + } else if hasDupe { + logutil.Logger(bc.ctx).Error(LitErrRemoteDupExistErr, + zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) + + if common.ErrFoundIndexConflictRecords.Equal(err) { + tErr, ok := errors.Cause(err).(*terror.Error) + if !ok { + return errors.Trace(tikv.ErrKeyExists) + } + if len(tErr.Args()) != 4 { + return errors.Trace(tikv.ErrKeyExists) + } + indexName := tErr.Args()[1] + valueStr := tErr.Args()[2] + + return errors.Trace(tikv.ErrKeyExists.FastGenByArgs(valueStr, indexName)) + } + return errors.Trace(tikv.ErrKeyExists) + } + return nil +} + // CollectRemoteDuplicateRows collects duplicate rows from remote TiKV. func (bc *litBackendCtx) CollectRemoteDuplicateRows(indexID int64, tbl table.Table) error { errorMgr := errormanager.New(nil, bc.cfg, log.Logger{Logger: logutil.Logger(bc.ctx)}) @@ -99,17 +128,8 @@ func (bc *litBackendCtx) CollectRemoteDuplicateRows(indexID int64, tbl table.Tab SQLMode: mysql.ModeStrictAllTables, SysVars: bc.sysVars, IndexID: indexID, - }) - if err != nil { - logutil.Logger(bc.ctx).Error(LitInfoRemoteDupCheck, zap.Error(err), - zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) - return err - } else if hasDupe { - logutil.Logger(bc.ctx).Error(LitErrRemoteDupExistErr, - zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) - return tikv.ErrKeyExists - } - return nil + }, lightning.ErrorOnDup) + return bc.handleErrorAfterCollectRemoteDuplicateRows(err, indexID, tbl, hasDupe) } // FinishImport imports all the key-values in engine into the storage, collects the duplicate errors if any, and @@ -140,16 +160,8 @@ func (bc *litBackendCtx) FinishImport(indexID int64, unique bool, tbl table.Tabl SQLMode: mysql.ModeStrictAllTables, SysVars: bc.sysVars, IndexID: ei.indexID, - }) - if err != nil { - logutil.Logger(bc.ctx).Error(LitInfoRemoteDupCheck, zap.Error(err), - zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) - return err - } else if hasDupe { - logutil.Logger(bc.ctx).Error(LitErrRemoteDupExistErr, - zap.String("table", tbl.Meta().Name.O), zap.Int64("index ID", indexID)) - return tikv.ErrKeyExists - } + }, lightning.ErrorOnDup) + return bc.handleErrorAfterCollectRemoteDuplicateRows(err, indexID, tbl, hasDupe) } return nil } diff --git a/pkg/ddl/ingest/config.go b/pkg/ddl/ingest/config.go index 8c92bb46459a8..baf2eff42aaf2 100644 --- a/pkg/ddl/ingest/config.go +++ b/pkg/ddl/ingest/config.go @@ -63,10 +63,10 @@ func genConfig(ctx context.Context, memRoot MemRoot, jobID int64, unique bool, r adjustImportMemory(ctx, memRoot, cfg) cfg.Checkpoint.Enable = true if unique { - cfg.TikvImporter.DuplicateResolution = lightning.DupeResAlgErr + cfg.Conflict.Strategy = lightning.ErrorOnDup cfg.Conflict.Threshold = math.MaxInt64 } else { - cfg.TikvImporter.DuplicateResolution = lightning.DupeResAlgNone + cfg.Conflict.Strategy = lightning.NoneOnDup } cfg.TiDB.Host = "127.0.0.1" cfg.TiDB.StatusPort = int(tidbCfg.Status.StatusPort) diff --git a/pkg/ddl/ingest/integration_test.go b/pkg/ddl/ingest/integration_test.go index c7dde01c84dbf..5a7ba138c525d 100644 --- a/pkg/ddl/ingest/integration_test.go +++ b/pkg/ddl/ingest/integration_test.go @@ -340,3 +340,30 @@ func TestAddIndexDuplicateMessage(t *testing.T) { tk.MustExec("admin check table t;") tk.MustQuery("select * from t;").Check(testkit.Rows("1 1 1", "2 1 2")) } + +func TestMultiSchemaAddIndexMerge(t *testing.T) { + store := testkit.CreateMockStore(t) + defer ingesttestutil.InjectMockBackendMgr(t, store)() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk2 := testkit.NewTestKit(t, store) + tk2.MustExec("use test") + + tk.MustExec("create table t (a int, b int);") + tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3);") + + first := true + var tk2Err error + ingest.MockExecAfterWriteRow = func() { + if !first { + return + } + _, tk2Err = tk2.Exec("insert into t values (4, 4);") + first = false + } + + tk.MustExec("alter table t add index idx1(a), add index idx2(b);") + require.False(t, first) + require.NoError(t, tk2Err) + tk.MustExec("admin check table t;") +} diff --git a/pkg/ddl/placement_policy_ddl_test.go b/pkg/ddl/placement_policy_ddl_test.go index 7cdcdfcf33019..1f29109f85da1 100644 --- a/pkg/ddl/placement_policy_ddl_test.go +++ b/pkg/ddl/placement_policy_ddl_test.go @@ -15,6 +15,7 @@ package ddl_test import ( "context" + "math" "testing" "github.com/pingcap/tidb/pkg/ddl" @@ -125,7 +126,7 @@ func TestPlacementPolicyInUse(t *testing.T) { 1, ) require.NoError(t, err) - is := builder.Build() + is := builder.Build(math.MaxUint64) ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) for _, policy := range []*model.PolicyInfo{p1, p2, p4, p5} { diff --git a/pkg/ddl/reorg.go b/pkg/ddl/reorg.go index 48b8e58589959..506e4cd6231d3 100644 --- a/pkg/ddl/reorg.go +++ b/pkg/ddl/reorg.go @@ -706,7 +706,10 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, tb = tbl.(table.PhysicalTable) } if mergingTmpIdx { - start, end = tablecodec.GetTableIndexKeyRange(pid, tablecodec.TempIndexPrefix|elements[0].ID) + firstElemTempID := tablecodec.TempIndexPrefix | elements[0].ID + lastElemTempID := tablecodec.TempIndexPrefix | elements[len(elements)-1].ID + start = tablecodec.EncodeIndexSeekKey(pid, firstElemTempID, nil) + end = tablecodec.EncodeIndexSeekKey(pid, lastElemTempID, []byte{255}) } else { start, end, err = getTableRange(ctx, d, tb, ver.Ver, job.Priority) if err != nil { diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go index 55d3da05d363b..63a71b3779628 100644 --- a/pkg/distsql/select_result.go +++ b/pkg/distsql/select_result.go @@ -502,7 +502,7 @@ func recordExecutionSummariesForTiFlashTasks(sctx *stmtctx.StatementContext, exe func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr.CopRuntimeStats, respTime time.Duration) (err error) { callee := copStats.CalleeAddress - if r.rootPlanID <= 0 || r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil || callee == "" { + if r.rootPlanID <= 0 || r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil || (callee == "" && len(copStats.Stats) == 0) { return } @@ -603,6 +603,13 @@ func (r *selectResult) Close() error { if respSize > 0 { r.memConsume(-respSize) } + if unconsumed, ok := r.resp.(copr.HasUnconsumedCopRuntimeStats); ok && unconsumed != nil { + unconsumedCopStats := unconsumed.CollectUnconsumedCopRuntimeStats() + for _, copStats := range unconsumedCopStats { + _ = r.updateCopRuntimeStats(context.Background(), copStats, time.Duration(0)) + r.ctx.GetSessionVars().StmtCtx.MergeExecDetails(&copStats.ExecDetails, nil) + } + } if r.stats != nil { defer func() { if ci, ok := r.resp.(copr.CopInfo); ok { diff --git a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go index 5889c86deb363..fde32079a0df7 100644 --- a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go +++ b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go @@ -37,7 +37,12 @@ func CheckSubtasksState(ctx context.Context, t *testing.T, taskID int64, state p require.NoError(t, err) historySubTasksCnt, err := testutil.GetSubtasksFromHistoryByTaskID(ctx, mgr, taskID) require.NoError(t, err) - require.Equal(t, expectedCnt, cntByStatesStepOne[state]+cntByStatesStepTwo[state]+int64(historySubTasksCnt)) + // all subtasks moved to history. + if historySubTasksCnt != 0 { + require.Equal(t, expectedCnt, int64(historySubTasksCnt)) + } else { + require.Equal(t, expectedCnt, cntByStatesStepOne[state]+cntByStatesStepTwo[state]) + } } func TestFrameworkPauseAndResume(t *testing.T) { diff --git a/pkg/disttask/framework/scheduler/scheduler.go b/pkg/disttask/framework/scheduler/scheduler.go index 96e1ddcb6eaaf..ea43e4d59092c 100644 --- a/pkg/disttask/framework/scheduler/scheduler.go +++ b/pkg/disttask/framework/scheduler/scheduler.go @@ -451,7 +451,7 @@ func (s *BaseScheduler) switch2NextStep() error { return s.handlePlanErr(err) } - if err = s.scheduleSubTask(nextStep, metas, eligibleNodes); err != nil { + if err = s.scheduleSubTask(&task, nextStep, metas, eligibleNodes); err != nil { return err } task.Step = nextStep @@ -462,10 +462,10 @@ func (s *BaseScheduler) switch2NextStep() error { } func (s *BaseScheduler) scheduleSubTask( + task *proto.Task, subtaskStep proto.Step, metas [][]byte, eligibleNodes []string) error { - task := s.GetTask() s.logger.Info("schedule subtasks", zap.Stringer("state", task.State), zap.String("step", proto.Step2Str(task.Type, subtaskStep)), diff --git a/pkg/disttask/framework/taskexecutor/BUILD.bazel b/pkg/disttask/framework/taskexecutor/BUILD.bazel index d5518555c8d27..ec3257162530f 100644 --- a/pkg/disttask/framework/taskexecutor/BUILD.bazel +++ b/pkg/disttask/framework/taskexecutor/BUILD.bazel @@ -21,8 +21,10 @@ go_library( "//pkg/disttask/framework/storage", "//pkg/disttask/framework/taskexecutor/execute", "//pkg/metrics", + "//pkg/sessionctx/variable", "//pkg/util", "//pkg/util/backoff", + "//pkg/util/cgroup", "//pkg/util/cpu", "//pkg/util/gctuner", "//pkg/util/intest", diff --git a/pkg/disttask/framework/taskexecutor/manager.go b/pkg/disttask/framework/taskexecutor/manager.go index 20abcbfc116a4..9aa2948936ee8 100644 --- a/pkg/disttask/framework/taskexecutor/manager.go +++ b/pkg/disttask/framework/taskexecutor/manager.go @@ -28,8 +28,10 @@ import ( "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" "github.com/pingcap/tidb/pkg/disttask/framework/storage" "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/sessionctx/variable" tidbutil "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/backoff" + "github.com/pingcap/tidb/pkg/util/cgroup" "github.com/pingcap/tidb/pkg/util/cpu" "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/memory" @@ -91,6 +93,22 @@ func NewManager(ctx context.Context, id string, taskTable TaskTable) (*Manager, if totalCPU <= 0 || totalMem <= 0 { return nil, errors.Errorf("invalid cpu or memory, cpu: %d, memory: %d", totalCPU, totalMem) } + cgroupLimit, version, err := cgroup.GetCgroupMemLimit() + // ignore the error of cgroup.GetCgroupMemLimit, as it's not a must-success step. + if err == nil && version == cgroup.V2 { + // see cgroup.detectMemLimitInV2 for more details. + // below are some real memory limits tested on GCP: + // node-spec real-limit percent + // 16c32g 27.83Gi 87% + // 32c64g 57.36Gi 89.6% + // we use 'limit', not totalMem for adjust, as totalMem = min(physical-mem, 'limit') + // content of 'memory.max' might be 'max', so we use the min of them. + adjustedMem := min(totalMem, uint64(float64(cgroupLimit)*0.88)) + logger.Info("adjust memory limit for cgroup v2", + zap.String("before", units.BytesSize(float64(totalMem))), + zap.String("after", units.BytesSize(float64(adjustedMem)))) + totalMem = adjustedMem + } logger.Info("build task executor manager", zap.Int("total-cpu", totalCPU), zap.String("total-mem", units.BytesSize(float64(totalMem)))) m := &Manager{ @@ -163,6 +181,9 @@ func (m *Manager) handleTasksLoop() { } m.handleTasks() + // service scope might change, so we call WithLabelValues every time. + metrics.DistTaskUsedSlotsGauge.WithLabelValues(variable.ServiceScope.Load()). + Set(float64(m.slotManager.usedSlots())) } } diff --git a/pkg/disttask/framework/taskexecutor/slot.go b/pkg/disttask/framework/taskexecutor/slot.go index 0e6ba81577b1b..edad2c7ea52b9 100644 --- a/pkg/disttask/framework/taskexecutor/slot.go +++ b/pkg/disttask/framework/taskexecutor/slot.go @@ -31,6 +31,7 @@ type slotManager struct { // the slice is sorted in reverse task order. executorTasks []*proto.TaskBase + capacity int // The number of slots that can be used by the executor. // Its initial value is always equal to CPU cores of the instance. available atomic.Int32 @@ -40,6 +41,7 @@ func newSlotManager(capacity int) *slotManager { sm := &slotManager{ taskID2Index: make(map[int64]int), executorTasks: make([]*proto.TaskBase, 0), + capacity: capacity, } sm.available.Store(int32(capacity)) return sm @@ -104,3 +106,7 @@ func (sm *slotManager) canAlloc(task *proto.TaskBase) (canAlloc bool, tasksNeedF func (sm *slotManager) availableSlots() int { return int(sm.available.Load()) } + +func (sm *slotManager) usedSlots() int { + return sm.capacity - int(sm.available.Load()) +} diff --git a/pkg/disttask/importinto/BUILD.bazel b/pkg/disttask/importinto/BUILD.bazel index 7794785911dde..612f8d934d478 100644 --- a/pkg/disttask/importinto/BUILD.bazel +++ b/pkg/disttask/importinto/BUILD.bazel @@ -54,7 +54,6 @@ go_library( "//pkg/table/tables", "//pkg/util", "//pkg/util/backoff", - "//pkg/util/dbterror/exeerrors", "//pkg/util/disttask", "//pkg/util/etcd", "//pkg/util/logutil", diff --git a/pkg/disttask/importinto/job.go b/pkg/disttask/importinto/job.go index 3b16f04be34fd..1235cbb56ae00 100644 --- a/pkg/disttask/importinto/job.go +++ b/pkg/disttask/importinto/job.go @@ -31,7 +31,6 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/tikv/client-go/v2/util" @@ -70,18 +69,6 @@ func doSubmitTask(ctx context.Context, plan *importer.Plan, stmt string, instanc if err = taskManager.WithNewTxn(ctx, func(se sessionctx.Context) error { var err2 error exec := se.(sqlexec.SQLExecutor) - // If 2 client try to execute IMPORT INTO concurrently, there's chance that both of them will pass the check. - // We can enforce ONLY one import job running by: - // - using LOCK TABLES, but it requires enable-table-lock=true, it's not enabled by default. - // - add a key to PD as a distributed lock, but it's a little complex, and we might support job queuing later. - // So we only add this simple soft check here and doc it. - activeJobCnt, err2 := importer.GetActiveJobCnt(ctx, exec) - if err2 != nil { - return err2 - } - if activeJobCnt > 0 { - return exeerrors.ErrLoadDataPreCheckFailed.FastGenByArgs("there's pending or running jobs") - } jobID, err2 = importer.CreateJob(ctx, exec, plan.DBName, plan.TableInfo.Name.L, plan.TableInfo.ID, plan.User, plan.Parameters, plan.TotalFileSize) if err2 != nil { @@ -125,7 +112,8 @@ func doSubmitTask(ctx context.Context, plan *importer.Plan, stmt string, instanc zap.Int64("job-id", jobID), zap.Int64("task-id", task.ID), zap.String("data-size", units.BytesSize(float64(plan.TotalFileSize))), - zap.Int("thread-cnt", plan.ThreadCnt)) + zap.Int("thread-cnt", plan.ThreadCnt), + zap.Bool("global-sort", plan.IsGlobalSort())) return jobID, task, nil } diff --git a/pkg/disttask/importinto/planner.go b/pkg/disttask/importinto/planner.go index 783302696ef43..4bf4ca46d0bd1 100644 --- a/pkg/disttask/importinto/planner.go +++ b/pkg/disttask/importinto/planner.go @@ -515,6 +515,5 @@ func getRangeSplitter(ctx context.Context, store storage.ExternalStorage, kvMeta int64(math.MaxInt64), regionSplitSize, regionSplitKeys, - false, ) } diff --git a/pkg/disttask/importinto/subtask_executor.go b/pkg/disttask/importinto/subtask_executor.go index 5202dc70d02d9..b9c87b104592a 100644 --- a/pkg/disttask/importinto/subtask_executor.go +++ b/pkg/disttask/importinto/subtask_executor.go @@ -106,7 +106,7 @@ func (e *importMinimalTaskExecutor) Run(ctx context.Context, dataWriter, indexWr } // postProcess does the post-processing for the task. -func postProcess(ctx context.Context, taskMeta *TaskMeta, subtaskMeta *PostProcessStepMeta, logger *zap.Logger) (err error) { +func postProcess(ctx context.Context, store kv.Storage, taskMeta *TaskMeta, subtaskMeta *PostProcessStepMeta, logger *zap.Logger) (err error) { failpoint.Inject("syncBeforePostProcess", func() { TestSyncChan <- struct{}{} <-TestSyncChan @@ -117,7 +117,7 @@ func postProcess(ctx context.Context, taskMeta *TaskMeta, subtaskMeta *PostProce callLog.End(zap.ErrorLevel, err) }() - if err = importer.RebaseAllocatorBases(ctx, subtaskMeta.MaxIDs, &taskMeta.Plan, logger); err != nil { + if err = importer.RebaseAllocatorBases(ctx, store, subtaskMeta.MaxIDs, &taskMeta.Plan, logger); err != nil { return err } diff --git a/pkg/disttask/importinto/task_executor.go b/pkg/disttask/importinto/task_executor.go index 00af49df6d272..22bf187a80e72 100644 --- a/pkg/disttask/importinto/task_executor.go +++ b/pkg/disttask/importinto/task_executor.go @@ -462,6 +462,7 @@ func (e *writeAndIngestStepExecutor) Cleanup(_ context.Context) (err error) { type postProcessStepExecutor struct { taskexecutor.EmptyStepExecutor taskID int64 + store tidbkv.Storage taskMeta *TaskMeta logger *zap.Logger } @@ -470,9 +471,10 @@ var _ execute.StepExecutor = &postProcessStepExecutor{} // NewPostProcessStepExecutor creates a new post process step executor. // exported for testing. -func NewPostProcessStepExecutor(taskID int64, taskMeta *TaskMeta, logger *zap.Logger) execute.StepExecutor { +func NewPostProcessStepExecutor(taskID int64, store tidbkv.Storage, taskMeta *TaskMeta, logger *zap.Logger) execute.StepExecutor { return &postProcessStepExecutor{ taskID: taskID, + store: store, taskMeta: taskMeta, logger: logger, } @@ -491,7 +493,7 @@ func (p *postProcessStepExecutor) RunSubtask(ctx context.Context, subtask *proto failpoint.Inject("waitBeforePostProcess", func() { time.Sleep(5 * time.Second) }) - return postProcess(ctx, p.taskMeta, &stepMeta, logger) + return postProcess(ctx, p.store, p.taskMeta, &stepMeta, logger) } type importExecutor struct { @@ -563,7 +565,7 @@ func (e *importExecutor) GetStepExecutor(task *proto.Task, stepResource *proto.S store: e.store, }, nil case proto.ImportStepPostProcess: - return NewPostProcessStepExecutor(task.ID, &taskMeta, logger), nil + return NewPostProcessStepExecutor(task.ID, e.store, &taskMeta, logger), nil default: return nil, errors.Errorf("unknown step %d for import task %d", task.Step, task.ID) } diff --git a/pkg/disttask/importinto/task_executor_testkit_test.go b/pkg/disttask/importinto/task_executor_testkit_test.go index 49924751fb5de..f2130f7fa505c 100644 --- a/pkg/disttask/importinto/task_executor_testkit_test.go +++ b/pkg/disttask/importinto/task_executor_testkit_test.go @@ -70,7 +70,7 @@ func TestPostProcessStepExecutor(t *testing.T) { bytes, err := json.Marshal(stepMeta) require.NoError(t, err) - executor := importinto.NewPostProcessStepExecutor(1, taskMeta, zap.NewExample()) + executor := importinto.NewPostProcessStepExecutor(1, store, taskMeta, zap.NewExample()) err = executor.RunSubtask(context.Background(), &proto.Subtask{Meta: bytes}) require.NoError(t, err) @@ -79,17 +79,17 @@ func TestPostProcessStepExecutor(t *testing.T) { stepMeta.Checksum[-1] = tmp bytes, err = json.Marshal(stepMeta) require.NoError(t, err) - executor = importinto.NewPostProcessStepExecutor(1, taskMeta, zap.NewExample()) + executor = importinto.NewPostProcessStepExecutor(1, store, taskMeta, zap.NewExample()) err = executor.RunSubtask(context.Background(), &proto.Subtask{Meta: bytes}) require.ErrorContains(t, err, "checksum mismatched remote vs local") taskMeta.Plan.Checksum = config.OpLevelOptional - executor = importinto.NewPostProcessStepExecutor(1, taskMeta, zap.NewExample()) + executor = importinto.NewPostProcessStepExecutor(1, store, taskMeta, zap.NewExample()) err = executor.RunSubtask(context.Background(), &proto.Subtask{Meta: bytes}) require.NoError(t, err) taskMeta.Plan.Checksum = config.OpLevelOff - executor = importinto.NewPostProcessStepExecutor(1, taskMeta, zap.NewExample()) + executor = importinto.NewPostProcessStepExecutor(1, store, taskMeta, zap.NewExample()) err = executor.RunSubtask(context.Background(), &proto.Subtask{Meta: bytes}) require.NoError(t, err) } diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index 28eb811b7d9b1..3868d130c4c90 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -248,7 +248,7 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i if is := do.infoCache.GetByVersion(neededSchemaVersion); is != nil { // try to insert here as well to correct the schemaTs if previous is wrong // the insert method check if schemaTs is zero - do.infoCache.Insert(is, uint64(schemaTs)) + do.infoCache.Insert(is, schemaTs) enableV2 := variable.SchemaCacheSize.Load() > 0 isV2 := infoschema.IsV2(is) @@ -271,10 +271,10 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i // 4. No regenrated schema diff. startTime := time.Now() if currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < LoadSchemaDiffVersionGapThreshold { - is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion) + is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion, schemaTs) if err == nil { infoschema_metrics.LoadSchemaDurationLoadDiff.Observe(time.Since(startTime).Seconds()) - do.infoCache.Insert(is, uint64(schemaTs)) + do.infoCache.Insert(is, schemaTs) logutil.BgLogger().Info("diff load InfoSchema success", zap.Int64("currentSchemaVersion", currentSchemaVersion), zap.Int64("neededSchemaVersion", neededSchemaVersion), @@ -315,13 +315,13 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i zap.Int64("neededSchemaVersion", neededSchemaVersion), zap.Duration("start time", time.Since(startTime))) - is := newISBuilder.Build() - do.infoCache.Insert(is, uint64(schemaTs)) + is := newISBuilder.Build(schemaTs) + do.infoCache.Insert(is, schemaTs) return is, false, currentSchemaVersion, nil, nil } // Returns the timestamp of a schema version, which is the commit timestamp of the schema diff -func (do *Domain) getTimestampForSchemaVersionWithNonEmptyDiff(m *meta.Meta, version int64, startTS uint64) (int64, error) { +func (do *Domain) getTimestampForSchemaVersionWithNonEmptyDiff(m *meta.Meta, version int64, startTS uint64) (uint64, error) { tikvStore, ok := do.Store().(helper.Storage) if ok { newHelper := helper.NewHelper(tikvStore) @@ -332,7 +332,7 @@ func (do *Domain) getTimestampForSchemaVersionWithNonEmptyDiff(m *meta.Meta, ver if mvccResp == nil || mvccResp.Info == nil || len(mvccResp.Info.Writes) == 0 { return 0, errors.Errorf("There is no Write MVCC info for the schema version") } - return int64(mvccResp.Info.Writes[0].CommitTs), nil + return mvccResp.Info.Writes[0].CommitTs, nil } return 0, errors.Errorf("cannot get store from domain") } @@ -439,7 +439,7 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don // Return true if the schema is loaded successfully. // Return false if the schema can not be loaded by schema diff, then we need to do full load. // The second returned value is the delta updated table and partition IDs. -func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) { +func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64, schemaTS uint64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) { var diffs []*model.SchemaDiff for usedVersion < newVersion { usedVersion++ @@ -480,7 +480,7 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64 } } - is := builder.Build() + is := builder.Build(schemaTS) relatedChange := transaction.RelatedSchemaChange{} relatedChange.PhyTblIDS = phyTblIDs relatedChange.ActionTypes = actions diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index 56e4ff37efd8c..56c7f406bd664 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -172,6 +172,7 @@ go_library( "//pkg/statistics/handle", "//pkg/statistics/handle/cache", "//pkg/statistics/handle/globalstats", + "//pkg/statistics/handle/storage", "//pkg/statistics/handle/util", "//pkg/store/driver/backoff", "//pkg/store/driver/txn", @@ -217,6 +218,7 @@ go_library( "//pkg/util/plancodec", "//pkg/util/printer", "//pkg/util/ranger", + "//pkg/util/redact", "//pkg/util/replayer", "//pkg/util/resourcegrouptag", "//pkg/util/rowDecoder", diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index c5346e298ca84..cd45db602ed7b 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -63,6 +63,7 @@ import ( "github.com/pingcap/tidb/pkg/util/hint" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/plancodec" + "github.com/pingcap/tidb/pkg/util/redact" "github.com/pingcap/tidb/pkg/util/replayer" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/pingcap/tidb/pkg/util/stmtsummary" @@ -90,7 +91,7 @@ type recordSet struct { fields []*ast.ResultField executor exec.Executor stmt *ExecStmt - lastErr error + lastErrs []error txnStartTS uint64 once sync.Once } @@ -156,7 +157,7 @@ func (a *recordSet) Next(ctx context.Context, req *chunk.Chunk) (err error) { err = a.stmt.next(ctx, a.executor, req) if err != nil { - a.lastErr = err + a.lastErrs = append(a.lastErrs, err) return err } numRows := req.NumRows() @@ -194,7 +195,7 @@ func (a *recordSet) Finish() error { } }) if err != nil { - a.lastErr = err + a.lastErrs = append(a.lastErrs, err) } return err } @@ -204,13 +205,13 @@ func (a *recordSet) Close() error { if err != nil { logutil.BgLogger().Error("close recordSet error", zap.Error(err)) } - a.stmt.CloseRecordSet(a.txnStartTS, a.lastErr) + a.stmt.CloseRecordSet(a.txnStartTS, errors.Join(a.lastErrs...)) return err } // OnFetchReturned implements commandLifeCycle#OnFetchReturned func (a *recordSet) OnFetchReturned() { - a.stmt.LogSlowQuery(a.txnStartTS, a.lastErr == nil, true) + a.stmt.LogSlowQuery(a.txnStartTS, len(a.lastErrs) == 0, true) } // ExecStmt implements the sqlexec.Statement interface, it builds a planner.Plan to an sqlexec.Statement. @@ -965,7 +966,7 @@ func (a *ExecStmt) handlePessimisticDML(ctx context.Context, e exec.Executor) (e // If it's not a retryable error, rollback current transaction instead of rolling back current statement like // in normal transactions, because we cannot locate and rollback the statement that leads to the lock error. // This is too strict, but since the feature is not for everyone, it's the easiest way to guarantee safety. - stmtText := parser.Normalize(a.OriginText(), sctx.GetSessionVars().EnableRedactNew) + stmtText := parser.Normalize(a.OriginText(), sctx.GetSessionVars().EnableRedactLog) logutil.Logger(ctx).Info("Transaction abort for the safety of lazy uniqueness check. "+ "Note this may not be a uniqueness violation.", zap.Error(err), @@ -1952,7 +1953,8 @@ func (a *ExecStmt) SummaryStmt(succ bool) { func (a *ExecStmt) GetTextToLog(keepHint bool) string { var sql string sessVars := a.Ctx.GetSessionVars() - if sessVars.EnableRedactLog { + rmode := sessVars.EnableRedactLog + if rmode == errors.RedactLogEnable { if keepHint { sql = parser.NormalizeKeepHint(sessVars.StmtCtx.OriginalSQL) } else { @@ -1961,7 +1963,7 @@ func (a *ExecStmt) GetTextToLog(keepHint bool) string { } else if sensitiveStmt, ok := a.StmtNode.(ast.SensitiveStmtNode); ok { sql = sensitiveStmt.SecureText() } else { - sql = sessVars.StmtCtx.OriginalSQL + sessVars.PlanCacheParams.String() + sql = redact.String(rmode, sessVars.StmtCtx.OriginalSQL+sessVars.PlanCacheParams.String()) } return sql } diff --git a/pkg/executor/aggfuncs/func_avg.go b/pkg/executor/aggfuncs/func_avg.go index 893754878c1c8..dd9e3fc953c7a 100644 --- a/pkg/executor/aggfuncs/func_avg.go +++ b/pkg/executor/aggfuncs/func_avg.go @@ -81,7 +81,7 @@ func (*baseAvgDecimal) ResetPartialResult(pr PartialResult) { p.count = int64(0) } -func (e *baseAvgDecimal) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error { +func (e *baseAvgDecimal) AppendFinalResult2Chunk(ctx AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error { p := (*partialResult4AvgDecimal)(pr) if p.count == 0 { chk.AppendNull(e.ordinal) @@ -89,7 +89,7 @@ func (e *baseAvgDecimal) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr Part } decimalCount := types.NewDecFromInt(p.count) finalResult := new(types.MyDecimal) - err := types.DecimalDiv(&p.sum, decimalCount, finalResult, types.DivFracIncr) + err := types.DecimalDiv(&p.sum, decimalCount, finalResult, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err != nil { return err } @@ -277,7 +277,7 @@ func (e *avgOriginal4DistinctDecimal) UpdatePartialResult(sctx AggFuncUpdateCont return memDelta, nil } -func (e *avgOriginal4DistinctDecimal) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error { +func (e *avgOriginal4DistinctDecimal) AppendFinalResult2Chunk(ctx AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error { p := (*partialResult4AvgDistinctDecimal)(pr) if p.count == 0 { chk.AppendNull(e.ordinal) @@ -285,7 +285,7 @@ func (e *avgOriginal4DistinctDecimal) AppendFinalResult2Chunk(_ AggFuncUpdateCon } decimalCount := types.NewDecFromInt(p.count) finalResult := new(types.MyDecimal) - err := types.DecimalDiv(&p.sum, decimalCount, finalResult, types.DivFracIncr) + err := types.DecimalDiv(&p.sum, decimalCount, finalResult, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err != nil { return err } diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index 7b0f418b97357..676bfebf13d01 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -2585,13 +2585,7 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown( PartitionName: task.PartitionName, SampleRateReason: sampleRateReason, } - var concurrency int - if b.ctx.GetSessionVars().InRestrictedSQL { - // In restricted SQL, we use the default value of DistSQLScanConcurrency. it is copied from tidb_sysproc_scan_concurrency. - concurrency = b.ctx.GetSessionVars().DistSQLScanConcurrency() - } else { - concurrency = b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency() - } + concurrency := b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency() base := baseAnalyzeExec{ ctx: b.ctx, tableID: task.TableID, @@ -2725,13 +2719,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown( failpoint.Inject("injectAnalyzeSnapshot", func(val failpoint.Value) { startTS = uint64(val.(int)) }) - var concurrency int - if b.ctx.GetSessionVars().InRestrictedSQL { - // In restricted SQL, we use the default value of DistSQLScanConcurrency. it is copied from tidb_sysproc_scan_concurrency. - concurrency = b.ctx.GetSessionVars().DistSQLScanConcurrency() - } else { - concurrency = b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency() - } + concurrency := b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency() base := baseAnalyzeExec{ ctx: b.ctx, tableID: task.TableID, diff --git a/pkg/executor/cluster_table_test.go b/pkg/executor/cluster_table_test.go index 9d5deb209fac0..4f8466b5867a0 100644 --- a/pkg/executor/cluster_table_test.go +++ b/pkg/executor/cluster_table_test.go @@ -15,10 +15,12 @@ package executor_test import ( + "compress/gzip" "context" "fmt" "net" "os" + "strings" "testing" "time" @@ -208,71 +210,78 @@ select 10;` fileName2 := "tidb-slow-20236-2020-02-16T19-04-05.01.log" fileName3 := "tidb-slow-20236-2020-02-17T18-00-05.01.log" fileName4 := "tidb-slow-20236.log" - fileNames := []string{fileName0, fileName1, fileName2, fileName3, fileName4} defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { conf.Log.SlowQueryFile = fileName4 }) - prepareLogs(t, logData, fileNames) - defer func() { - removeFiles(t, fileNames) - }() - tk := testkit.NewTestKit(t, store) - loc, err := time.LoadLocation("Asia/Shanghai") - require.NoError(t, err) - tk.Session().GetSessionVars().TimeZone = loc - tk.MustExec("use information_schema") - cases := []struct { - prepareSQL string - sql string - result []string - }{ - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000'", - result: []string{"2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000' order by time desc", - result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:01:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time", - result: []string{"2020-02-15 18:00:01.000000", "2020-02-15 19:00:05.000000", "2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:01:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time desc", - result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000", "2020-02-15 19:00:05.000000", "2020-02-15 18:00:01.000000"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select count(*) from cluster_slow_query where time > '2020-02-15 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc", - result: []string{"9"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select count(*) from cluster_slow_query where (time > '2020-02-16 18:00:00' and time < '2020-05-14 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-17 20:00:00')", - result: []string{"6"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select count(*) from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-02-17 20:00:00.000000' order by time desc", - result: []string{"5"}, - }, - { - prepareSQL: "set @@time_zone = '+08:00'", - sql: "select time from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc limit 3", - result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"}, - }, - } - for _, cas := range cases { - if len(cas.prepareSQL) > 0 { - tk.MustExec(cas.prepareSQL) + for k := 0; k < 2; k++ { + // k = 0 for normal files + // k = 1 for compressed files + var fileNames []string + if k == 0 { + fileNames = []string{fileName0, fileName1, fileName2, fileName3, fileName4} + } else { + fileNames = []string{fileName0 + ".gz", fileName1 + ".gz", fileName2 + ".gz", fileName3 + ".gz", fileName4} + } + prepareLogs(t, logData, fileNames) + tk := testkit.NewTestKit(t, store) + loc, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + tk.Session().GetSessionVars().TimeZone = loc + tk.MustExec("use information_schema") + cases := []struct { + prepareSQL string + sql string + result []string + }{ + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000'", + result: []string{"2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000' order by time desc", + result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time", + result: []string{"2020-02-15 18:00:01.000000", "2020-02-15 19:00:05.000000", "2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time desc", + result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000", "2020-02-15 19:00:05.000000", "2020-02-15 18:00:01.000000"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select count(*) from cluster_slow_query where time > '2020-02-15 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc", + result: []string{"9"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select count(*) from cluster_slow_query where (time > '2020-02-16 18:00:00' and time < '2020-05-14 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-17 20:00:00')", + result: []string{"6"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select count(*) from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-02-17 20:00:00.000000' order by time desc", + result: []string{"5"}, + }, + { + prepareSQL: "set @@time_zone = '+08:00'", + sql: "select time from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc limit 3", + result: []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"}, + }, + } + for _, cas := range cases { + if len(cas.prepareSQL) > 0 { + tk.MustExec(cas.prepareSQL) + } + tk.MustQuery(cas.sql).Check(testkit.RowsWithSep("|", cas.result...)) } - tk.MustQuery(cas.sql).Check(testkit.RowsWithSep("|", cas.result...)) + removeFiles(t, fileNames) } } @@ -308,12 +317,26 @@ func TestSQLDigestTextRetriever(t *testing.T) { } func prepareLogs(t *testing.T, logData []string, fileNames []string) { + writeFile := func(file string, data string) { + if strings.HasSuffix(file, ".gz") { + f, err := os.Create(file) + require.NoError(t, err) + gz := gzip.NewWriter(f) + _, err = gz.Write([]byte(data)) + require.NoError(t, err) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + } else { + f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + require.NoError(t, err) + _, err = f.Write([]byte(data)) + require.NoError(t, err) + require.NoError(t, f.Close()) + } + } + for i, log := range logData { - f, err := os.OpenFile(fileNames[i], os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - require.NoError(t, err) - _, err = f.Write([]byte(log)) - require.NoError(t, err) - require.NoError(t, f.Close()) + writeFile(fileNames[i], log) } } diff --git a/pkg/executor/compact_table.go b/pkg/executor/compact_table.go index 33b4b19cd4348..ea9f7f43bf36b 100644 --- a/pkg/executor/compact_table.go +++ b/pkg/executor/compact_table.go @@ -15,7 +15,6 @@ package executor import ( - "bytes" "context" "encoding/hex" "time" @@ -317,7 +316,7 @@ func (task *storeCompactTask) compactOnePhysicalTable(physicalTableID int64) (bo // Let's send more compact requests, as there are remaining data to compact. lastEndKey := resp.GetCompactedEndKey() - if len(lastEndKey) == 0 || bytes.Compare(lastEndKey, startKey) <= 0 { + if len(lastEndKey) == 0 { // The TiFlash server returned an invalid compacted end key. // This is unexpected... warn := errors.NewNoStackErrorf("compact on store %s failed: internal error (check logs for details)", task.targetStore.Address) diff --git a/pkg/executor/delete.go b/pkg/executor/delete.go index 330f39192ec57..cbd328e14dafe 100644 --- a/pkg/executor/delete.go +++ b/pkg/executor/delete.go @@ -144,6 +144,11 @@ func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error { rowCount++ } chk = chunk.Renew(chk, e.MaxChunkSize()) + if txn, _ := e.Ctx().Txn(false); txn != nil { + if err := txn.MayFlush(); err != nil { + return err + } + } } return nil @@ -222,6 +227,11 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error { } } chk = exec.TryNewCacheChunk(e.Children(0)) + if txn, _ := e.Ctx().Txn(false); txn != nil { + if err := txn.MayFlush(); err != nil { + return err + } + } } return e.removeRowsInTblRowMap(tblRowMap) diff --git a/pkg/executor/importer/BUILD.bazel b/pkg/executor/importer/BUILD.bazel index 3144f19be5e06..66f50d2648ae9 100644 --- a/pkg/executor/importer/BUILD.bazel +++ b/pkg/executor/importer/BUILD.bazel @@ -76,7 +76,6 @@ go_library( "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_log//:log", "@com_github_prometheus_client_golang//prometheus", - "@com_github_tikv_client_go_v2//config", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//:client", @@ -157,7 +156,6 @@ go_test( "@com_github_pingcap_log//:log", "@com_github_prometheus_client_golang//prometheus", "@com_github_stretchr_testify//require", - "@com_github_tikv_client_go_v2//config", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//:client", diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go index 3cca6969c7e40..a9351ad74c264 100644 --- a/pkg/executor/importer/import.go +++ b/pkg/executor/importer/import.go @@ -39,7 +39,6 @@ import ( "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/disttask/framework/handle" "github.com/pingcap/tidb/pkg/expression" - tidbkv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" pformat "github.com/pingcap/tidb/pkg/parser/format" @@ -60,7 +59,6 @@ import ( "github.com/pingcap/tidb/pkg/util/filter" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/stringutil" - kvconfig "github.com/tikv/client-go/v2/config" pd "github.com/tikv/pd/client" "go.uber.org/zap" ) @@ -171,9 +169,6 @@ func (t DataSourceType) String() string { } var ( - // GetKVStore returns a kv.Storage. - // kv encoder of physical mode needs it. - GetKVStore func(path string, tls kvconfig.Security) (tidbkv.Storage, error) // NewClientWithContext returns a kv.Client. NewClientWithContext = pd.NewClientWithContext ) @@ -1276,13 +1271,13 @@ func (e *LoadDataController) toMyDumpFiles() []mydump.FileInfo { } // IsLocalSort returns true if we sort data on local disk. -func (e *LoadDataController) IsLocalSort() bool { - return e.Plan.CloudStorageURI == "" +func (p *Plan) IsLocalSort() bool { + return p.CloudStorageURI == "" } // IsGlobalSort returns true if we sort data on global storage. -func (e *LoadDataController) IsGlobalSort() bool { - return !e.IsLocalSort() +func (p *Plan) IsGlobalSort() bool { + return !p.IsLocalSort() } // CreateColAssignExprs creates the column assignment expressions using session context. diff --git a/pkg/executor/importer/importer_testkit_test.go b/pkg/executor/importer/importer_testkit_test.go index 25ab01fe68d67..1c79171ffdafe 100644 --- a/pkg/executor/importer/importer_testkit_test.go +++ b/pkg/executor/importer/importer_testkit_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ngaut/pools" - "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/lightning/checkpoints" "github.com/pingcap/tidb/br/pkg/lightning/common" @@ -47,7 +46,6 @@ import ( "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/stretchr/testify/require" - kvconfig "github.com/tikv/client-go/v2/config" "go.etcd.io/etcd/tests/v3/integration" "go.uber.org/mock/gomock" "go.uber.org/zap" @@ -200,10 +198,6 @@ func TestPostProcess(t *testing.T) { return tk.Session(), nil }, 1, 1, time.Second) defer pool.Close() - bak := importer.GetKVStore - defer func() { - importer.GetKVStore = bak - }() tk.MustExec("create database db") tk.MustExec("create table db.tb(id int primary key)") @@ -231,19 +225,11 @@ func TestPostProcess(t *testing.T) { localChecksum = verify.NewKVGroupChecksumForAdd() localChecksum.AddRawGroup(verify.DataKVGroupID, 1, 1, 1) require.NoError(t, importer.PostProcess(ctx, tk.Session(), nil, plan, localChecksum, logger)) - // get KV store failed - importer.GetKVStore = func(path string, tls kvconfig.Security) (kv.Storage, error) { - return nil, errors.New("mock get kv store failed") - } + // rebase success tk.MustExec("create table db.tb2(id int auto_increment primary key)") table, err = do.InfoSchema().TableByName(model.NewCIStr("db"), model.NewCIStr("tb2")) require.NoError(t, err) plan.TableInfo, plan.DesiredTableInfo = table.Meta(), table.Meta() - require.ErrorContains(t, importer.PostProcess(ctx, tk.Session(), nil, plan, localChecksum, logger), "mock get kv store failed") - // rebase success - importer.GetKVStore = func(path string, tls kvconfig.Security) (kv.Storage, error) { - return store, nil - } integration.BeforeTestExternal(t) testEtcdCluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { diff --git a/pkg/executor/importer/precheck.go b/pkg/executor/importer/precheck.go index cb9b5ed45ae17..6a2f208318e36 100644 --- a/pkg/executor/importer/precheck.go +++ b/pkg/executor/importer/precheck.go @@ -53,6 +53,11 @@ func (e *LoadDataController) CheckRequirements(ctx context.Context, conn sqlexec if err := e.checkTotalFileSize(); err != nil { return err } + // run global sort with < 16 thread might OOM on merge step + // TODO: remove this limit after control memory usage. + if e.IsGlobalSort() && e.ThreadCnt < 16 { + return exeerrors.ErrLoadDataPreCheckFailed.FastGenByArgs("global sort requires at least 16 threads") + } } if err := e.checkTableEmpty(ctx, conn); err != nil { return err diff --git a/pkg/executor/importer/precheck_test.go b/pkg/executor/importer/precheck_test.go index 59bdb8b0c948f..db3a7532b4bea 100644 --- a/pkg/executor/importer/precheck_test.go +++ b/pkg/executor/importer/precheck_test.go @@ -79,6 +79,7 @@ func TestCheckRequirements(t *testing.T) { tableObj, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) + // source data file size = 0 c := &importer.LoadDataController{ Plan: &importer.Plan{ DBName: "test", @@ -88,8 +89,18 @@ func TestCheckRequirements(t *testing.T) { } require.ErrorIs(t, c.CheckRequirements(ctx, conn), exeerrors.ErrLoadDataPreCheckFailed) - // now checkTotalFileSize pass, and try next pre-check item + // make checkTotalFileSize pass c.TotalFileSize = 1 + // global sort with thread count < 16 + c.ThreadCnt = 15 + c.CloudStorageURI = "s3://test" + err = c.CheckRequirements(ctx, conn) + require.ErrorIs(t, err, exeerrors.ErrLoadDataPreCheckFailed) + require.ErrorContains(t, err, "global sort requires at least 16 threads") + + // reset fields, make global sort thread check pass + c.ThreadCnt = 1 + c.CloudStorageURI = "" // non-empty table _, err = conn.Execute(ctx, "insert into test.t values(1)") require.NoError(t, err) @@ -155,6 +166,7 @@ func TestCheckRequirements(t *testing.T) { require.NoError(t, c.CheckRequirements(ctx, conn)) // with global sort + c.Plan.ThreadCnt = 16 c.Plan.CloudStorageURI = ":" require.ErrorIs(t, c.CheckRequirements(ctx, conn), exeerrors.ErrLoadDataInvalidURI) c.Plan.CloudStorageURI = "sdsdsdsd://sdsdsdsd" diff --git a/pkg/executor/importer/table_import.go b/pkg/executor/importer/table_import.go index 6afb4c089c2da..b4e702d7740f2 100644 --- a/pkg/executor/importer/table_import.go +++ b/pkg/executor/importer/table_import.go @@ -16,7 +16,6 @@ package importer import ( "context" - "fmt" "io" "math" "net" @@ -127,19 +126,6 @@ func prepareSortDir(e *LoadDataController, id string, tidbCfg *tidb.Config) (str return sortDir, nil } -// GetCachedKVStoreFrom gets a cached kv store from PD address. -// Callers should NOT close the kv store. -func GetCachedKVStoreFrom(pdAddr string, tls *common.TLS) (tidbkv.Storage, error) { - // Disable GC because TiDB enables GC already. - keySpaceName := tidb.GetGlobalKeyspaceName() - // the kv store we get is a cached store, so we can't close it. - kvStore, err := GetKVStore(fmt.Sprintf("tikv://%s?disableGC=true&keyspaceName=%s", pdAddr, keySpaceName), tls.ToTiKVSecurityConfig()) - if err != nil { - return nil, errors.Trace(err) - } - return kvStore, nil -} - // GetRegionSplitSizeKeys gets the region split size and keys from PD. func GetRegionSplitSizeKeys(ctx context.Context) (regionSplitSize int64, regionSplitKeys int64, err error) { tidbCfg := tidb.GetGlobalConfig() @@ -517,6 +503,9 @@ func (ti *TableImporter) OpenDataEngine(ctx context.Context, engineID int32) (*b func (ti *TableImporter) ImportAndCleanup(ctx context.Context, closedEngine *backend.ClosedEngine) (int64, error) { var kvCount int64 importErr := closedEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys) + if common.ErrFoundDuplicateKeys.Equal(importErr) { + importErr = local.ConvertToErrFoundConflictRecords(importErr, ti.encTable) + } if closedEngine.GetID() != common.IndexEngineID { // todo: change to a finer-grain progress later. // each row is encoded into 1 data key @@ -605,6 +594,9 @@ func (ti *TableImporter) CheckDiskQuota(ctx context.Context) { int64(config.SplitRegionSize)*int64(config.MaxSplitRegionSizeRatio), int64(config.SplitRegionKeys)*int64(config.MaxSplitRegionSizeRatio), ); err != nil { + if common.ErrFoundDuplicateKeys.Equal(err) { + err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + } importErr = multierr.Append(importErr, err) } } @@ -699,6 +691,9 @@ func (ti *TableImporter) ImportSelectedRows(ctx context.Context, se sessionctx.C failpoint.Return(nil, errors.New("mock import from select error")) }) if err = closedDataEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys); err != nil { + if common.ErrFoundDuplicateKeys.Equal(err) { + err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + } return nil, err } dataKVCount := ti.backend.GetImportedKVCount(closedDataEngine.GetUUID()) @@ -708,6 +703,9 @@ func (ti *TableImporter) ImportSelectedRows(ctx context.Context, se sessionctx.C return nil, err } if err = closedIndexEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys); err != nil { + if common.ErrFoundDuplicateKeys.Equal(err) { + err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + } return nil, err } @@ -768,7 +766,7 @@ func PostProcess( callLog.End(zap.ErrorLevel, err) }() - if err = RebaseAllocatorBases(ctx, maxIDs, plan, logger); err != nil { + if err = RebaseAllocatorBases(ctx, se.GetStore(), maxIDs, plan, logger); err != nil { return err } @@ -789,7 +787,7 @@ func (r *autoIDRequirement) AutoIDClient() *autoid.ClientDiscover { } // RebaseAllocatorBases rebase the allocator bases. -func RebaseAllocatorBases(ctx context.Context, maxIDs map[autoid.AllocatorType]int64, plan *Plan, logger *zap.Logger) (err error) { +func RebaseAllocatorBases(ctx context.Context, kvStore tidbkv.Storage, maxIDs map[autoid.AllocatorType]int64, plan *Plan, logger *zap.Logger) (err error) { callLog := log.BeginTask(logger, "rebase allocators") defer func() { callLog.End(zap.ErrorLevel, err) @@ -812,11 +810,6 @@ func RebaseAllocatorBases(ctx context.Context, maxIDs map[autoid.AllocatorType]i return err2 } - // no need to close kvStore, since it's a cached store. - kvStore, err2 := GetCachedKVStoreFrom(tidbCfg.Path, tls) - if err2 != nil { - return errors.Trace(err2) - } addrs := strings.Split(tidbCfg.Path, ",") etcdCli, err := clientv3.New(clientv3.Config{ Endpoints: addrs, diff --git a/pkg/executor/insert.go b/pkg/executor/insert.go index 9b24962c8c1b4..8876180daf64d 100644 --- a/pkg/executor/insert.go +++ b/pkg/executor/insert.go @@ -120,7 +120,7 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { e.stats.CheckInsertTime += time.Since(start) } } - return nil + return txn.MayFlush() } func prefetchUniqueIndices(ctx context.Context, txn kv.Transaction, rows []toBeCheckedRow) (map[string][]byte, error) { diff --git a/pkg/executor/internal/builder/BUILD.bazel b/pkg/executor/internal/builder/BUILD.bazel index b1344b35468d8..93eeff4a64c6a 100644 --- a/pkg/executor/internal/builder/BUILD.bazel +++ b/pkg/executor/internal/builder/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//pkg/planner/context", "//pkg/planner/core", "//pkg/sessionctx", + "//pkg/sessionctx/variable", "//pkg/util/timeutil", "@com_github_pingcap_tipb//go-tipb", ], diff --git a/pkg/executor/internal/builder/builder_utils.go b/pkg/executor/internal/builder/builder_utils.go index ed2781bc4aeeb..79311cd133a68 100644 --- a/pkg/executor/internal/builder/builder_utils.go +++ b/pkg/executor/internal/builder/builder_utils.go @@ -20,6 +20,7 @@ import ( planctx "github.com/pingcap/tidb/pkg/planner/context" plannercore "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/timeutil" "github.com/pingcap/tipb/go-tipb" ) @@ -53,6 +54,10 @@ func ConstructDAGReq(ctx sessionctx.Context, plans []plannercore.PhysicalPlan, s dagReq.CollectExecutionSummaries = &collExec } dagReq.Flags = sc.PushDownFlags() + if ctx.GetSessionVars().GetDivPrecisionIncrement() != variable.DefDivPrecisionIncrement { + var divPrecIncr uint32 = uint32(ctx.GetSessionVars().GetDivPrecisionIncrement()) + dagReq.DivPrecisionIncrement = &divPrecIncr + } if storeType == kv.TiFlash { var executors []*tipb.Executor executors, err = ConstructTreeBasedDistExec(ctx.GetPlanCtx(), plans[0]) diff --git a/pkg/executor/replace.go b/pkg/executor/replace.go index d2f30e4333c8c..687ddaf00a333 100644 --- a/pkg/executor/replace.go +++ b/pkg/executor/replace.go @@ -188,7 +188,7 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error { } } e.memTracker.Consume(int64(txn.Size() - txnSize)) - return nil + return txn.MayFlush() } // Next implements the Executor Next interface. diff --git a/pkg/executor/set_test.go b/pkg/executor/set_test.go index 8532ddc776566..425b76a862ae7 100644 --- a/pkg/executor/set_test.go +++ b/pkg/executor/set_test.go @@ -1759,3 +1759,34 @@ func TestSetTopSQLVariables(t *testing.T) { tk.MustQuery("show variables like '%top_sql%'").Check(testkit.Rows("tidb_enable_top_sql OFF", "tidb_top_sql_max_meta_count 5000", "tidb_top_sql_max_time_series_count 20")) tk.MustQuery("show global variables like '%top_sql%'").Check(testkit.Rows("tidb_enable_top_sql OFF", "tidb_top_sql_max_meta_count 5000", "tidb_top_sql_max_time_series_count 20")) } + +func TestDivPrecisionIncrement(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + // Default is 4. + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("4")) + + tk.MustExec("set @@div_precision_increment = 4") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("4")) + // Min val is 0. + tk.MustExec("set @@div_precision_increment = -1") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("0")) + + tk.MustExec("set @@div_precision_increment = 0") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("0")) + + tk.MustExec("set @@div_precision_increment = 30") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("30")) + + tk.MustExec("set @@div_precision_increment = 8") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("8")) + + // Max val is 30. + tk.MustExec("set @@div_precision_increment = 31") + tk.MustQuery("select @@div_precision_increment;").Check(testkit.Rows("30")) + + // Test set global. + tk.MustExec("set global div_precision_increment = 4") +} diff --git a/pkg/executor/show_placement_test.go b/pkg/executor/show_placement_test.go index 9491db53ac0fe..4989afbcbbd07 100644 --- a/pkg/executor/show_placement_test.go +++ b/pkg/executor/show_placement_test.go @@ -47,6 +47,7 @@ func TestShowPlacement(t *testing.T) { tk.MustExec("create placement policy pa1 " + "PRIMARY_REGION=\"cn-east-1\" " + "REGIONS=\"cn-east-1,cn-east-2\"" + + "SURVIVAL_PREFERENCES=\"[zone, dc, host]\"" + "SCHEDULE=\"EVEN\"") defer tk.MustExec("drop placement policy pa1") @@ -80,26 +81,26 @@ func TestShowPlacement(t *testing.T) { defer tk.MustExec("drop table if exists db2.t2") tk.MustQuery("show placement").Check(testkit.Rows( - "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" NULL", + "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" NULL", "POLICY pa2 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" NULL", "POLICY pb1 CONSTRAINTS=\"[+disk=ssd]\" VOTERS=5 VOTER_CONSTRAINTS=\"[+region=bj]\" LEARNERS=3 LEARNER_CONSTRAINTS=\"[+region=sh]\" NULL", "DATABASE db2 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" PENDING", "TABLE db2.t2 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" PENDING", - "TABLE test.t1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" PENDING", - "TABLE test.t3 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" PENDING", + "TABLE test.t1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" PENDING", + "TABLE test.t3 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" PENDING", "TABLE test.t3 PARTITION p0 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" PENDING", - "TABLE test.t3 PARTITION p1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" PENDING", - "TABLE test.t3 PARTITION p2 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" PENDING", + "TABLE test.t3 PARTITION p1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" PENDING", + "TABLE test.t3 PARTITION p2 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" PENDING", )) tk.MustQuery("show placement like 'POLICY%'").Check(testkit.Rows( - "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" NULL", + "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" NULL", "POLICY pa2 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" NULL", "POLICY pb1 CONSTRAINTS=\"[+disk=ssd]\" VOTERS=5 VOTER_CONSTRAINTS=\"[+region=bj]\" LEARNERS=3 LEARNER_CONSTRAINTS=\"[+region=sh]\" NULL", )) tk.MustQuery("show placement like 'POLICY pa%'").Check(testkit.Rows( - "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" NULL", + "POLICY pa1 PRIMARY_REGION=\"cn-east-1\" REGIONS=\"cn-east-1,cn-east-2\" SCHEDULE=\"EVEN\" SURVIVAL_PREFERENCES=\"[zone, dc, host]\" NULL", "POLICY pa2 LEADER_CONSTRAINTS=\"[+region=us-east-1]\" FOLLOWERS=3 FOLLOWER_CONSTRAINTS=\"[+region=us-east-2]\" NULL", )) diff --git a/pkg/executor/show_stats.go b/pkg/executor/show_stats.go index 2da8ff961b612..4b018cc309987 100644 --- a/pkg/executor/show_stats.go +++ b/pkg/executor/show_stats.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/cardinality" "github.com/pingcap/tidb/pkg/statistics" + statsStorage "github.com/pingcap/tidb/pkg/statistics/handle/storage" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/collate" "github.com/tikv/client-go/v2/oracle" @@ -558,7 +559,8 @@ func (e *ShowExec) appendTableForStatsHealthy(dbName, tblName, partitionName str } func (e *ShowExec) fetchShowHistogramsInFlight() { - e.appendRow([]any{statistics.HistogramNeededItems.Length()}) + statsHandle := domain.GetDomain(e.Ctx()).StatsHandle() + e.appendRow([]any{statsStorage.CleanFakeItemsForShowHistInFlights(statsHandle)}) } func (e *ShowExec) fetchShowAnalyzeStatus(ctx context.Context) error { diff --git a/pkg/executor/show_stats_test.go b/pkg/executor/show_stats_test.go index 725d500cc2d9b..951e77662383e 100644 --- a/pkg/executor/show_stats_test.go +++ b/pkg/executor/show_stats_test.go @@ -282,7 +282,6 @@ func TestShowStatusSnapshot(t *testing.T) { tk.MustExec("drop database if exists test;") tk.MustExec("create database test;") tk.MustExec("use test;") - tk.MustExec("create table t (a int);") // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. safePointName := "tikv_gc_safe_point" @@ -293,13 +292,17 @@ func TestShowStatusSnapshot(t *testing.T) { UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) tk.MustExec(updateSafePoint) - snapshotTime := time.Now() - - tk.MustExec("drop table t;") - tk.MustQuery("show table status;").Check(testkit.Rows()) - tk.MustExec("set @@tidb_snapshot = '" + snapshotTime.Format("2006-01-02 15:04:05.999999") + "'") - result := tk.MustQuery("show table status;") - require.Equal(t, "t", result.Rows()[0][0]) + for _, cacheSize := range []int{1024, 0} { + tk.MustExec("set @@global.tidb_schema_cache_size = ?", cacheSize) + tk.MustExec("create table t (a int);") + snapshotTime := time.Now() + tk.MustExec("drop table t;") + tk.MustQuery("show table status;").Check(testkit.Rows()) + tk.MustExec("set @@tidb_snapshot = '" + snapshotTime.Format("2006-01-02 15:04:05.999999") + "'") + result := tk.MustQuery("show table status;") + require.Equal(t, "t", result.Rows()[0][0]) + tk.MustExec("set @@tidb_snapshot = null;") + } } func TestShowStatsExtended(t *testing.T) { diff --git a/pkg/executor/slow_query.go b/pkg/executor/slow_query.go index 77e231e6381f2..fb1e3373342e1 100644 --- a/pkg/executor/slow_query.go +++ b/pkg/executor/slow_query.go @@ -16,6 +16,7 @@ package executor import ( "bufio" + "compress/gzip" "context" "fmt" "io" @@ -138,18 +139,11 @@ func (e *slowQueryRetriever) initialize(ctx context.Context, sctx sessionctx.Con e.initialized = true e.files, err = e.getAllFiles(ctx, sctx, sctx.GetSessionVars().SlowQueryFile) if e.extractor.Desc { - e.reverseLogFiles() + slices.Reverse(e.files) } return err } -func (e *slowQueryRetriever) reverseLogFiles() { - for i := 0; i < len(e.files)/2; i++ { - j := len(e.files) - i - 1 - e.files[i], e.files[j] = e.files[j], e.files[i] - } -} - func (e *slowQueryRetriever) close() error { for _, f := range e.files { err := f.file.Close() @@ -169,10 +163,11 @@ type parsedSlowLog struct { err error } -func (e *slowQueryRetriever) getNextFile() *os.File { +func (e *slowQueryRetriever) getNextFile() *logFile { if e.fileIdx >= len(e.files) { return nil } + ret := &e.files[e.fileIdx] file := e.files[e.fileIdx].file e.fileIdx++ if e.stats != nil { @@ -183,33 +178,60 @@ func (e *slowQueryRetriever) getNextFile() *os.File { e.stats.readFileNum++ } } - return file + return ret } -func (e *slowQueryRetriever) getPreviousFile() *os.File { +func (e *slowQueryRetriever) getPreviousReader() (*bufio.Reader, error) { fileIdx := e.fileIdx // fileIdx refer to the next file which should be read // so we need to set fileIdx to fileIdx - 2 to get the previous file. fileIdx = fileIdx - 2 if fileIdx < 0 { - return nil + return nil, nil } - file := e.files[fileIdx].file - _, err := file.Seek(0, io.SeekStart) + file := e.files[fileIdx] + _, err := file.file.Seek(0, io.SeekStart) if err != nil { - return nil + return nil, err } - return file + var reader *bufio.Reader + if !file.compressed { + reader = bufio.NewReader(file.file) + } else { + gr, err := gzip.NewReader(file.file) + if err != nil { + return nil, err + } + reader = bufio.NewReader(gr) + } + return reader, nil } -func (e *slowQueryRetriever) parseDataForSlowLog(ctx context.Context, sctx sessionctx.Context) { - defer e.wg.Done() +func (e *slowQueryRetriever) getNextReader() (*bufio.Reader, error) { file := e.getNextFile() if file == nil { + return nil, nil + } + var reader *bufio.Reader + if !file.compressed { + reader = bufio.NewReader(file.file) + } else { + gr, err := gzip.NewReader(file.file) + if err != nil { + return nil, err + } + reader = bufio.NewReader(gr) + } + return reader, nil +} + +func (e *slowQueryRetriever) parseDataForSlowLog(ctx context.Context, sctx sessionctx.Context) { + defer e.wg.Done() + reader, _ := e.getNextReader() + if reader == nil { close(e.taskList) return } - reader := bufio.NewReader(file) e.parseSlowLog(ctx, sctx, reader, ParseSlowLogBatchSize) } @@ -303,12 +325,12 @@ func (e *slowQueryRetriever) getBatchLog(ctx context.Context, reader *bufio.Read if err != nil { if err == io.EOF { e.fileLine = 0 - file := e.getNextFile() - if file == nil { - return [][]string{log}, nil + newReader, err := e.getNextReader() + if newReader == nil || err != nil { + return [][]string{log}, err } offset.length = len(log) - reader.Reset(file) + reader.Reset(newReader) continue } return [][]string{log}, err @@ -330,9 +352,9 @@ func (e *slowQueryRetriever) getBatchLogForReversedScan(ctx context.Context, rea // reader maybe change when read previous file. inputReader := reader defer func() { - file := e.getNextFile() - if file != nil { - inputReader.Reset(file) + newReader, _ := e.getNextReader() + if newReader != nil { + inputReader.Reset(newReader) } }() var line string @@ -355,11 +377,10 @@ func (e *slowQueryRetriever) getBatchLogForReversedScan(ctx context.Context, rea return decomposedSlowLogTasks, nil } e.fileLine = 0 - file := e.getPreviousFile() - if file == nil { + reader, err = e.getPreviousReader() + if reader == nil || err != nil { return decomposeToSlowLogTasks(logs, num), nil } - reader = bufio.NewReader(file) scanPreviousFile = true continue } @@ -825,7 +846,8 @@ func ParseTime(s string) (time.Time, error) { type logFile struct { file *os.File // The opened file handle - start, end time.Time // The start/end time of the log file + start time.Time // The start time of the log file + compressed bool // The file is compressed or not } // getAllFiles is used to get all slow-log needed to parse, it is exported for test. @@ -873,6 +895,7 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co if !strings.HasPrefix(path, prefix) { return nil } + compressed := strings.HasSuffix(path, ".gz") if isCtxDone(ctx) { return ctx.Err() } @@ -888,7 +911,7 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co } }() // Get the file start time. - fileStartTime, err := e.getFileStartTime(ctx, file) + fileStartTime, err := e.getFileStartTime(ctx, file, compressed) if err != nil { return handleErr(err) } @@ -904,30 +927,34 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co return nil } - // Get the file end time. - fileEndTime, err := e.getFileEndTime(ctx, file) - if err != nil { - return handleErr(err) - } - end := types.NewTime(types.FromGoTime(fileEndTime), mysql.TypeDatetime, types.MaxFsp) - inTimeRanges := false - for _, tr := range e.checker.timeRanges { - if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) { - inTimeRanges = true - break + // If we want to get the end time from a compressed file, + // we need uncompress the whole file which is very slow and consume a lot of memeory. + if !compressed { + // Get the file end time. + fileEndTime, err := e.getFileEndTime(ctx, file) + if err != nil { + return handleErr(err) + } + end := types.NewTime(types.FromGoTime(fileEndTime), mysql.TypeDatetime, types.MaxFsp) + inTimeRanges := false + for _, tr := range e.checker.timeRanges { + if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) { + inTimeRanges = true + break + } + } + if !inTimeRanges { + return nil } - } - if !inTimeRanges { - return nil } _, err = file.Seek(0, io.SeekStart) if err != nil { return handleErr(err) } logFiles = append(logFiles, logFile{ - file: file, - start: fileStartTime, - end: fileEndTime, + file: file, + start: fileStartTime, + compressed: compressed, }) skip = true return nil @@ -942,16 +969,46 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co slices.SortFunc(logFiles, func(i, j logFile) int { return i.start.Compare(j.start) }) - return logFiles, err + // Assume no time range overlap in log files and remove unnecessary log files for compressed files. + var ret []logFile + for i, file := range logFiles { + if i == len(logFiles)-1 || !file.compressed { + ret = append(ret, file) + continue + } + start := types.NewTime(types.FromGoTime(logFiles[i].start), mysql.TypeDatetime, types.MaxFsp) + // use next file.start as endTime + end := types.NewTime(types.FromGoTime(logFiles[i+1].start), mysql.TypeDatetime, types.MaxFsp) + inTimeRanges := false + for _, tr := range e.checker.timeRanges { + if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) { + inTimeRanges = true + break + } + } + if inTimeRanges { + ret = append(ret, file) + } + } + return ret, err } -func (*slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File) (time.Time, error) { +func (*slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File, compressed bool) (time.Time, error) { var t time.Time _, err := file.Seek(0, io.SeekStart) if err != nil { return t, err } - reader := bufio.NewReader(file) + var reader *bufio.Reader + if !compressed { + reader = bufio.NewReader(file) + } else { + gr, err := gzip.NewReader(file) + if err != nil { + return t, err + } + reader = bufio.NewReader(gr) + } maxNum := 128 for { lineByte, err := getOneLine(reader) diff --git a/pkg/executor/slow_query_test.go b/pkg/executor/slow_query_test.go index 03d46d6c19393..54ec398bd56af 100644 --- a/pkg/executor/slow_query_test.go +++ b/pkg/executor/slow_query_test.go @@ -17,8 +17,10 @@ package executor import ( "bufio" "bytes" + "compress/gzip" "context" "fmt" + "math" "os" "runtime/pprof" "strings" @@ -61,7 +63,7 @@ func newSlowQueryRetriever() (*slowQueryRetriever, error) { if err != nil { return nil, err } - is := newISBuilder.Build() + is := newISBuilder.Build(math.MaxUint64) tbl, err := is.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableSlowQuery)) if err != nil { return nil, err @@ -367,133 +369,153 @@ select 7;` config.UpdateGlobal(func(conf *config.Config) { conf.Log.SlowQueryFile = fileName3 }) - fileNames := []string{fileName0, fileName1, fileName2, fileName3} - prepareLogs(t, logData, fileNames) - defer func() { - removeFiles(fileNames) - }() - - cases := []struct { - startTime string - endTime string - files []string - querys []string - }{ - { - startTime: "2020-02-15T18:00:00.000000+08:00", - endTime: "2020-02-17T20:00:00.000000+08:00", - files: []string{fileName1, fileName2, fileName3}, - querys: []string{ - "select 1;", - "select 2;", - "select 3;", - "select 4;", - "select 5;", - "select 6;", + for k := 0; k < 2; k++ { + // k = 0 for normal files + // k = 1 for compressed files + var fileNames []string + if k == 0 { + fileNames = []string{fileName0, fileName1, fileName2, fileName3} + } else { + fileNames = []string{fileName0 + ".gz", fileName1 + ".gz", fileName2 + ".gz", fileName3} + } + prepareLogs(t, logData, fileNames) + + cases := []struct { + startTime string + endTime string + files []string + querys []string + }{ + { + startTime: "2020-02-15T18:00:00.000000+08:00", + endTime: "2020-02-17T20:00:00.000000+08:00", + files: []string{fileName1, fileName2, fileName3}, + querys: []string{ + "select 1;", + "select 2;", + "select 3;", + "select 4;", + "select 5;", + "select 6;", + }, }, - }, - { - startTime: "2020-02-15T18:00:02.000000+08:00", - endTime: "2020-02-16T20:00:00.000000+08:00", - files: []string{fileName1, fileName2, fileName3}, - querys: []string{ - "select 2;", - "select 3;", - "select 4;", - "select 5;", + { + startTime: "2020-02-15T18:00:02.000000+08:00", + endTime: "2020-02-16T20:00:00.000000+08:00", + files: []string{fileName1, fileName2, fileName3}, + querys: []string{ + "select 2;", + "select 3;", + "select 4;", + "select 5;", + }, }, - }, - { - startTime: "2020-02-16T18:00:03.000000+08:00", - endTime: "2020-02-16T18:59:00.000000+08:00", - files: []string{fileName2}, - querys: []string{ - "select 4;", + { + startTime: "2020-02-16T18:00:03.000000+08:00", + endTime: "2020-02-16T18:59:00.000000+08:00", + files: []string{fileName2}, + querys: []string{ + "select 4;", + }, }, - }, - { - startTime: "2020-02-16T18:00:03.000000+08:00", - endTime: "2020-02-16T20:00:00.000000+08:00", - files: []string{fileName2, fileName3}, - querys: []string{ - "select 4;", - "select 5;", + { + startTime: "2020-02-16T18:00:03.000000+08:00", + endTime: "2020-02-16T20:00:00.000000+08:00", + files: []string{fileName2, fileName3}, + querys: []string{ + "select 4;", + "select 5;", + }, }, - }, - { - startTime: "2020-02-16T19:00:00.000000+08:00", - endTime: "2020-02-17T17:00:00.000000+08:00", - files: []string{fileName3}, - querys: []string{ - "select 5;", + { + startTime: "2020-02-16T19:00:00.000000+08:00", + endTime: "2020-02-17T17:00:00.000000+08:00", + files: []string{fileName3}, + querys: []string{ + "select 5;", + }, }, - }, - { - startTime: "2010-01-01T00:00:00.000000+08:00", - endTime: "2010-01-01T01:00:00.000000+08:00", - files: []string{}, - }, - { - startTime: "2020-03-01T00:00:00.000000+08:00", - endTime: "2010-03-01T01:00:00.000000+08:00", - files: []string{}, - }, - { - startTime: "", - endTime: "", - files: []string{fileName3}, - querys: []string{ - "select 5;", - "select 6;", - "select 7;", + { + startTime: "2010-01-01T00:00:00.000000+08:00", + endTime: "2010-01-01T01:00:00.000000+08:00", + files: []string{}, }, - }, - { - startTime: "2020-04-15T18:00:05.299063744+08:00", - endTime: "2020-04-15T18:00:05.299063744+08:00", - files: []string{fileName3}, - querys: []string{ - "select 7;", + { + startTime: "2020-03-01T00:00:00.000000+08:00", + endTime: "2010-03-01T01:00:00.000000+08:00", + files: []string{}, + }, + { + startTime: "", + endTime: "", + files: []string{fileName3}, + querys: []string{ + "select 5;", + "select 6;", + "select 7;", + }, + }, + { + startTime: "2020-04-15T18:00:05.299063744+08:00", + endTime: "2020-04-15T18:00:05.299063744+08:00", + files: []string{fileName3}, + querys: []string{ + "select 7;", + }, }, - }, - } - - loc, err := time.LoadLocation("Asia/Shanghai") - require.NoError(t, err) - sctx := mock.NewContext() - sctx.ResetSessionAndStmtTimeZone(loc) - sctx.GetSessionVars().SlowQueryFile = fileName3 - for i, cas := range cases { - extractor := &plannercore.SlowQueryExtractor{Enable: len(cas.startTime) > 0 && len(cas.endTime) > 0} - if extractor.Enable { - startTime, err := ParseTime(cas.startTime) - require.NoError(t, err) - endTime, err := ParseTime(cas.endTime) - require.NoError(t, err) - extractor.TimeRanges = []*plannercore.TimeRange{{StartTime: startTime, EndTime: endTime}} } - retriever, err := newSlowQueryRetriever() - require.NoError(t, err) - retriever.extractor = extractor - err = retriever.initialize(context.Background(), sctx) + + loc, err := time.LoadLocation("Asia/Shanghai") require.NoError(t, err) - comment := fmt.Sprintf("case id: %v", i) - require.Equal(t, len(retriever.files), len(cas.files), comment) - if len(retriever.files) > 0 { - reader := bufio.NewReader(retriever.files[0].file) - rows, err := parseLog(retriever, sctx, reader) + sctx := mock.NewContext() + sctx.ResetSessionAndStmtTimeZone(loc) + sctx.GetSessionVars().SlowQueryFile = fileName3 + for i, cas := range cases { + extractor := &plannercore.SlowQueryExtractor{Enable: len(cas.startTime) > 0 && len(cas.endTime) > 0} + if extractor.Enable { + startTime, err := ParseTime(cas.startTime) + require.NoError(t, err) + endTime, err := ParseTime(cas.endTime) + require.NoError(t, err) + extractor.TimeRanges = []*plannercore.TimeRange{{StartTime: startTime, EndTime: endTime}} + } + retriever, err := newSlowQueryRetriever() require.NoError(t, err) - require.Equal(t, len(rows), len(cas.querys), comment) - for i, row := range rows { - require.Equal(t, row[len(row)-1].GetString(), cas.querys[i], comment) + retriever.extractor = extractor + err = retriever.initialize(context.Background(), sctx) + require.NoError(t, err) + comment := fmt.Sprintf("compressed: %v, case id: %v", k, i) + if len(retriever.files) > 0 { + var reader *bufio.Reader + reader, err := retriever.getNextReader() + require.NoError(t, err, comment) + rows, err := parseLog(retriever, sctx, reader) + require.NoError(t, err, comment) + require.Equal(t, len(rows), len(cas.querys), comment) + for i, row := range rows { + require.Equal(t, row[len(row)-1].GetString(), cas.querys[i], comment) + } } - } - for i, file := range retriever.files { - require.Equal(t, file.file.Name(), cas.files[i]) - require.NoError(t, file.file.Close()) + if k == 0 { + require.Equal(t, len(retriever.files), len(cas.files), comment) + for i, file := range retriever.files { + require.Equal(t, file.file.Name(), cas.files[i], comment) + } + } else { + // for compressed file, sometimes it will contains one more file. + require.True(t, (len(retriever.files) == len(cas.files)) || (len(retriever.files) == len(cas.files)+1), comment) + var fileNames []string + for _, file := range retriever.files { + fileNames = append(fileNames, strings.TrimSuffix(file.file.Name(), ".gz")) + } + for _, file := range cas.files { + require.Contains(t, fileNames, file, comment) + } + } + require.NoError(t, retriever.close()) } - require.NoError(t, retriever.close()) + removeFiles(fileNames) } } @@ -719,11 +741,21 @@ func checkGoroutineExists(keyword string) bool { func prepareLogs(t *testing.T, logData []string, fileNames []string) { writeFile := func(file string, data string) { - f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - require.NoError(t, err) - _, err = f.Write([]byte(data)) - require.NoError(t, err) - require.NoError(t, f.Close()) + if strings.HasSuffix(file, ".gz") { + f, err := os.Create(file) + require.NoError(t, err) + gz := gzip.NewWriter(f) + _, err = gz.Write([]byte(data)) + require.NoError(t, err) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + } else { + f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + require.NoError(t, err) + _, err = f.Write([]byte(data)) + require.NoError(t, err) + require.NoError(t, f.Close()) + } } for i, log := range logData { diff --git a/pkg/executor/sortexec/parallel_sort_test.go b/pkg/executor/sortexec/parallel_sort_test.go index 520b0a95dc217..76afe38b799a5 100644 --- a/pkg/executor/sortexec/parallel_sort_test.go +++ b/pkg/executor/sortexec/parallel_sort_test.go @@ -36,6 +36,8 @@ func executeInFailpoint(t *testing.T, exe *sortexec.SortExec) { tmpCtx := context.Background() err := exe.Open(tmpCtx) require.NoError(t, err) + exe.IsUnparallel = false + exe.InitInParallelModeForTest() goRoutineWaiter := sync.WaitGroup{} goRoutineWaiter.Add(1) @@ -77,13 +79,14 @@ func parallelSortTest(t *testing.T, ctx *mock.Context, exe *sortexec.SortExec, s ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = true + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = true if exe == nil { exe = buildSortExec(ctx, sortCase, dataSource) } dataSource.PrepareChunks() - resultChunks := executeSortExecutor(t, exe) + resultChunks := executeSortExecutor(t, exe, true) err := exe.Close() require.NoError(t, err) @@ -96,7 +99,8 @@ func failpointTest(t *testing.T, ctx *mock.Context, exe *sortexec.SortExec, sort ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = true + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = true if exe == nil { exe = buildSortExec(ctx, sortCase, dataSource) } diff --git a/pkg/executor/sortexec/sort.go b/pkg/executor/sortexec/sort.go index d3dcb2a6f6137..d3c17886c6f91 100644 --- a/pkg/executor/sortexec/sort.go +++ b/pkg/executor/sortexec/sort.go @@ -136,7 +136,7 @@ func (e *SortExec) Open(ctx context.Context) error { e.diskTracker.AttachTo(e.Ctx().GetSessionVars().StmtCtx.DiskTracker) } - e.IsUnparallel = !e.Ctx().GetSessionVars().EnableParallelSort + e.IsUnparallel = true if e.IsUnparallel { e.Unparallel.Idx = 0 } else { @@ -153,6 +153,18 @@ func (e *SortExec) Open(ctx context.Context) error { return exec.Open(ctx, e.Children(0)) } +// InitInParallelModeForTest is a function for test +// After system variable is added, we can delete this function +func (e *SortExec) InitInParallelModeForTest() { + e.Parallel.workers = make([]*parallelSortWorker, e.Ctx().GetSessionVars().ExecutorConcurrency) + e.Parallel.chunkChannel = make(chan *chunk.Chunk, len(e.Parallel.workers)) + e.Parallel.sortedRowsIters = make([]*chunk.Iterator4Slice, len(e.Parallel.workers)) + e.Parallel.resultChannel = make(chan rowWithError, e.MaxChunkSize()) + for i := range e.Parallel.sortedRowsIters { + e.Parallel.sortedRowsIters[i] = chunk.NewIterator4Slice(nil) + } +} + // Next implements the Executor Next interface. // Sort constructs the result following these step in unparallel mode: // 1. Read as mush as rows into memory. diff --git a/pkg/executor/sortexec/sort_spill_test.go b/pkg/executor/sortexec/sort_spill_test.go index f0996b24fff52..ea1e6a03bd64e 100644 --- a/pkg/executor/sortexec/sort_spill_test.go +++ b/pkg/executor/sortexec/sort_spill_test.go @@ -163,10 +163,14 @@ func buildSortExec(ctx *mock.Context, sortCase *testutil.SortCase, dataSource *t return exe } -func executeSortExecutor(t *testing.T, exe *sortexec.SortExec) []*chunk.Chunk { +func executeSortExecutor(t *testing.T, exe *sortexec.SortExec, isParallelSort bool) []*chunk.Chunk { tmpCtx := context.Background() err := exe.Open(tmpCtx) require.NoError(t, err) + if isParallelSort { + exe.IsUnparallel = false + exe.InitInParallelModeForTest() + } resultChunks := make([]*chunk.Chunk, 0) chk := exec.NewFirstChunk(exe) @@ -219,11 +223,12 @@ func onePartitionAndAllDataInMemoryCase(t *testing.T, ctx *mock.Context, sortCas ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, 1048576) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = false + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = false schema := expression.NewSchema(sortCase.Columns()...) dataSource := buildDataSource(ctx, sortCase, schema) exe := buildSortExec(ctx, sortCase, dataSource) - resultChunks := executeSortExecutor(t, exe) + resultChunks := executeSortExecutor(t, exe, false) require.Equal(t, exe.GetSortPartitionListLenForTest(), 1) require.Equal(t, false, exe.IsSpillTriggeredInOnePartitionForTest(0)) @@ -241,14 +246,15 @@ func onePartitionAndAllDataInDiskCase(t *testing.T, ctx *mock.Context, sortCase ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, 50000) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = false + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = false schema := expression.NewSchema(sortCase.Columns()...) dataSource := buildDataSource(ctx, sortCase, schema) exe := buildSortExec(ctx, sortCase, dataSource) // To ensure that spill has been trigger before getting chunk, or we may get chunk from memory. failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/waitForSpill", `return(true)`) - resultChunks := executeSortExecutor(t, exe) + resultChunks := executeSortExecutor(t, exe, false) failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/waitForSpill", `return(false)`) require.Equal(t, exe.GetSortPartitionListLenForTest(), 1) @@ -270,14 +276,16 @@ func multiPartitionCase(t *testing.T, ctx *mock.Context, sortCase *testutil.Sort ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, 10000) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = false + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = false schema := expression.NewSchema(sortCase.Columns()...) dataSource := buildDataSource(ctx, sortCase, schema) exe := buildSortExec(ctx, sortCase, dataSource) + exe.IsUnparallel = true if enableFailPoint { failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/unholdSyncLock", `return(true)`) } - resultChunks := executeSortExecutor(t, exe) + resultChunks := executeSortExecutor(t, exe, false) if enableFailPoint { failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/unholdSyncLock", `return(false)`) } @@ -309,7 +317,8 @@ func inMemoryThenSpillCase(t *testing.T, ctx *mock.Context, sortCase *testutil.S ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, hardLimit) ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1) ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker) - ctx.GetSessionVars().EnableParallelSort = false + // TODO use variable to choose parallel mode after system variable is added + // ctx.GetSessionVars().EnableParallelSort = false schema := expression.NewSchema(sortCase.Columns()...) dataSource := buildDataSource(ctx, sortCase, schema) exe := buildSortExec(ctx, sortCase, dataSource) @@ -364,7 +373,7 @@ func TestFallBackAction(t *testing.T) { schema := expression.NewSchema(sortCase.Columns()...) dataSource := buildDataSource(ctx, sortCase, schema) exe := buildSortExec(ctx, sortCase, dataSource) - executeSortExecutor(t, exe) + executeSortExecutor(t, exe, false) err := exe.Close() require.NoError(t, err) diff --git a/pkg/executor/stale_txn_test.go b/pkg/executor/stale_txn_test.go index 2fad8edaff159..b8ca85ff2b6a6 100644 --- a/pkg/executor/stale_txn_test.go +++ b/pkg/executor/stale_txn_test.go @@ -892,6 +892,14 @@ func TestSetTransactionInfoSchema(t *testing.T) { ON DUPLICATE KEY UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) tk.MustExec(updateSafePoint) + + for _, cacheSize := range []int{1024, 0} { + tk.MustExec("set @@global.tidb_schema_cache_size = ?", cacheSize) + testSetTransactionInfoSchema(t, tk) + } +} + +func testSetTransactionInfoSchema(t *testing.T, tk *testkit.TestKit) { tk.MustExec("use test") tk.MustExec("drop table if exists t") defer tk.MustExec("drop table if exists t") diff --git a/pkg/executor/stmtsummary_test.go b/pkg/executor/stmtsummary_test.go index 37bac3548c9b5..f93e7d45e3e1f 100644 --- a/pkg/executor/stmtsummary_test.go +++ b/pkg/executor/stmtsummary_test.go @@ -16,6 +16,7 @@ package executor import ( "context" + "math" "os" "testing" "time" @@ -33,7 +34,7 @@ func TestStmtSummaryRetriverV2_TableStatementsSummary(t *testing.T) { data := infoschema.NewData() infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0) require.NoError(t, err) - infoSchema := infoSchemaBuilder.Build() + infoSchema := infoSchemaBuilder.Build(math.MaxUint64) table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummary)) require.NoError(t, err) columns := table.Meta().Columns @@ -77,7 +78,7 @@ func TestStmtSummaryRetriverV2_TableStatementsSummaryEvicted(t *testing.T) { data := infoschema.NewData() infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0) require.NoError(t, err) - infoSchema := infoSchemaBuilder.Build() + infoSchema := infoSchemaBuilder.Build(math.MaxUint64) table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryEvicted)) require.NoError(t, err) columns := table.Meta().Columns @@ -156,7 +157,7 @@ func TestStmtSummaryRetriverV2_TableStatementsSummaryHistory(t *testing.T) { data := infoschema.NewData() infoSchemaBuilder, err := infoschema.NewBuilder(nil, nil, data).InitWithDBInfos(nil, nil, nil, 0) require.NoError(t, err) - infoSchema := infoSchemaBuilder.Build() + infoSchema := infoSchemaBuilder.Build(math.MaxUint64) table, err := infoSchema.TableByName(util.InformationSchemaName, model.NewCIStr(infoschema.TableStatementsSummaryHistory)) require.NoError(t, err) columns := table.Meta().Columns diff --git a/pkg/executor/temporary_table_test.go b/pkg/executor/temporary_table_test.go index e0f7f3d9c2cab..0add5255f507f 100644 --- a/pkg/executor/temporary_table_test.go +++ b/pkg/executor/temporary_table_test.go @@ -75,6 +75,12 @@ func assertTemporaryTableNoNetwork(t *testing.T, createTable func(*testkit.TestK tk.MustExec("use test") tk1.MustExec("use test") + + if tk.MustQuery("select @@tidb_schema_cache_size > 0").Equal(testkit.Rows("1")) { + // infoschema v2 requires network, so it cannot be tested this way. + t.Skip() + } + tk.MustExec("drop table if exists normal, tmp_t") tk.MustExec("create table normal (id int, a int, index(a))") createTable(tk) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 70bac0122b983..06659eae424ea 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1056,7 +1056,7 @@ func TestCheckFailReport(t *testing.T) { store := testkit.CreateMockStore(t) tk := newInconsistencyKit(t, testkit.NewAsyncTestKit(t, store), newDefaultOpt()) - rmode := tk.sctx.GetSessionVars().EnableRedactNew + rmode := tk.sctx.GetSessionVars().EnableRedactLog // row more than unique index func() { diff --git a/pkg/executor/test/analyzetest/analyze_test.go b/pkg/executor/test/analyzetest/analyze_test.go index 613c1bc553b93..90fbcff71bc9f 100644 --- a/pkg/executor/test/analyzetest/analyze_test.go +++ b/pkg/executor/test/analyzetest/analyze_test.go @@ -1096,7 +1096,9 @@ func TestSavedAnalyzeColumnOptions(t *testing.T) { require.Equal(t, lastVersion, tblStats.Columns[tblInfo.Columns[2].ID].LastUpdateVersion) tk.MustExec("analyze table t columns a") - tblStats = h.GetTableStats(tblInfo) + // TODO: the a's meta should be keep. Or the previous a's meta should be clear. + tblStats, err = h.TableStatsFromStorage(tblInfo, tblInfo.ID, true, 0) + require.NoError(t, err) require.Less(t, lastVersion, tblStats.Version) lastVersion = tblStats.Version // column a is analyzed @@ -1106,7 +1108,9 @@ func TestSavedAnalyzeColumnOptions(t *testing.T) { tk.MustQuery(fmt.Sprintf("select column_choice, column_ids from mysql.analyze_options where table_id = %v", tblInfo.ID)).Check(testkit.Rows(fmt.Sprintf("LIST %v", tblInfo.Columns[0].ID))) tk.MustExec("analyze table t all columns") - tblStats = h.GetTableStats(tblInfo) + // TODO: the a's meta should be keep. Or the previous a's meta should be clear. + tblStats, err = h.TableStatsFromStorage(tblInfo, tblInfo.ID, true, 0) + require.NoError(t, err) require.Less(t, lastVersion, tblStats.Version) lastVersion = tblStats.Version // column a, b, c are analyzed @@ -2337,9 +2341,11 @@ PARTITION BY RANGE ( a ) ( tbl = h.GetTableStats(tableInfo) require.Greater(t, tbl.Version, lastVersion) lastVersion = tbl.Version - p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) - p1 = h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) - require.NotEqual(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + p0, err = h.TableStatsFromStorage(tableInfo, pi.Definitions[0].ID, true, 0) + require.NoError(t, err) + p1, err = h.TableStatsFromStorage(tableInfo, pi.Definitions[1].ID, true, 0) + require.NoError(t, err) + require.Equal(t, 0, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) @@ -2695,7 +2701,8 @@ PARTITION BY RANGE ( a ) ( tk.MustExec("analyze table t partition p1 columns a") tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") tk.MustExec("analyze table t partition p0") - tbl := h.GetTableStats(tableInfo) + tbl, err := h.TableStatsFromStorage(table.Meta(), table.Meta().ID, true, 0) + require.NoError(t, err) require.Equal(t, int64(6), tbl.Columns[tableInfo.Columns[0].ID].Histogram.NDV) } @@ -2796,6 +2803,7 @@ func TestAnalyzeColumnsSkipMVIndexJsonCol(t *testing.T) { // TestAnalyzeMVIndex tests analyzing the mv index use some real data in the table. // It checks the analyze jobs, async loading and the stats content in the memory. func TestAnalyzeMVIndex(t *testing.T) { + t.Skip() require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/DebugAnalyzeJobOperations", "return(true)")) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/statistics/handle/DebugAnalyzeJobOperations", "return(true)")) defer func() { @@ -2987,25 +2995,25 @@ func TestAnalyzeMVIndex(t *testing.T) { tk.MustExec("analyze table t with 1 samplerate, 3 topn") // 3.5. turn on the sync loading, stats on mv indexes should be loaded tk.MustExec("set session tidb_stats_load_sync_wait = 1000") - tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( - "IndexMerge 3.84 root type: union", - "├─IndexRangeScan(Build) 3.84 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 3.84 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_signed) */ * from t where 1 member of (j->'$.signed')").Check(testkit.Rows( + "IndexMerge 27.00 root type: union", + "├─IndexRangeScan(Build) 27.00 cop[tikv] table:t, index:ij_signed(cast(json_extract(`j`, _utf8mb4'$.signed') as signed array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 27.00 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( - "IndexMerge 3.60 root type: union", - "├─IndexRangeScan(Build) 3.60 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 3.60 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_unsigned) */ * from t where 1 member of (j->'$.unsigned')").Check(testkit.Rows( + "IndexMerge 18.00 root type: union", + "├─IndexRangeScan(Build) 18.00 cop[tikv] table:t, index:ij_unsigned(cast(json_extract(`j`, _utf8mb4'$.unsigned') as unsigned array)) range:[1,1], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 18.00 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( - "IndexMerge 1.55 root type: union", - "├─IndexRangeScan(Build) 1.55 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 1.55 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_binary) */ * from t where '1' member of (j->'$.bin')").Check(testkit.Rows( + "IndexMerge 14.83 root type: union", + "├─IndexRangeScan(Build) 14.83 cop[tikv] table:t, index:ij_binary(cast(json_extract(`j`, _utf8mb4'$.bin') as binary(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 14.83 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) - tk.MustQuery("explain format = brief select * from t where '1' member of (j->'$.char')").Check(testkit.Rows( - "IndexMerge 1.93 root type: union", - "├─IndexRangeScan(Build) 1.93 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", - "└─TableRowIDScan(Probe) 1.93 cop[tikv] table:t keep order:false, stats:partial[ia:allEvicted, j:unInitialized]", + tk.MustQuery("explain format = brief select /*+ use_index_merge(t, ij_char) */ * from t where '1' member of (j->'$.char')").Check(testkit.Rows( + "IndexMerge 13.50 root type: union", + "├─IndexRangeScan(Build) 13.50 cop[tikv] table:t, index:ij_char(cast(json_extract(`j`, _utf8mb4'$.char') as char(50) array)) range:[\"1\",\"1\"], keep order:false, stats:partial[j:unInitialized]", + "└─TableRowIDScan(Probe) 13.50 cop[tikv] table:t keep order:false, stats:partial[j:unInitialized]", )) // 4. check stats content in the memory diff --git a/pkg/executor/test/executor/executor_test.go b/pkg/executor/test/executor/executor_test.go index eb792ba36f971..5d499f251819b 100644 --- a/pkg/executor/test/executor/executor_test.go +++ b/pkg/executor/test/executor/executor_test.go @@ -2797,8 +2797,17 @@ func TestIssue38756(t *testing.T) { } func TestIssue50043(t *testing.T) { + testIssue50043WithInitSQL(t, "") +} + +func TestIssue50043WithPipelinedDML(t *testing.T) { + testIssue50043WithInitSQL(t, "set @@tidb_dml_type=bulk") +} + +func testIssue50043WithInitSQL(t *testing.T, initSQL string) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) + tk.MustExec(initSQL) // Test simplified case by update. tk.MustExec("use test") tk.MustExec("create table t (c1 boolean ,c2 decimal ( 37 , 17 ), unique key idx1 (c1 ,c2),unique key idx2 ( c1 ));") @@ -2896,3 +2905,27 @@ func TestIssue51324(t *testing.T) { tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1364 Field 'a' doesn't have a default value", "Warning 1364 Field 'b' doesn't have a default value", "Warning 1364 Field 'c' doesn't have a default value", "Warning 1364 Field 'd' doesn't have a default value")) tk.MustQuery("select * from t order by id").Check(testkit.Rows("13 0 0", "21 1 0 1", "22 1 0 1")) } + +func TestDecimalDivPrecisionIncrement(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t (a decimal(3,0), b decimal(3,0))") + tk.MustExec("insert into t values (8, 7), (9, 7)") + tk.MustQuery("select a/b from t").Check(testkit.Rows("1.1429", "1.2857")) + + tk.MustExec("set div_precision_increment = 7") + tk.MustQuery("select a/b from t").Check(testkit.Rows("1.1428571", "1.2857143")) + + tk.MustExec("set div_precision_increment = 30") + tk.MustQuery("select a/b from t").Check(testkit.Rows("1.142857142857142857142857142857", "1.285714285714285714285714285714")) + + tk.MustExec("set div_precision_increment = 4") + tk.MustQuery("select avg(a) from t").Check(testkit.Rows("8.5000")) + + tk.MustExec("set div_precision_increment = 4") + tk.MustQuery("select avg(a/b) from t").Check(testkit.Rows("1.21428571")) + + tk.MustExec("set div_precision_increment = 10") + tk.MustQuery("select avg(a/b) from t").Check(testkit.Rows("1.21428571428571428550")) +} diff --git a/pkg/executor/test/showtest/show_test.go b/pkg/executor/test/showtest/show_test.go index c9ddeb6629192..0fdc53c95843a 100644 --- a/pkg/executor/test/showtest/show_test.go +++ b/pkg/executor/test/showtest/show_test.go @@ -1049,12 +1049,17 @@ func TestShowCreatePlacementPolicy(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("CREATE PLACEMENT POLICY xyz PRIMARY_REGION='us-east-1' REGIONS='us-east-1,us-east-2' FOLLOWERS=4") tk.MustQuery("SHOW CREATE PLACEMENT POLICY xyz").Check(testkit.Rows("xyz CREATE PLACEMENT POLICY `xyz` PRIMARY_REGION=\"us-east-1\" REGIONS=\"us-east-1,us-east-2\" FOLLOWERS=4")) + tk.MustExec("CREATE PLACEMENT POLICY xyz2 FOLLOWERS=1 SURVIVAL_PREFERENCES=\"[zone, dc, host]\"") + tk.MustQuery("SHOW CREATE PLACEMENT POLICY xyz2").Check(testkit.Rows("xyz2 CREATE PLACEMENT POLICY `xyz2` FOLLOWERS=1 SURVIVAL_PREFERENCES=\"[zone, dc, host]\"")) + tk.MustExec("DROP PLACEMENT POLICY xyz2") // non existent policy err := tk.QueryToErr("SHOW CREATE PLACEMENT POLICY doesnotexist") require.Equal(t, infoschema.ErrPlacementPolicyNotExists.GenWithStackByArgs("doesnotexist").Error(), err.Error()) // alter and try second example tk.MustExec("ALTER PLACEMENT POLICY xyz FOLLOWERS=4") tk.MustQuery("SHOW CREATE PLACEMENT POLICY xyz").Check(testkit.Rows("xyz CREATE PLACEMENT POLICY `xyz` FOLLOWERS=4")) + tk.MustExec("ALTER PLACEMENT POLICY xyz FOLLOWERS=4 SURVIVAL_PREFERENCES=\"[zone, dc, host]\"") + tk.MustQuery("SHOW CREATE PLACEMENT POLICY xyz").Check(testkit.Rows("xyz CREATE PLACEMENT POLICY `xyz` FOLLOWERS=4 SURVIVAL_PREFERENCES=\"[zone, dc, host]\"")) tk.MustExec("DROP PLACEMENT POLICY xyz") } diff --git a/pkg/executor/update.go b/pkg/executor/update.go index 3b33644cfd8b9..6f3c2d43a8356 100644 --- a/pkg/executor/update.go +++ b/pkg/executor/update.go @@ -218,6 +218,9 @@ func (e *UpdateExec) exec(ctx context.Context, _ *expression.Schema, row, newDat } return err1 } + if txn, _ := e.Ctx().Txn(false); txn != nil { + return txn.MayFlush() + } return nil } diff --git a/pkg/expression/aggregation/aggregation_test.go b/pkg/expression/aggregation/aggregation_test.go index 2156bf94b5aeb..3d20fb5f22a81 100644 --- a/pkg/expression/aggregation/aggregation_test.go +++ b/pkg/expression/aggregation/aggregation_test.go @@ -38,6 +38,7 @@ func createAggFuncSuite() (s *mockAggFuncSuite) { s = new(mockAggFuncSuite) s.ctx = mock.NewContext() s.ctx.GetSessionVars().GlobalVarsAccessor = variable.NewMockGlobalAccessor4Tests() + s.ctx.GetSessionVars().DivPrecisionIncrement = variable.DefDivPrecisionIncrement s.rows = make([]chunk.Row, 0, 5050) for i := 1; i <= 100; i++ { for j := 0; j < i; j++ { diff --git a/pkg/expression/aggregation/avg.go b/pkg/expression/aggregation/avg.go index e936565ea2d19..76e6139df20e1 100644 --- a/pkg/expression/aggregation/avg.go +++ b/pkg/expression/aggregation/avg.go @@ -82,7 +82,7 @@ func (af *avgFunction) GetResult(evalCtx *AggEvaluateContext) (d types.Datum) { x := evalCtx.Value.GetMysqlDecimal() y := types.NewDecFromInt(evalCtx.Count) to := new(types.MyDecimal) - err := types.DecimalDiv(x, y, to, types.DivFracIncr) + err := types.DecimalDiv(x, y, to, evalCtx.Ctx.GetSessionVars().GetDivPrecisionIncrement()) terror.Log(err) frac := af.RetTp.GetDecimal() if frac == -1 { diff --git a/pkg/expression/aggregation/base_func.go b/pkg/expression/aggregation/base_func.go index df6aa389f001b..b02be094bf0ce 100644 --- a/pkg/expression/aggregation/base_func.go +++ b/pkg/expression/aggregation/base_func.go @@ -96,7 +96,7 @@ func (a *baseFuncDesc) TypeInfer(ctx expression.BuildContext) error { case ast.AggFuncSum: a.typeInfer4Sum() case ast.AggFuncAvg: - a.typeInfer4Avg() + a.typeInfer4Avg(ctx.GetSessionVars().GetDivPrecisionIncrement()) case ast.AggFuncGroupConcat: a.typeInfer4GroupConcat(ctx) case ast.AggFuncMax, ast.AggFuncMin, ast.AggFuncFirstRow, @@ -220,16 +220,16 @@ func (a *baseFuncDesc) TypeInfer4FinalCount(finalCountRetType *types.FieldType) // typeInfer4Avg should returns a "decimal", otherwise it returns a "double". // Because child returns integer or decimal type. -func (a *baseFuncDesc) typeInfer4Avg() { +func (a *baseFuncDesc) typeInfer4Avg(divPrecIncre int) { switch a.Args[0].GetType().GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: a.RetTp = types.NewFieldType(mysql.TypeNewDecimal) - a.RetTp.SetDecimalUnderLimit(types.DivFracIncr) + a.RetTp.SetDecimalUnderLimit(divPrecIncre) flen, _ := mysql.GetDefaultFieldLengthAndDecimal(a.Args[0].GetType().GetType()) - a.RetTp.SetFlenUnderLimit(flen + types.DivFracIncr) + a.RetTp.SetFlenUnderLimit(flen + divPrecIncre) case mysql.TypeYear, mysql.TypeNewDecimal: a.RetTp = types.NewFieldType(mysql.TypeNewDecimal) - a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(), types.DivFracIncr, types.DivFracIncr) + a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(), divPrecIncre, divPrecIncre) case mysql.TypeDouble, mysql.TypeFloat: a.RetTp = types.NewFieldType(mysql.TypeDouble) a.RetTp.SetFlen(mysql.MaxRealWidth) diff --git a/pkg/expression/builtin_arithmetic.go b/pkg/expression/builtin_arithmetic.go index 189b051f4b919..578fb9c356d7f 100644 --- a/pkg/expression/builtin_arithmetic.go +++ b/pkg/expression/builtin_arithmetic.go @@ -58,10 +58,6 @@ var ( _ builtinFunc = &builtinArithmeticModDecimalSig{} ) -// precIncrement indicates the number of digits by which to increase the scale of the result of division operations -// performed with the / operator. -const precIncrement = 4 - // isConstantBinaryLiteral return true if expr is constant binary literal func isConstantBinaryLiteral(expr Expression) bool { if types.IsBinaryStr(expr.GetType()) { @@ -139,7 +135,7 @@ func setFlenDecimal4RealOrDecimal(retTp *types.FieldType, arg0, arg1 Expression, } } -func (c *arithmeticDivideFunctionClass) setType4DivDecimal(retTp, a, b *types.FieldType) { +func (c *arithmeticDivideFunctionClass) setType4DivDecimal(retTp, a, b *types.FieldType, divPrecIncrement int) { var deca, decb = a.GetDecimal(), b.GetDecimal() if deca == types.UnspecifiedFsp { deca = 0 @@ -147,13 +143,13 @@ func (c *arithmeticDivideFunctionClass) setType4DivDecimal(retTp, a, b *types.Fi if decb == types.UnspecifiedFsp { decb = 0 } - retTp.SetDecimalUnderLimit(deca + precIncrement) + retTp.SetDecimalUnderLimit(deca + divPrecIncrement) if a.GetFlen() == types.UnspecifiedLength { retTp.SetFlen(mysql.MaxDecimalWidth) return } aPrec := types.DecimalLength2Precision(a.GetFlen(), a.GetDecimal(), mysql.HasUnsignedFlag(a.GetFlag())) - retTp.SetFlenUnderLimit(aPrec + decb + precIncrement) + retTp.SetFlenUnderLimit(aPrec + decb + divPrecIncrement) retTp.SetFlenUnderLimit(types.Precision2LengthNoTruncation(retTp.GetFlen(), retTp.GetDecimal(), mysql.HasUnsignedFlag(retTp.GetFlag()))) } @@ -665,7 +661,7 @@ func (c *arithmeticDivideFunctionClass) getFunction(ctx BuildContext, args []Exp if err != nil { return nil, err } - c.setType4DivDecimal(bf.tp, lhsTp, rhsTp) + c.setType4DivDecimal(bf.tp, lhsTp, rhsTp, ctx.GetSessionVars().GetDivPrecisionIncrement()) sig := &builtinArithmeticDivideDecimalSig{bf} sig.setPbCode(tipb.ScalarFuncSig_DivideDecimal) return sig, nil @@ -718,7 +714,7 @@ func (s *builtinArithmeticDivideDecimalSig) evalDecimal(ctx EvalContext, row chu } c := &types.MyDecimal{} - err = types.DecimalDiv(a, b, c, types.DivFracIncr) + err = types.DecimalDiv(a, b, c, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err == types.ErrDivByZero { return c, true, handleDivisionByZeroError(ctx) } else if err == types.ErrTruncated { @@ -833,7 +829,7 @@ func (s *builtinArithmeticIntDivideDecimalSig) evalInt(ctx EvalContext, row chun } c := &types.MyDecimal{} - err = types.DecimalDiv(num[0], num[1], c, types.DivFracIncr) + err = types.DecimalDiv(num[0], num[1], c, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err == types.ErrDivByZero { return 0, true, handleDivisionByZeroError(ctx) } diff --git a/pkg/expression/builtin_arithmetic_vec.go b/pkg/expression/builtin_arithmetic_vec.go index 5c5aa6a54fe2e..12e8a07a4ef5f 100644 --- a/pkg/expression/builtin_arithmetic_vec.go +++ b/pkg/expression/builtin_arithmetic_vec.go @@ -87,7 +87,7 @@ func (b *builtinArithmeticDivideDecimalSig) vecEvalDecimal(ctx EvalContext, inpu if result.IsNull(i) { continue } - err = types.DecimalDiv(&x[i], &y[i], &to, types.DivFracIncr) + err = types.DecimalDiv(&x[i], &y[i], &to, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err == types.ErrDivByZero { if err = handleDivisionByZeroError(ctx); err != nil { return err @@ -596,7 +596,7 @@ func (b *builtinArithmeticIntDivideDecimalSig) vecEvalInt(ctx EvalContext, input } c := &types.MyDecimal{} - err = types.DecimalDiv(&num[0][i], &num[1][i], c, types.DivFracIncr) + err = types.DecimalDiv(&num[0][i], &num[1][i], c, ctx.GetSessionVars().GetDivPrecisionIncrement()) if err == types.ErrDivByZero { if err = handleDivisionByZeroError(ctx); err != nil { return err diff --git a/pkg/expression/builtin_cast.go b/pkg/expression/builtin_cast.go index e87109bea088d..36a5a7ce9b625 100644 --- a/pkg/expression/builtin_cast.go +++ b/pkg/expression/builtin_cast.go @@ -878,7 +878,7 @@ func (b *builtinCastStringAsJSONSig) evalJSON(ctx EvalContext, row chunk.Row) (r typ := b.args[0].GetType() if types.IsBinaryStr(typ) { buf := []byte(val) - if typ.GetType() == mysql.TypeString { + if typ.GetType() == mysql.TypeString && typ.GetFlen() > 0 { // the tailing zero should also be in the opaque json buf = make([]byte, typ.GetFlen()) copy(buf, val) diff --git a/pkg/expression/builtin_cast_vec.go b/pkg/expression/builtin_cast_vec.go index 56c60c8b2158c..bce2383171338 100644 --- a/pkg/expression/builtin_cast_vec.go +++ b/pkg/expression/builtin_cast_vec.go @@ -853,7 +853,7 @@ func (b *builtinCastStringAsJSONSig) vecEvalJSON(ctx EvalContext, input *chunk.C val := buf.GetBytes(i) resultBuf := val - if typ.GetType() == mysql.TypeString { + if typ.GetType() == mysql.TypeString && typ.GetFlen() > 0 { // only for BINARY: the tailing zero should also be in the opaque json resultBuf = make([]byte, typ.GetFlen()) copy(resultBuf, val) diff --git a/pkg/infoschema/BUILD.bazel b/pkg/infoschema/BUILD.bazel index 7d8e8bff33b7e..c24cb1061be0e 100644 --- a/pkg/infoschema/BUILD.bazel +++ b/pkg/infoschema/BUILD.bazel @@ -83,7 +83,7 @@ go_test( ], embed = [":infoschema"], flaky = True, - shard_count = 12, + shard_count = 13, deps = [ "//pkg/ddl/placement", "//pkg/domain", diff --git a/pkg/infoschema/builder.go b/pkg/infoschema/builder.go index d3c2b5469e9a1..23e3cca7b9b53 100644 --- a/pkg/infoschema/builder.go +++ b/pkg/infoschema/builder.go @@ -18,7 +18,6 @@ import ( "cmp" "context" "fmt" - "math" "slices" "strings" @@ -99,7 +98,7 @@ func (b *Builder) ApplyDiff(m *meta.Meta, diff *model.SchemaDiff) ([]int64, erro case model.ActionFlashbackCluster: return []int64{-1}, nil default: - return b.applyDefaultAction(m, diff) + return applyDefaultAction(b, m, diff) } } @@ -132,7 +131,7 @@ func applyTruncateTableOrPartition(b *Builder, m *meta.Meta, diff *model.SchemaD return tblIDs, nil } -func (b *Builder) applyDropTableOrPartition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyDropTableOrPartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { tblIDs, err := applyTableUpdate(b, m, diff) if err != nil { return nil, errors.Trace(err) @@ -146,7 +145,7 @@ func (b *Builder) applyDropTableOrPartition(m *meta.Meta, diff *model.SchemaDiff return tblIDs, nil } -func (b *Builder) applyReorganizePartition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyReorganizePartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { tblIDs, err := applyTableUpdate(b, m, diff) if err != nil { return nil, errors.Trace(err) @@ -165,7 +164,7 @@ func (b *Builder) applyReorganizePartition(m *meta.Meta, diff *model.SchemaDiff) return tblIDs, nil } -func (b *Builder) applyExchangeTablePartition(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyExchangeTablePartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { // It is not in StatePublic. if diff.OldTableID == diff.TableID && diff.OldSchemaID == diff.SchemaID { ntIDs, err := applyTableUpdate(b, m, diff) @@ -243,7 +242,7 @@ func (b *Builder) applyExchangeTablePartition(m *meta.Meta, diff *model.SchemaDi return append(ptIDs, ntIDs...), nil } -func (b *Builder) applyRecoverTable(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyRecoverTable(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { tblIDs, err := applyTableUpdate(b, m, diff) if err != nil { return nil, errors.Trace(err) @@ -311,7 +310,7 @@ func (b *Builder) applyAffectedOpts(m *meta.Meta, tblIDs []int64, diff *model.Sc return tblIDs, nil } -func (b *Builder) applyDefaultAction(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyDefaultAction(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { tblIDs, err := applyTableUpdate(b, m, diff) if err != nil { return nil, errors.Trace(err) @@ -367,7 +366,7 @@ func (b *Builder) updateBundleForTableUpdate(diff *model.SchemaDiff, newTableID, } } -func (b *Builder) dropTableForUpdate(newTableID, oldTableID int64, dbInfo *model.DBInfo, diff *model.SchemaDiff) ([]int64, autoid.Allocators, error) { +func dropTableForUpdate(b *Builder, newTableID, oldTableID int64, dbInfo *model.DBInfo, diff *model.SchemaDiff) ([]int64, autoid.Allocators, error) { tblIDs := make([]int64, 0, 2) var newAllocs autoid.Allocators // We try to reuse the old allocator, so the cached auto ID can be reused. @@ -398,9 +397,9 @@ func (b *Builder) dropTableForUpdate(newTableID, oldTableID int64, dbInfo *model ) } oldDBInfo := b.getSchemaAndCopyIfNecessary(oldRoDBInfo.Name.L) - tmpIDs = b.applyDropTable(diff, oldDBInfo, oldTableID, tmpIDs) + tmpIDs = applyDropTable(b, diff, oldDBInfo, oldTableID, tmpIDs) } else { - tmpIDs = b.applyDropTable(diff, dbInfo, oldTableID, tmpIDs) + tmpIDs = applyDropTable(b, diff, dbInfo, oldTableID, tmpIDs) } if oldTableID != newTableID { @@ -423,7 +422,7 @@ func (b *Builder) applyTableUpdate(m *meta.Meta, diff *model.SchemaDiff) ([]int6 b.updateBundleForTableUpdate(diff, newTableID, oldTableID) b.copySortedTables(oldTableID, newTableID) - tblIDs, allocs, err := b.dropTableForUpdate(newTableID, oldTableID, dbInfo, diff) + tblIDs, allocs, err := dropTableForUpdate(b, newTableID, oldTableID, dbInfo, diff) if err != nil { return nil, err } @@ -431,7 +430,7 @@ func (b *Builder) applyTableUpdate(m *meta.Meta, diff *model.SchemaDiff) ([]int6 if tableIDIsValid(newTableID) { // All types except DropTableOrView. var err error - tblIDs, err = b.applyCreateTable(m, dbInfo, newTableID, allocs, diff.Type, tblIDs, diff.Version) + tblIDs, err = applyCreateTable(b, m, dbInfo, newTableID, allocs, diff.Type, tblIDs, diff.Version) if err != nil { return nil, errors.Trace(err) } @@ -547,31 +546,6 @@ func (b *Builder) applyDropSchema(diff *model.SchemaDiff) []int64 { return tableIDs } -func (b *Builder) applyDropTableV2(diff *model.SchemaDiff, dbInfo *model.DBInfo, tableID int64, affected []int64) []int64 { - // Remove the table in temporaryTables - if b.infoSchemaMisc.temporaryTableIDs != nil { - delete(b.infoSchemaMisc.temporaryTableIDs, tableID) - } - - table, ok := b.infoschemaV2.TableByID(tableID) - - if !ok { - return nil - } - - b.infoData.delete(tableItem{ - dbName: dbInfo.Name.L, - dbID: dbInfo.ID, - tableName: table.Meta().Name.L, - tableID: table.Meta().ID, - schemaVersion: diff.Version, - }) - - // The old DBInfo still holds a reference to old table info, we need to remove it. - b.deleteReferredForeignKeys(dbInfo, tableID) - return affected -} - func (b *Builder) applyRecoverSchema(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { if di, ok := b.infoSchema.SchemaByID(diff.SchemaID); ok { return nil, ErrDatabaseExists.GenWithStackByArgs( @@ -655,7 +629,7 @@ func (b *Builder) buildAllocsForCreateTable(tp model.ActionType, dbInfo *model.D return autoid.NewAllocatorsFromTblInfo(b.Requirement, dbInfo.ID, tblInfo) } -func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID int64, allocs autoid.Allocators, tp model.ActionType, affected []int64, schemaVersion int64) ([]int64, error) { +func applyCreateTable(b *Builder, m *meta.Meta, dbInfo *model.DBInfo, tableID int64, allocs autoid.Allocators, tp model.ActionType, affected []int64, schemaVersion int64) ([]int64, error) { tblInfo, err := m.GetTable(dbInfo.ID, tableID) if err != nil { return nil, errors.Trace(err) @@ -751,9 +725,6 @@ func ConvertOldVersionUTF8ToUTF8MB4IfNeed(tbInfo *model.TableInfo) { } func (b *Builder) applyDropTable(diff *model.SchemaDiff, dbInfo *model.DBInfo, tableID int64, affected []int64) []int64 { - if b.enableV2 { - return b.applyDropTableV2(diff, dbInfo, tableID, affected) - } bucketIdx := tableBucketIdx(tableID) sortedTbls := b.infoSchema.sortedTablesBuckets[bucketIdx] idx := sortedTbls.searchTable(tableID) @@ -792,13 +763,13 @@ func (b *Builder) deleteReferredForeignKeys(dbInfo *model.DBInfo, tableID int64) } // Build builds and returns the built infoschema. -func (b *Builder) Build() InfoSchema { - b.updateInfoSchemaBundles(b.infoSchema) +func (b *Builder) Build(schemaTS uint64) InfoSchema { if b.enableV2 { - b.infoschemaV2.ts = math.MaxUint64 // TODO: should be the correct TS - b.infoschemaV2.schemaVersion = b.infoSchema.SchemaMetaVersion() + b.infoschemaV2.ts = schemaTS + updateInfoSchemaBundles(b) return &b.infoschemaV2 } + updateInfoSchemaBundles(b) return b.infoSchema } @@ -810,12 +781,10 @@ func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) (*Builder, error) return nil, errors.New("builder's infoschema mismatch, return error to trigger full reload") } - var oldIS *infoSchema if schemaV2, ok := oldSchema.(*infoschemaV2); ok { - oldIS = schemaV2.infoSchema - } else { - oldIS = oldSchema.(*infoSchema) + b.infoschemaV2.ts = schemaV2.ts } + oldIS := oldSchema.base() b.initBundleInfoBuilder() b.infoSchema.schemaMetaVersion = oldIS.schemaMetaVersion b.copySchemasMap(oldIS) diff --git a/pkg/infoschema/bundle_builder.go b/pkg/infoschema/bundle_builder.go index 4aa385bbe6aa9..133357709f235 100644 --- a/pkg/infoschema/bundle_builder.go +++ b/pkg/infoschema/bundle_builder.go @@ -112,8 +112,9 @@ func (b *bundleInfoBuilder) completeUpdateTables(is *infoSchema) { } } -func (b *bundleInfoBuilder) updateTableBundles(is *infoSchema, tableID int64) { - tbl, ok := is.TableByID(tableID) +func (b *bundleInfoBuilder) updateTableBundles(infoSchemaInterface InfoSchema, tableID int64) { + is := infoSchemaInterface.base() + tbl, ok := infoSchemaInterface.TableByID(tableID) if !ok { b.deleteBundle(is, tableID) return diff --git a/pkg/infoschema/infoschema.go b/pkg/infoschema/infoschema.go index 1604354fd3bc2..8cecbe2afa9c0 100644 --- a/pkg/infoschema/infoschema.go +++ b/pkg/infoschema/infoschema.go @@ -59,12 +59,12 @@ type infoSchema struct { // sortedTablesBuckets is a slice of sortedTables, a table's bucket index is (tableID % bucketCount). sortedTablesBuckets []sortedTables +} +type infoSchemaMisc struct { // schemaMetaVersion is the version of schema, and we should check version when change schema. schemaMetaVersion int64 -} -type infoSchemaMisc struct { // ruleBundleMap stores all placement rules ruleBundleMap map[int64]*placement.Bundle @@ -151,6 +151,10 @@ func MockInfoSchemaWithSchemaVer(tbList []*model.TableInfo, schemaVer int64) Inf var _ InfoSchema = (*infoSchema)(nil) +func (is *infoSchema) base() *infoSchema { + return is +} + func (is *infoSchema) SchemaByName(schema model.CIStr) (val *model.DBInfo, ok bool) { tableNames, ok := is.schemaMap[schema.L] if !ok { @@ -159,10 +163,6 @@ func (is *infoSchema) SchemaByName(schema model.CIStr) (val *model.DBInfo, ok bo return tableNames.dbInfo, true } -func (is *infoSchema) SchemaMetaVersion() int64 { - return is.schemaMetaVersion -} - func (is *infoSchema) SchemaExists(schema model.CIStr) bool { _, ok := is.schemaMap[schema.L] return ok @@ -308,6 +308,10 @@ func (is *infoSchemaMisc) HasTemporaryTable() bool { return len(is.temporaryTableIDs) != 0 } +func (is *infoSchemaMisc) SchemaMetaVersion() int64 { + return is.schemaMetaVersion +} + // GetSequenceByName gets the sequence by name. func GetSequenceByName(is InfoSchema, schema, sequence model.CIStr) (util.SequenceTable, error) { tbl, err := is.TableByName(schema, sequence) diff --git a/pkg/infoschema/infoschema_test.go b/pkg/infoschema/infoschema_test.go index 4f1a87186bfb1..a74f78b9a4041 100644 --- a/pkg/infoschema/infoschema_test.go +++ b/pkg/infoschema/infoschema_test.go @@ -17,6 +17,7 @@ package infoschema_test import ( "context" "encoding/json" + "math" "strings" "testing" @@ -100,7 +101,7 @@ func TestBasic(t *testing.T) { dbInfos := []*model.DBInfo{dbInfo} internal.AddDB(t, re.Store(), dbInfo) - builder, err := infoschema.NewBuilder(re, nil, nil).InitWithDBInfos(dbInfos, nil, nil, 1) + builder, err := infoschema.NewBuilder(re, nil, infoschema.NewData()).InitWithDBInfos(dbInfos, nil, nil, 1) require.NoError(t, err) txn, err := re.Store().Begin() @@ -110,7 +111,7 @@ func TestBasic(t *testing.T) { err = txn.Rollback() require.NoError(t, err) - is := builder.Build() + is := builder.Build(math.MaxUint64) schemaNames := infoschema.AllSchemaNames(is) require.Len(t, schemaNames, 3) @@ -186,7 +187,7 @@ func TestBasic(t *testing.T) { require.NoError(t, err) err = txn.Rollback() require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) schema, ok = is.SchemaByID(dbID) require.True(t, ok) require.Equal(t, 1, len(schema.Tables)) @@ -238,7 +239,7 @@ func TestInfoTables(t *testing.T) { builder, err := infoschema.NewBuilder(re, nil, nil).InitWithDBInfos(nil, nil, nil, 0) require.NoError(t, err) - is := builder.Build() + is := builder.Build(math.MaxUint64) infoTables := []string{ "SCHEMATA", @@ -301,7 +302,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) { dbInfos := []*model.DBInfo{dbInfo} builder, err := infoschema.NewBuilder(re, nil, nil).InitWithDBInfos(dbInfos, nil, nil, 1) require.NoError(t, err) - is := builder.Build() + is := builder.Build(math.MaxUint64) require.False(t, is.HasTemporaryTable()) db, ok := is.SchemaByName(model.NewCIStr("test")) require.True(t, ok) @@ -322,7 +323,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) { builder, err := infoschema.NewBuilder(re, nil, nil).InitWithOldInfoSchema(curIs) require.NoError(t, err) change(m, builder) - curIs = builder.Build() + curIs = builder.Build(math.MaxUint64) } return nil }) @@ -400,7 +401,7 @@ func TestBuildSchemaWithGlobalTemporaryTable(t *testing.T) { require.True(t, ok) builder, err = infoschema.NewBuilder(re, nil, nil).InitWithDBInfos([]*model.DBInfo{newDB}, newIS.AllPlacementPolicies(), newIS.AllResourceGroups(), newIS.SchemaMetaVersion()) require.NoError(t, err) - require.True(t, builder.Build().HasTemporaryTable()) + require.True(t, builder.Build(math.MaxUint64).HasTemporaryTable()) // create and then drop tbID, err = internal.GenGlobalID(re.Store()) @@ -525,7 +526,7 @@ func TestBuildBundle(t *testing.T) { builder, err := infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos([]*model.DBInfo{db}, is.AllPlacementPolicies(), is.AllResourceGroups(), is.SchemaMetaVersion()) require.NoError(t, err) - is2 := builder.Build() + is2 := builder.Build(math.MaxUint64) assertBundle(is2, tbl1.Meta().ID, tb1Bundle) assertBundle(is2, tbl2.Meta().ID, nil) assertBundle(is2, p1.ID, p1Bundle) @@ -843,6 +844,7 @@ func TestInfoSchemaCreateTableLike(t *testing.T) { } func TestEnableInfoSchemaV2(t *testing.T) { + t.Skip("This feature is not enabled yet") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) // Test the @@tidb_enable_infoschema_v2 variable. @@ -900,7 +902,7 @@ func (tc *infoschemaTestContext) createSchema() { // init infoschema builder, err := infoschema.NewBuilder(tc.re, nil, tc.data).InitWithDBInfos([]*model.DBInfo{}, nil, nil, 1) require.NoError(tc.t, err) - tc.is = builder.Build() + tc.is = builder.Build(math.MaxUint64) } func (tc *infoschemaTestContext) runCreateSchema() { @@ -924,6 +926,17 @@ func (tc *infoschemaTestContext) runDropSchema() { }) } +func (tc *infoschemaTestContext) runRecoverSchema() { + tc.runDropSchema() + // recover schema + internal.AddDB(tc.t, tc.re.Store(), tc.dbInfo) + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionRecoverSchema, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { + dbInfo, ok := tc.is.SchemaByID(tc.dbInfo.ID) + require.True(tc.t, ok) + require.Equal(tc.t, dbInfo.Name, tc.dbInfo.Name) + }) +} + func (tc *infoschemaTestContext) runCreateTable(tblName string) int64 { if tc.dbInfo == nil { tc.runCreateSchema() @@ -940,6 +953,30 @@ func (tc *infoschemaTestContext) runCreateTable(tblName string) int64 { return tblInfo.ID } +func (tc *infoschemaTestContext) runCreateTables(tblNames []string) { + if tc.dbInfo == nil { + tc.runCreateSchema() + } + diff := model.SchemaDiff{Type: model.ActionCreateTables, SchemaID: tc.dbInfo.ID} + diff.AffectedOpts = make([]*model.AffectedOption, len(tblNames)) + for i, tblName := range tblNames { + tblInfo := internal.MockTableInfo(tc.t, tc.re.Store(), tblName) + internal.AddTable(tc.t, tc.re.Store(), tc.dbInfo, tblInfo) + diff.AffectedOpts[i] = &model.AffectedOption{ + SchemaID: tc.dbInfo.ID, + TableID: tblInfo.ID, + } + } + + tc.applyDiffAndCheck(&diff, func(tc *infoschemaTestContext) { + for i, opt := range diff.AffectedOpts { + tbl, ok := tc.is.TableByID(opt.TableID) + require.True(tc.t, ok) + require.Equal(tc.t, tbl.Meta().Name.O, tblNames[i]) + } + }) +} + func (tc *infoschemaTestContext) runDropTable(tblName string) { // createTable tblID := tc.runCreateTable(tblName) @@ -1021,6 +1058,28 @@ func (tc *infoschemaTestContext) modifyColumn(tblInfo *model.TableInfo) { require.NoError(tc.t, err) } +func (tc *infoschemaTestContext) runModifySchemaCharsetAndCollate(charset, collate string) { + tc.dbInfo.Charset = charset + tc.dbInfo.Collate = collate + internal.UpdateDB(tc.t, tc.re.Store(), tc.dbInfo) + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaCharsetAndCollate, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { + schema, ok := tc.is.SchemaByID(tc.dbInfo.ID) + require.True(tc.t, ok) + require.Equal(tc.t, charset, schema.Charset) + require.Equal(tc.t, collate, schema.Collate) + }) +} + +func (tc *infoschemaTestContext) runModifySchemaDefaultPlacement(policy *model.PolicyRefInfo) { + tc.dbInfo.PlacementPolicyRef = policy + internal.UpdateDB(tc.t, tc.re.Store(), tc.dbInfo) + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaDefaultPlacement, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { + schema, ok := tc.is.SchemaByID(tc.dbInfo.ID) + require.True(tc.t, ok) + require.Equal(tc.t, policy, schema.PlacementPolicyRef) + }) +} + func (tc *infoschemaTestContext) applyDiffAndCheck(diff *model.SchemaDiff, checkFn func(tc *infoschemaTestContext)) { txn, err := tc.re.Store().Begin() require.NoError(tc.t, err) @@ -1030,7 +1089,7 @@ func (tc *infoschemaTestContext) applyDiffAndCheck(diff *model.SchemaDiff, check // applyDiff _, err = builder.ApplyDiff(meta.NewMeta(txn), diff) require.NoError(tc.t, err) - tc.is = builder.Build() + tc.is = builder.Build(math.MaxUint64) checkFn(tc) } @@ -1058,6 +1117,8 @@ func TestApplyDiff(t *testing.T) { ctx: kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL), data: infoschema.NewData(), } + tc.runRecoverSchema() + tc.clear() tc.runCreateSchema() tc.clear() tc.runDropSchema() @@ -1070,6 +1131,13 @@ func TestApplyDiff(t *testing.T) { tc.runCreateTable("test") tc.runModifyTable("test", model.ActionAddColumn) tc.runModifyTable("test", model.ActionModifyColumn) + + tc.runModifySchemaCharsetAndCollate("utf8mb4", "utf8mb4_general_ci") + tc.runModifySchemaCharsetAndCollate("utf8", "utf8_unicode_ci") + tc.runModifySchemaDefaultPlacement(&model.PolicyRefInfo{ + Name: model.NewCIStr("test"), + }) + tc.runCreateTables([]string{"test1", "test2"}) } // TODO(ywqzzy): check all actions. } diff --git a/pkg/infoschema/infoschema_v2.go b/pkg/infoschema/infoschema_v2.go index 85e30d0535c4d..a7fce1f570c2a 100644 --- a/pkg/infoschema/infoschema_v2.go +++ b/pkg/infoschema/infoschema_v2.go @@ -41,6 +41,7 @@ type tableItem struct { tableName string tableID int64 schemaVersion int64 + tomb bool } type schemaItem struct { @@ -162,7 +163,9 @@ func (isd *Data) addDB(schemaVersion int64, dbInfo *model.DBInfo) { isd.schemaMap.Set(schemaItem{schemaVersion: schemaVersion, dbInfo: dbInfo}) } -func (isd *Data) delete(item tableItem) { +func (isd *Data) remove(item tableItem) { + item.tomb = true + isd.byName.Set(item) isd.tableCache.Remove(tableCacheKey{item.tableID, item.schemaVersion}) } @@ -212,7 +215,14 @@ func compareByName(a, b tableItem) bool { return false } - return a.tableID < b.tableID + if a.tableID < b.tableID { + return true + } + if a.tableID > b.tableID { + return false + } + + return a.schemaVersion < b.schemaVersion } func compareSchemaItem(a, b schemaItem) bool { @@ -228,10 +238,9 @@ func compareSchemaItem(a, b schemaItem) bool { var _ InfoSchema = &infoschemaV2{} type infoschemaV2 struct { - *infoSchema // in fact, we only need the infoSchemaMisc inside it, but the builder rely on it. - r autoid.Requirement - ts uint64 - schemaVersion int64 + *infoSchema // in fact, we only need the infoSchemaMisc inside it, but the builder rely on it. + r autoid.Requirement + ts uint64 *Data } @@ -277,19 +286,27 @@ func search(bt *btree.BTreeG[tableItem], schemaVersion int64, end tableItem, mat } return true }) + if ok && target.tomb { + // If the item is a tomb record, the table is dropped. + ok = false + } return target, ok } +func (is *infoschemaV2) base() *infoSchema { + return is.infoSchema +} + func (is *infoschemaV2) TableByID(id int64) (val table.Table, ok bool) { // Get from the cache. - key := tableCacheKey{id, is.schemaVersion} + key := tableCacheKey{id, is.infoSchema.schemaMetaVersion} tbl, found := is.tableCache.Get(key) if found && tbl != nil { return tbl, true } eq := func(a, b *tableItem) bool { return a.tableID == b.tableID } - itm, ok := search(is.byID, is.schemaVersion, tableItem{tableID: id, dbID: math.MaxInt64}, eq) + itm, ok := search(is.byID, is.infoSchema.schemaMetaVersion, tableItem{tableID: id, dbID: math.MaxInt64}, eq) if !ok { // TODO: in the future, this may happen and we need to check tikv to see whether table exists. return nil, false @@ -304,7 +321,7 @@ func (is *infoschemaV2) TableByID(id int64) (val table.Table, ok bool) { } // Maybe the table is evicted? need to reload. - ret, err := loadTableInfo(is.r, is.Data, id, itm.dbID, is.ts, is.schemaVersion) + ret, err := loadTableInfo(is.r, is.Data, id, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion) if err != nil || ret == nil { return nil, false } @@ -330,21 +347,21 @@ func (is *infoschemaV2) TableByName(schema, tbl model.CIStr) (t table.Table, err } eq := func(a, b *tableItem) bool { return a.dbName == b.dbName && a.tableName == b.tableName } - itm, ok := search(is.byName, is.schemaVersion, tableItem{dbName: schema.L, tableName: tbl.L, tableID: math.MaxInt64}, eq) + itm, ok := search(is.byName, is.infoSchema.schemaMetaVersion, tableItem{dbName: schema.L, tableName: tbl.L, tableID: math.MaxInt64}, eq) if !ok { // TODO: in the future, this may happen and we need to check tikv to see whether table exists. return nil, ErrTableNotExists.GenWithStackByArgs(schema, tbl) } // Get from the cache. - key := tableCacheKey{itm.tableID, is.schemaVersion} + key := tableCacheKey{itm.tableID, is.infoSchema.schemaMetaVersion} res, found := is.tableCache.Get(key) if found && res != nil { return res, nil } // Maybe the table is evicted? need to reload. - ret, err := loadTableInfo(is.r, is.Data, itm.tableID, itm.dbID, is.ts, is.schemaVersion) + ret, err := loadTableInfo(is.r, is.Data, itm.tableID, itm.dbID, is.ts, is.infoSchema.schemaMetaVersion) if err != nil { return nil, errors.Trace(err) } @@ -364,7 +381,7 @@ func (is *infoschemaV2) SchemaByName(schema model.CIStr) (val *model.DBInfo, ok ok = false return false } - if item.schemaVersion <= is.schemaVersion { + if item.schemaVersion <= is.infoSchema.schemaMetaVersion { ok = true val = item.dbInfo return false @@ -398,10 +415,6 @@ func (is *infoschemaV2) AllSchemaNames() []model.CIStr { return rs } -func (is *infoschemaV2) SchemaMetaVersion() int64 { - return is.schemaVersion -} - func (is *infoschemaV2) SchemaExists(schema model.CIStr) bool { var ok bool if isSpecialDB(schema.L) { @@ -595,6 +608,20 @@ func applyRecoverSchema(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int return b.applyRecoverSchema(m, diff) } +func (b *Builder) applyRecoverSchemaV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { + if di, ok := b.infoschemaV2.SchemaByID(diff.SchemaID); ok { + return nil, ErrDatabaseExists.GenWithStackByArgs( + fmt.Sprintf("(Schema ID %d)", di.ID), + ) + } + di, err := m.GetDatabase(diff.SchemaID) + if err != nil { + return nil, errors.Trace(err) + } + b.infoschemaV2.addDB(diff.Version, di) + return applyCreateTables(b, m, diff) +} + func applyModifySchemaCharsetAndCollate(b *Builder, m *meta.Meta, diff *model.SchemaDiff) error { if b.enableV2 { return b.applyModifySchemaCharsetAndCollateV2(m, diff) @@ -609,39 +636,23 @@ func applyModifySchemaDefaultPlacement(b *Builder, m *meta.Meta, diff *model.Sch return b.applyModifySchemaDefaultPlacement(m, diff) } -func applyDropTableOrPartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - if b.enableV2 { - // return b.applyDropTableOrPartitionV2(m, diff) - } - return b.applyDropTableOrPartition(m, diff) -} - -func applyRecoverTable(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func applyDropTable(b *Builder, diff *model.SchemaDiff, dbInfo *model.DBInfo, tableID int64, affected []int64) []int64 { if b.enableV2 { - return b.applyRecoverTableV2(m, diff) + return b.applyDropTableV2(diff, dbInfo, tableID, affected) } - return b.applyRecoverTable(m, diff) + return b.applyDropTable(diff, dbInfo, tableID, affected) } func applyCreateTables(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - if b.enableV2 { - return b.applyCreateTablesV2(m, diff) - } return b.applyCreateTables(m, diff) } -func applyReorganizePartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { +func updateInfoSchemaBundles(b *Builder) { if b.enableV2 { - return b.applyReorganizePartitionV2(m, diff) + b.updateInfoSchemaBundlesV2(&b.infoschemaV2) + } else { + b.updateInfoSchemaBundles(b.infoSchema) } - return b.applyReorganizePartition(m, diff) -} - -func applyExchangeTablePartition(b *Builder, m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - if b.enableV2 { - return b.applyExchangeTablePartitionV2(m, diff) - } - return b.applyExchangeTablePartition(m, diff) } // TODO: more UT to check the correctness. @@ -656,7 +667,7 @@ func (b *Builder) applyTableUpdateV2(m *meta.Meta, diff *model.SchemaDiff) ([]in oldTableID, newTableID := b.getTableIDs(diff) b.updateBundleForTableUpdate(diff, newTableID, oldTableID) - tblIDs, allocs, err := b.dropTableForUpdate(newTableID, oldTableID, oldDBInfo, diff) + tblIDs, allocs, err := dropTableForUpdate(b, newTableID, oldTableID, oldDBInfo, diff) if err != nil { return nil, err } @@ -664,7 +675,7 @@ func (b *Builder) applyTableUpdateV2(m *meta.Meta, diff *model.SchemaDiff) ([]in if tableIDIsValid(newTableID) { // All types except DropTableOrView. var err error - tblIDs, err = b.applyCreateTable(m, oldDBInfo, newTableID, allocs, diff.Type, tblIDs, diff.Version) + tblIDs, err = applyCreateTable(b, m, oldDBInfo, newTableID, allocs, diff.Type, tblIDs, diff.Version) if err != nil { return nil, errors.Trace(err) } @@ -692,38 +703,108 @@ func (b *Builder) applyDropSchemaV2(diff *model.SchemaDiff) []int64 { return tableIDs } -func (b *Builder) applyRecoverSchemaV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") +func (b *Builder) applyDropTableV2(diff *model.SchemaDiff, dbInfo *model.DBInfo, tableID int64, affected []int64) []int64 { + // Remove the table in temporaryTables + if b.infoSchemaMisc.temporaryTableIDs != nil { + delete(b.infoSchemaMisc.temporaryTableIDs, tableID) + } + + table, ok := b.infoschemaV2.TableByID(tableID) + if !ok { + return nil + } + + b.infoData.remove(tableItem{ + dbName: dbInfo.Name.L, + dbID: dbInfo.ID, + tableName: table.Meta().Name.L, + tableID: table.Meta().ID, + schemaVersion: diff.Version, + }) + + // The old DBInfo still holds a reference to old table info, we need to remove it. + b.deleteReferredForeignKeys(dbInfo, tableID) + return affected } func (b *Builder) applyModifySchemaCharsetAndCollateV2(m *meta.Meta, diff *model.SchemaDiff) error { - panic("TODO") + di, err := m.GetDatabase(diff.SchemaID) + if err != nil { + return errors.Trace(err) + } + if di == nil { + // This should never happen. + return ErrDatabaseNotExists.GenWithStackByArgs( + fmt.Sprintf("(Schema ID %d)", diff.SchemaID), + ) + } + newDBInfo, _ := b.infoschemaV2.SchemaByID(diff.SchemaID) + newDBInfo.Charset = di.Charset + newDBInfo.Collate = di.Collate + b.infoschemaV2.deleteDB(di.Name) + b.infoschemaV2.addDB(diff.Version, newDBInfo) + return nil } func (b *Builder) applyModifySchemaDefaultPlacementV2(m *meta.Meta, diff *model.SchemaDiff) error { - panic("TODO") -} - -func (b *Builder) applyTruncateTableOrPartitionV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") + di, err := m.GetDatabase(diff.SchemaID) + if err != nil { + return errors.Trace(err) + } + if di == nil { + // This should never happen. + return ErrDatabaseNotExists.GenWithStackByArgs( + fmt.Sprintf("(Schema ID %d)", diff.SchemaID), + ) + } + newDBInfo, _ := b.infoschemaV2.SchemaByID(diff.SchemaID) + newDBInfo.PlacementPolicyRef = di.PlacementPolicyRef + b.infoschemaV2.deleteDB(di.Name) + b.infoschemaV2.addDB(diff.Version, newDBInfo) + return nil } -func (b *Builder) applyDropTableOrPartitionV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") -} +func (b *bundleInfoBuilder) updateInfoSchemaBundlesV2(is *infoschemaV2) { + if b.deltaUpdate { + b.completeUpdateTablesV2(is) + for tblID := range b.updateTables { + b.updateTableBundles(is, tblID) + } + return + } -func (b *Builder) applyRecoverTableV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") + // do full update bundles + // TODO: This is quite inefficient! we need some better way or avoid this API. + is.ruleBundleMap = make(map[int64]*placement.Bundle) + for _, dbInfo := range is.AllSchemas() { + for _, tbl := range is.SchemaTables(dbInfo.Name) { + b.updateTableBundles(is, tbl.Meta().ID) + } + } } -func (b *Builder) applyCreateTablesV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") -} +func (b *bundleInfoBuilder) completeUpdateTablesV2(is *infoschemaV2) { + if len(b.updatePolicies) == 0 && len(b.updatePartitions) == 0 { + return + } -func (b *Builder) applyReorganizePartitionV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") -} + // TODO: This is quite inefficient! we need some better way or avoid this API. + for _, dbInfo := range is.AllSchemas() { + for _, tbl := range is.SchemaTables(dbInfo.Name) { + tblInfo := tbl.Meta() + if tblInfo.PlacementPolicyRef != nil { + if _, ok := b.updatePolicies[tblInfo.PlacementPolicyRef.ID]; ok { + b.markTableBundleShouldUpdate(tblInfo.ID) + } + } -func (b *Builder) applyExchangeTablePartitionV2(m *meta.Meta, diff *model.SchemaDiff) ([]int64, error) { - panic("TODO") + if tblInfo.Partition != nil { + for _, par := range tblInfo.Partition.Definitions { + if _, ok := b.updatePartitions[par.ID]; ok { + b.markTableBundleShouldUpdate(tblInfo.ID) + } + } + } + } + } } diff --git a/pkg/infoschema/infoschema_v2_test.go b/pkg/infoschema/infoschema_v2_test.go index 029a0ecf8fd70..22595f3d49c69 100644 --- a/pkg/infoschema/infoschema_v2_test.go +++ b/pkg/infoschema/infoschema_v2_test.go @@ -15,6 +15,7 @@ package infoschema import ( + "math" "testing" "github.com/pingcap/tidb/pkg/infoschema/internal" @@ -38,13 +39,13 @@ func TestV2Basic(t *testing.T) { is.Data.addDB(1, dbInfo) internal.AddDB(t, r.Store(), dbInfo) tblInfo := internal.MockTableInfo(t, r.Store(), tableName.O) - is.Data.add(tableItem{schemaName.L, dbInfo.ID, tableName.L, tblInfo.ID, 2}, internal.MockTable(t, r.Store(), tblInfo)) + is.Data.add(tableItem{schemaName.L, dbInfo.ID, tableName.L, tblInfo.ID, 2, false}, internal.MockTable(t, r.Store(), tblInfo)) internal.AddTable(t, r.Store(), dbInfo, tblInfo) require.Equal(t, 1, len(is.AllSchemas())) require.Equal(t, 0, len(is.SchemaTables(is.AllSchemas()[0].Name))) ver, err := r.Store().CurrentVersion(kv.GlobalTxnScope) require.NoError(t, err) - is.schemaVersion = 2 + is.base().schemaMetaVersion = 2 is.ts = ver.Ver require.Equal(t, 1, len(is.AllSchemas())) require.Equal(t, 1, len(is.SchemaTables(is.AllSchemas()[0].Name))) @@ -79,7 +80,7 @@ func TestMisc(t *testing.T) { builder, err := NewBuilder(r, nil, NewData()).InitWithDBInfos(nil, nil, nil, 1) require.NoError(t, err) - is := builder.Build() + is := builder.Build(math.MaxUint64) require.Len(t, is.AllResourceGroups(), 0) // test create resource group @@ -89,7 +90,7 @@ func TestMisc(t *testing.T) { require.NoError(t, err) err = applyCreateOrAlterResourceGroup(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: resourceGroupInfo.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllResourceGroups(), 1) getResourceGroupInfo, ok := is.ResourceGroupByName(resourceGroupInfo.Name) require.True(t, ok) @@ -103,7 +104,7 @@ func TestMisc(t *testing.T) { require.NoError(t, err) err = applyCreateOrAlterResourceGroup(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: resourceGroupInfo2.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllResourceGroups(), 2) getResourceGroupInfo, ok = is.ResourceGroupByName(resourceGroupInfo2.Name) require.True(t, ok) @@ -117,7 +118,7 @@ func TestMisc(t *testing.T) { require.NoError(t, err) err = applyCreateOrAlterResourceGroup(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: resourceGroupInfo.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllResourceGroups(), 2) getResourceGroupInfo, ok = is.ResourceGroupByName(resourceGroupInfo.Name) require.True(t, ok) @@ -129,7 +130,7 @@ func TestMisc(t *testing.T) { txn, err = r.Store().Begin() require.NoError(t, err) _ = applyDropResourceGroup(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: resourceGroupInfo.ID}) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllResourceGroups(), 1) getResourceGroupInfo, ok = is.ResourceGroupByName(resourceGroupInfo2.Name) require.True(t, ok) @@ -143,7 +144,7 @@ func TestMisc(t *testing.T) { require.NoError(t, err) err = applyCreatePolicy(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: policyInfo.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllPlacementPolicies(), 1) getPolicyInfo, ok := is.PolicyByName(policyInfo.Name) require.True(t, ok) @@ -157,7 +158,7 @@ func TestMisc(t *testing.T) { require.NoError(t, err) err = applyCreatePolicy(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: policyInfo2.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllPlacementPolicies(), 2) getPolicyInfo, ok = is.PolicyByName(policyInfo2.Name) require.True(t, ok) @@ -169,9 +170,9 @@ func TestMisc(t *testing.T) { internal.UpdatePolicy(t, r.Store(), policyInfo) txn, err = r.Store().Begin() require.NoError(t, err) - err = applyCreatePolicy(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: policyInfo.ID}) + _, err = applyAlterPolicy(builder, meta.NewMeta(txn), &model.SchemaDiff{SchemaID: policyInfo.ID}) require.NoError(t, err) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllPlacementPolicies(), 2) getPolicyInfo, ok = is.PolicyByName(policyInfo.Name) require.True(t, ok) @@ -183,10 +184,91 @@ func TestMisc(t *testing.T) { txn, err = r.Store().Begin() require.NoError(t, err) _ = applyDropPolicy(builder, policyInfo.ID) - is = builder.Build() + is = builder.Build(math.MaxUint64) require.Len(t, is.AllPlacementPolicies(), 1) getPolicyInfo, ok = is.PolicyByName(policyInfo2.Name) require.True(t, ok) require.Equal(t, policyInfo2, getPolicyInfo) require.NoError(t, txn.Rollback()) } + +func TestBundles(t *testing.T) { + r := internal.CreateAutoIDRequirement(t) + defer func() { + r.Store().Close() + }() + + schemaName := model.NewCIStr("testDB") + tableName := model.NewCIStr("test") + builder, err := NewBuilder(r, nil, NewData()).InitWithDBInfos(nil, nil, nil, 1) + require.NoError(t, err) + is := builder.Build(math.MaxUint64) + require.Equal(t, 2, len(is.AllSchemas())) + + // create database + dbInfo := internal.MockDBInfo(t, r.Store(), schemaName.O) + internal.AddDB(t, r.Store(), dbInfo) + txn, err := r.Store().Begin() + require.NoError(t, err) + _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionCreateSchema, Version: 1, SchemaID: dbInfo.ID}) + require.NoError(t, err) + is = builder.Build(math.MaxUint64) + require.Equal(t, 3, len(is.AllSchemas())) + require.NoError(t, txn.Rollback()) + + // create table + tblInfo := internal.MockTableInfo(t, r.Store(), tableName.O) + tblInfo.Partition = &model.PartitionInfo{Definitions: []model.PartitionDefinition{{ID: 1}, {ID: 2}}} + internal.AddTable(t, r.Store(), dbInfo, tblInfo) + txn, err = r.Store().Begin() + require.NoError(t, err) + _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionCreateTable, Version: 2, SchemaID: dbInfo.ID, TableID: tblInfo.ID}) + require.NoError(t, err) + is = builder.Build(math.MaxUint64) + require.Equal(t, 1, len(is.SchemaTables(dbInfo.Name))) + require.NoError(t, txn.Rollback()) + + // test create policy + policyInfo := internal.MockPolicyInfo(t, r.Store(), "test") + internal.CreatePolicy(t, r.Store(), policyInfo) + txn, err = r.Store().Begin() + require.NoError(t, err) + _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionCreatePlacementPolicy, Version: 3, SchemaID: policyInfo.ID}) + require.NoError(t, err) + is = builder.Build(math.MaxUint64) + require.Len(t, is.AllPlacementPolicies(), 1) + getPolicyInfo, ok := is.PolicyByName(policyInfo.Name) + require.True(t, ok) + require.Equal(t, policyInfo, getPolicyInfo) + require.NoError(t, txn.Rollback()) + + // markTableBundleShouldUpdate + // test alter table placement + policyRefInfo := internal.MockPolicyRefInfo(t, r.Store(), "test") + tblInfo.PlacementPolicyRef = policyRefInfo + internal.UpdateTable(t, r.Store(), dbInfo, tblInfo) + txn, err = r.Store().Begin() + require.NoError(t, err) + _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionAlterTablePlacement, Version: 4, SchemaID: dbInfo.ID, TableID: tblInfo.ID}) + require.NoError(t, err) + is = builder.Build(math.MaxUint64) + getTableInfo, err := is.TableByName(schemaName, tableName) + require.NoError(t, err) + require.Equal(t, policyRefInfo, getTableInfo.Meta().PlacementPolicyRef) + require.NoError(t, txn.Rollback()) + + // markBundlesReferPolicyShouldUpdate + // test alter policy + policyInfo.State = model.StatePublic + internal.UpdatePolicy(t, r.Store(), policyInfo) + txn, err = r.Store().Begin() + require.NoError(t, err) + _, err = builder.ApplyDiff(meta.NewMeta(txn), &model.SchemaDiff{Type: model.ActionAlterPlacementPolicy, Version: 5, SchemaID: policyInfo.ID}) + require.NoError(t, err) + is = builder.Build(math.MaxUint64) + getTableInfo, err = is.TableByName(schemaName, tableName) + require.NoError(t, err) + getPolicyInfo, ok = is.PolicyByName(getTableInfo.Meta().PlacementPolicyRef.Name) + require.True(t, ok) + require.Equal(t, policyInfo, getPolicyInfo) +} diff --git a/pkg/infoschema/interface.go b/pkg/infoschema/interface.go index 06858a8574f8c..202b225d6230d 100644 --- a/pkg/infoschema/interface.go +++ b/pkg/infoschema/interface.go @@ -36,6 +36,7 @@ type InfoSchema interface { SchemaMetaVersion() int64 FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition) Misc + base() *infoSchema } // Misc contains the methods that are not closely related to InfoSchema. diff --git a/pkg/infoschema/internal/testkit.go b/pkg/infoschema/internal/testkit.go index 600307bf40ae9..e5bb87f368abf 100644 --- a/pkg/infoschema/internal/testkit.go +++ b/pkg/infoschema/internal/testkit.go @@ -200,6 +200,16 @@ func MockPolicyInfo(t *testing.T, store kv.Storage, policyName string) *model.Po } } +// MockPolicyRefInfo mock policy ref info for testing. +func MockPolicyRefInfo(t *testing.T, store kv.Storage, policyName string) *model.PolicyRefInfo { + id, err := GenGlobalID(store) + require.NoError(t, err) + return &model.PolicyRefInfo{ + ID: id, + Name: model.NewCIStr(policyName), + } +} + // AddTable add mock table for testing. func AddTable(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) { ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) @@ -211,6 +221,17 @@ func AddTable(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *mod require.NoError(t, err) } +// UpdateTable update mock table for testing. +func UpdateTable(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) + err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error { + err := meta.NewMeta(txn).UpdateTable(dbInfo.ID, tblInfo) + require.NoError(t, err) + return errors.Trace(err) + }) + require.NoError(t, err) +} + // DropTable drop mock table for testing. func DropTable(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblID int64, tblName string) { ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) @@ -245,6 +266,17 @@ func DropDB(t *testing.T, store kv.Storage, dbInfo *model.DBInfo) { require.NoError(t, err) } +// UpdateDB update mock db for testing. +func UpdateDB(t *testing.T, store kv.Storage, dbInfo *model.DBInfo) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) + err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error { + err := meta.NewMeta(txn).UpdateDatabase(dbInfo) + require.NoError(t, err) + return errors.Trace(err) + }) + require.NoError(t, err) +} + // AddResourceGroup add mock resource group for testing. func AddResourceGroup(t *testing.T, store kv.Storage, group *model.ResourceGroupInfo) { ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) diff --git a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go index c93004b9ee4fe..8315c968fa2ae 100644 --- a/pkg/infoschema/test/clustertablestest/cluster_tables_test.go +++ b/pkg/infoschema/test/clustertablestest/cluster_tables_test.go @@ -714,6 +714,8 @@ select * from t1; tk.MustExec("set global tidb_mem_oom_action='CANCEL'") defer tk.MustExec("set global tidb_mem_oom_action='LOG'") tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name())) + // Align with the timezone in slow log files + tk.MustExec("set @@time_zone='+08:00'") checkFn := func(quota int) { tk.MustExec("set tidb_mem_quota_query=" + strconv.Itoa(quota)) // session @@ -1262,12 +1264,14 @@ func TestErrorCasesCreateBindingFromHistory(t *testing.T) { sql := "select * from t1 where t1.id in (select id from t2)" tk.MustExec(sql) planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows() - tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with sub query") + tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0])) + tk.MustQuery(`show warnings`).Check(testkit.Rows("Warning 1105 auto-generated hint for queries with sub queries might not be complete, the plan might change even after creating this binding")) sql = "select * from t1, t2, t3 where t1.id = t2.id and t2.id = t3.id" tk.MustExec(sql) planDigest = tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.statements_summary where query_sample_text = '%s'", sql)).Rows() - tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with more than two table join") + tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0])) + tk.MustQuery(`show warnings`).Check(testkit.Rows("Warning 1105 auto-generated hint for queries with more than 3 table join might not be complete, the plan might change even after creating this binding")) } // withMockTiFlash sets the mockStore to have N TiFlash stores (naming as tiflash0, tiflash1, ...). @@ -1313,7 +1317,8 @@ func TestBindingFromHistoryWithTiFlashBindable(t *testing.T) { sql := "select * from t" tk.MustExec(sql) planDigest := tk.MustQuery(fmt.Sprintf("select plan_digest from information_schema.cluster_statements_summary where query_sample_text = '%s'", sql)).Rows() - tk.MustGetErrMsg(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0]), "can't create binding for query with tiflash engine") + tk.MustExec(fmt.Sprintf("create binding from history using plan digest '%s'", planDigest[0][0])) + tk.MustQuery(`show warnings`).Check(testkit.Rows("Warning 1105 auto-generated hint for queries accessing TiFlash might not be complete, the plan might change even after creating this binding")) } func TestSetBindingStatusBySQLDigest(t *testing.T) { diff --git a/pkg/infoschema/test/infoschemav2test/v2_test.go b/pkg/infoschema/test/infoschemav2test/v2_test.go index 0186c99013cb3..3a351b9cc9a16 100644 --- a/pkg/infoschema/test/infoschemav2test/v2_test.go +++ b/pkg/infoschema/test/infoschemav2test/v2_test.go @@ -25,6 +25,7 @@ import ( ) func TestSpecialSchemas(t *testing.T) { + t.Skip("This feature is not enabled yet") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) diff --git a/pkg/kv/interface_mock_test.go b/pkg/kv/interface_mock_test.go index 83b6be1a662a4..9ea952cfb750f 100644 --- a/pkg/kv/interface_mock_test.go +++ b/pkg/kv/interface_mock_test.go @@ -182,6 +182,7 @@ func (t *mockTxn) CancelFairLocking(_ context.Context) error { return nil } func (t *mockTxn) DoneFairLocking(_ context.Context) error { return nil } func (t *mockTxn) IsInFairLockingMode() bool { return false } func (t *mockTxn) IsPipelined() bool { return false } +func (t *mockTxn) MayFlush() error { return nil } // newMockTxn new a mockTxn. func newMockTxn() Transaction { diff --git a/pkg/kv/kv.go b/pkg/kv/kv.go index bec0221e57f05..39c59f2dd5144 100644 --- a/pkg/kv/kv.go +++ b/pkg/kv/kv.go @@ -189,8 +189,11 @@ type MemBuffer interface { // RemoveFromBuffer removes the entry from the buffer. It's used for testing. RemoveFromBuffer(Key) - // MayFlush will be called in pipelined txn - MayFlush() error + // GetLocal checks if the key exists in the buffer in local memory. + GetLocal(context.Context, []byte) ([]byte, error) + + // BatchGet gets values from the memory buffer. + BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) } // FindKeysInStage returns all keys in the given stage that satisfies the given condition. @@ -281,6 +284,8 @@ type Transaction interface { UpdateMemBufferFlags(key []byte, flags ...FlagsOp) // IsPipelined returns whether the transaction is used for pipelined DML. IsPipelined() bool + // MayFlush flush the pipelined memdb if the keys or size exceeds threshold, no effect for standard DML. + MayFlush() error } // AssertionProto is an interface defined for the assertion protocol. diff --git a/pkg/metrics/bindinfo.go b/pkg/metrics/bindinfo.go index 71ba09f686209..3139e49bc79b5 100644 --- a/pkg/metrics/bindinfo.go +++ b/pkg/metrics/bindinfo.go @@ -18,16 +18,48 @@ import "github.com/prometheus/client_golang/prometheus" // bindinfo metrics. var ( - BindUsageCounter *prometheus.CounterVec + BindingCacheHitCounter prometheus.Counter + BindingCacheMissCounter prometheus.Counter + BindingCacheMemUsage prometheus.Gauge + BindingCacheMemLimit prometheus.Gauge + BindingCacheNumBindings prometheus.Gauge ) // InitBindInfoMetrics initializes bindinfo metrics. func InitBindInfoMetrics() { - BindUsageCounter = NewCounterVec( + BindingCacheHitCounter = NewCounter( prometheus.CounterOpts{ Namespace: "tidb", - Subsystem: "bindinfo", - Name: "bind_usage_counter", - Help: "Counter of query using sql bind", - }, []string{LabelScope}) + Subsystem: "server", + Name: "binding_cache_hit_total", + Help: "Counter of binding cache hit.", + }) + BindingCacheMissCounter = NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "binding_cache_miss_total", + Help: "Counter of binding cache miss.", + }) + BindingCacheMemUsage = NewGauge( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "binding_cache_mem_usage", + Help: "Memory usage of binding cache.", + }) + BindingCacheMemLimit = NewGauge( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "binding_cache_mem_limit", + Help: "Memory limit of binding cache.", + }) + BindingCacheNumBindings = NewGauge( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "binding_cache_num_bindings", + Help: "Number of bindings in binding cache.", + }) } diff --git a/pkg/metrics/disttask.go b/pkg/metrics/disttask.go index f5642d2087bee..ac9b8d767a800 100644 --- a/pkg/metrics/disttask.go +++ b/pkg/metrics/disttask.go @@ -41,12 +41,10 @@ const ( var ( //DistTaskGauge is the gauge of dist task count. DistTaskGauge *prometheus.GaugeVec - //DistTaskStarttimeGauge is the gauge of dist task count. - DistTaskStarttimeGauge *prometheus.GaugeVec - // DistTaskSubTaskCntGauge is the gauge of dist task subtask count. - DistTaskSubTaskCntGauge *prometheus.GaugeVec - // DistTaskSubTaskDurationGauge is the gauge of dist task subtask duration. - DistTaskSubTaskDurationGauge *prometheus.GaugeVec + //DistTaskStartTimeGauge is the gauge of dist task count. + DistTaskStartTimeGauge *prometheus.GaugeVec + // DistTaskUsedSlotsGauge is the gauge of used slots on executor node. + DistTaskUsedSlotsGauge *prometheus.GaugeVec ) // InitDistTaskMetrics initializes disttask metrics. @@ -59,31 +57,38 @@ func InitDistTaskMetrics() { Help: "Gauge of disttask.", }, []string{lblTaskType, lblTaskStatus}) - DistTaskStarttimeGauge = NewGaugeVec( + DistTaskStartTimeGauge = NewGaugeVec( prometheus.GaugeOpts{ Namespace: "tidb", Subsystem: "disttask", Name: "start_time", Help: "Gauge of start_time of disttask.", }, []string{lblTaskType, lblTaskStatus, lblTaskID}) + DistTaskUsedSlotsGauge = NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "disttask", + Name: "used_slots", + Help: "Gauge of used slots on a executor node.", + }, []string{"service_scope"}) } // UpdateMetricsForAddTask update metrics when a task is added func UpdateMetricsForAddTask(task *proto.TaskBase) { DistTaskGauge.WithLabelValues(task.Type.String(), WaitingStatus).Inc() - DistTaskStarttimeGauge.WithLabelValues(task.Type.String(), WaitingStatus, fmt.Sprint(task.ID)).Set(float64(time.Now().UnixMicro())) + DistTaskStartTimeGauge.WithLabelValues(task.Type.String(), WaitingStatus, fmt.Sprint(task.ID)).Set(float64(time.Now().UnixMicro())) } // UpdateMetricsForScheduleTask update metrics when a task is added func UpdateMetricsForScheduleTask(id int64, taskType proto.TaskType) { DistTaskGauge.WithLabelValues(taskType.String(), WaitingStatus).Dec() - DistTaskStarttimeGauge.DeleteLabelValues(taskType.String(), WaitingStatus, fmt.Sprint(id)) - DistTaskStarttimeGauge.WithLabelValues(taskType.String(), SchedulingStatus, fmt.Sprint(id)).SetToCurrentTime() + DistTaskStartTimeGauge.DeleteLabelValues(taskType.String(), WaitingStatus, fmt.Sprint(id)) + DistTaskStartTimeGauge.WithLabelValues(taskType.String(), SchedulingStatus, fmt.Sprint(id)).SetToCurrentTime() } // UpdateMetricsForRunTask update metrics when a task starts running func UpdateMetricsForRunTask(task *proto.Task) { - DistTaskStarttimeGauge.DeleteLabelValues(task.Type.String(), SchedulingStatus, fmt.Sprint(task.ID)) + DistTaskStartTimeGauge.DeleteLabelValues(task.Type.String(), SchedulingStatus, fmt.Sprint(task.ID)) DistTaskGauge.WithLabelValues(task.Type.String(), SchedulingStatus).Dec() DistTaskGauge.WithLabelValues(task.Type.String(), RunningStatus).Inc() } diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index e6d7aad5b081f..1838ada2fd85e 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -2657,6 +2657,114 @@ "alignLevel": null } }, + { + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 2, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 292, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}[1m])", + "legendFormat": "gc-rate - {{instance}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tidb\"}", + "hide": false, + "legendFormat": "gomemlimit - {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Runtime GC rate and GOMEMLIMIT", + "type": "timeseries" + }, { "aliasColors": {}, "bars": false, @@ -11419,6 +11527,188 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The slow score calculated by time cost of some specific TiKV RPC requests.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 337, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.27", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(tidb_tikvclient_store_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "store-{{store}}", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Client-side slow score", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:327", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:328", + "format": "Bps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The slow score calculated by TiKV rafstore and sent to TiDB via health feedback.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 36 + }, + "hiddenSeries": false, + "id": 338, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.27", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(tidb_tikvclient_feedback_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "store-{{store}}", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TiKV-side slow score", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:327", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:328", + "format": "Bps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } } ], "repeat": null, @@ -15488,25 +15778,118 @@ ], "title": "Uncompleted Subtask Distribution on TiDB Nodes", "type": "piechart" - } - ], - "repeat": null, - "title": "Dist Execute Framework", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 150, - "panels": [ + }, { - "aliasColors": {}, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 328, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "tidb_disttask_used_slots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "legendFormat": "Used-{{instance}} - {{service_scope}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\", component=\"tidb\"}", + "hide": false, + "legendFormat": "Capacity - {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Slots usage", + "type": "timeseries" + } + ], + "repeat": null, + "title": "Dist Execute Framework", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 150, + "panels": [ + { + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, @@ -15589,8 +15972,8 @@ }, "yaxes": [ { - "$$hashKey": "object:354", - "format": "s", + "$$hashKey": "object:354", + "format": "s", "label": null, "logBase": 2, "max": null, @@ -15598,7 +15981,7 @@ "show": true }, { - "$$hashKey": "object:355", + "$$hashKey": "object:355", "label": null, "logBase": 1, "max": null, @@ -16781,7 +17164,7 @@ "alignLevel": null } }, - { + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -16978,10 +17361,317 @@ "align": false, "alignLevel": null } + }, + { + "type": "graph", + "title": "Binding Cache Memory Usage", + "gridPos": { + "x": 0, + "y": 56, + "w": 8, + "h": 8 + }, + "id": 23763572008, + "targets": [ + { + "expr": "tidb_server_binding_cache_mem_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "legendFormat": "{{instance}} usage", + "interval": "", + "exemplar": true, + "refId": "A", + "queryType": "randomWalk" + }, + { + "expr": "tidb_server_binding_cache_mem_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "legendFormat": "{{instance}} limit", + "interval": "", + "exemplar": true, + "refId": "B", + "hide": false + } + ], + "options": { + "alertThreshold": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "pluginVersion": "7.5.10", + "renderer": "flot", + "yaxes": [ + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "bytes", + "$$hashKey": "object:80" + }, + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short", + "$$hashKey": "object:81" + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [], + "buckets": null + }, + "yaxis": { + "align": false, + "alignLevel": null + }, + "lines": true, + "fill": 1, + "linewidth": 1, + "dashLength": 10, + "spaceLength": 10, + "pointradius": 2, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "nullPointMode": "null", + "tooltip": { + "value_type": "individual", + "shared": true, + "sort": 0 + }, + "aliasColors": {}, + "seriesOverrides": [], + "thresholds": [], + "timeRegions": [], + "datasource": "${DS_TEST-CLUSTER}", + "fillGradient": 0, + "dashes": false, + "hiddenSeries": false, + "points": false, + "bars": false, + "stack": false, + "percentage": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null + }, + { + "type": "graph", + "title": "Binding Cache Hit / Miss OPS", + "gridPos": { + "x": 8, + "y": 56, + "w": 8, + "h": 8 + }, + "id": 23763572009, + "targets": [ + { + "expr": "sum(rate(tidb_server_binding_cache_hit_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "legendFormat": "{{instance}} hit", + "interval": "", + "exemplar": true, + "refId": "A", + "queryType": "randomWalk" + }, + { + "expr": "sum(rate(tidb_server_binding_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "legendFormat": "{{instance}} miss", + "interval": "", + "exemplar": true, + "refId": "B", + "hide": false + } + ], + "options": { + "alertThreshold": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "pluginVersion": "7.5.10", + "renderer": "flot", + "yaxes": [ + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short", + "$$hashKey": "object:80" + }, + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short", + "$$hashKey": "object:81" + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [], + "buckets": null + }, + "yaxis": { + "align": false, + "alignLevel": null + }, + "lines": true, + "fill": 1, + "linewidth": 1, + "dashLength": 10, + "spaceLength": 10, + "pointradius": 2, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "nullPointMode": "null", + "tooltip": { + "value_type": "individual", + "shared": true, + "sort": 0 + }, + "aliasColors": {}, + "seriesOverrides": [], + "thresholds": [], + "timeRegions": [], + "datasource": "${DS_TEST-CLUSTER}", + "fillGradient": 0, + "dashes": false, + "hiddenSeries": false, + "points": false, + "bars": false, + "stack": false, + "percentage": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null + }, + { + "type": "graph", + "title": "Number of Bindings in Cache", + "gridPos": { + "x": 16, + "y": 56, + "w": 8, + "h": 8 + }, + "id": 23763572010, + "targets": [ + { + "expr": "tidb_server_binding_cache_num_bindings{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "interval": "", + "exemplar": true, + "refId": "A", + "queryType": "randomWalk" + } + ], + "options": { + "alertThreshold": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "pluginVersion": "7.5.10", + "renderer": "flot", + "yaxes": [ + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short", + "$$hashKey": "object:80" + }, + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short", + "$$hashKey": "object:81" + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [], + "buckets": null + }, + "yaxis": { + "align": false, + "alignLevel": null + }, + "lines": true, + "fill": 1, + "linewidth": 1, + "dashLength": 10, + "spaceLength": 10, + "pointradius": 2, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "nullPointMode": "null", + "tooltip": { + "value_type": "individual", + "shared": true, + "sort": 0 + }, + "aliasColors": {}, + "seriesOverrides": [], + "thresholds": [], + "timeRegions": [], + "datasource": "${DS_TEST-CLUSTER}", + "fillGradient": 0, + "dashes": false, + "hiddenSeries": false, + "points": false, + "bars": false, + "stack": false, + "percentage": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null } ], "repeat": null, - "title": "Statistics", + "title": "Statistics & Plan Management", "type": "row" }, { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 3a2e0301ec86f..8d1530c3eb080 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -123,7 +123,6 @@ func RegisterMetrics() { prometheus.MustRegister(AutoAnalyzeHistogram) prometheus.MustRegister(AutoIDHistogram) prometheus.MustRegister(BatchAddIdxHistogram) - prometheus.MustRegister(BindUsageCounter) prometheus.MustRegister(CampaignOwnerCounter) prometheus.MustRegister(ConnGauge) prometheus.MustRegister(DisconnectionCounter) @@ -269,7 +268,8 @@ func RegisterMetrics() { prometheus.MustRegister(PlanReplayerRegisterTaskGauge) prometheus.MustRegister(DistTaskGauge) - prometheus.MustRegister(DistTaskStarttimeGauge) + prometheus.MustRegister(DistTaskStartTimeGauge) + prometheus.MustRegister(DistTaskUsedSlotsGauge) prometheus.MustRegister(RunawayCheckerCounter) prometheus.MustRegister(GlobalSortWriteToCloudStorageDuration) prometheus.MustRegister(GlobalSortWriteToCloudStorageRate) @@ -278,6 +278,12 @@ func RegisterMetrics() { prometheus.MustRegister(GlobalSortIngestWorkerCnt) prometheus.MustRegister(AddIndexScanRate) + prometheus.MustRegister(BindingCacheHitCounter) + prometheus.MustRegister(BindingCacheMissCounter) + prometheus.MustRegister(BindingCacheMemUsage) + prometheus.MustRegister(BindingCacheMemLimit) + prometheus.MustRegister(BindingCacheNumBindings) + tikvmetrics.InitMetrics(TiDB, TiKVClient) tikvmetrics.RegisterMetrics() tikvmetrics.TiKVPanicCounter = PanicCounter // reset tidb metrics for tikv metrics diff --git a/pkg/parser/digester.go b/pkg/parser/digester.go index a5efa6c9ed33b..b4fbcccce57dc 100644 --- a/pkg/parser/digester.go +++ b/pkg/parser/digester.go @@ -23,6 +23,7 @@ import ( "sync" "unsafe" + "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser/charset" ) @@ -87,7 +88,7 @@ func DigestNormalized(normalized string) (digest *Digest) { // for example, when "ON": Normalize('select 1 from b where a = 1') => 'select ? from b where a = ?' // for example, when "MARKER": Normalize('select 1 from b where a = 1') => 'select ‹1› from b where a = ‹1›' func Normalize(sql string, redact string) (result string) { - if redact == "" || redact == "OFF" { + if redact == "" || redact == errors.RedactLogDisable { return sql } d := digesterPool.Get().(*sqlDigester) @@ -115,7 +116,7 @@ func NormalizeForBinding(sql string, forPlanReplayerReload bool) (result string) // for example: Normalize('select /*+ use_index(t, primary) */ 1 from b where a = 1') => 'select /*+ use_index(t, primary) */ ? from b where a = ?' func NormalizeKeepHint(sql string) (result string) { d := digesterPool.Get().(*sqlDigester) - result = d.doNormalize(sql, "ON", true) + result = d.doNormalize(sql, errors.RedactLogEnable, true) digesterPool.Put(d) return } @@ -167,7 +168,7 @@ func (d *sqlDigester) doDigestNormalized(normalized string) (digest *Digest) { } func (d *sqlDigester) doDigest(sql string) (digest *Digest) { - d.normalize(sql, "ON", false, false, false) + d.normalize(sql, errors.RedactLogEnable, false, false, false) d.hasher.Write(d.buffer.Bytes()) d.buffer.Reset() digest = NewDigest(d.hasher.Sum(nil)) @@ -183,14 +184,14 @@ func (d *sqlDigester) doNormalize(sql string, redact string, keepHint bool) (res } func (d *sqlDigester) doNormalizeForBinding(sql string, keepHint bool, forPlanReplayerReload bool) (result string) { - d.normalize(sql, "ON", keepHint, true, forPlanReplayerReload) + d.normalize(sql, errors.RedactLogEnable, keepHint, true, forPlanReplayerReload) result = d.buffer.String() d.buffer.Reset() return } func (d *sqlDigester) doNormalizeDigest(sql string) (normalized string, digest *Digest) { - d.normalize(sql, "ON", false, false, false) + d.normalize(sql, errors.RedactLogEnable, false, false, false) normalized = d.buffer.String() d.hasher.Write(d.buffer.Bytes()) d.buffer.Reset() @@ -200,7 +201,7 @@ func (d *sqlDigester) doNormalizeDigest(sql string) (normalized string, digest * } func (d *sqlDigester) doNormalizeDigestForBinding(sql string) (normalized string, digest *Digest) { - d.normalize(sql, "ON", false, true, false) + d.normalize(sql, errors.RedactLogEnable, false, true, false) normalized = d.buffer.String() d.hasher.Write(d.buffer.Bytes()) d.buffer.Reset() @@ -324,7 +325,7 @@ func (d *sqlDigester) reduceLit(currTok *token, redact string, forBinding bool, return } - if redact == "MARKER" && !forBinding && !forPlanReplayer { + if redact == errors.RedactLogMarker && !forBinding && !forPlanReplayer { switch currTok.lit { case "?", "*": return diff --git a/pkg/parser/go.mod b/pkg/parser/go.mod index dbe7eb28986d8..b5f9b3b646370 100644 --- a/pkg/parser/go.mod +++ b/pkg/parser/go.mod @@ -7,7 +7,7 @@ require ( github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 github.com/go-sql-driver/mysql v1.7.1 - github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 + github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c github.com/pingcap/log v1.1.0 github.com/stretchr/testify v1.8.4 diff --git a/pkg/parser/go.sum b/pkg/parser/go.sum index 562fbb0b57450..d5f140b45ed5d 100644 --- a/pkg/parser/go.sum +++ b/pkg/parser/go.sum @@ -22,6 +22,10 @@ github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM= github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk= +github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/errors v0.11.5-0.20240311081613-f97970b88865 h1:xzbnMPvFtnqLsY5HLALUov6JU1ONyfQUXVurE/Nqb8Y= +github.com/pingcap/errors v0.11.5-0.20240311081613-f97970b88865/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8= diff --git a/pkg/parser/model/model.go b/pkg/parser/model/model.go index 763797ebbb14f..12aedb0671755 100644 --- a/pkg/parser/model/model.go +++ b/pkg/parser/model/model.go @@ -1780,9 +1780,16 @@ func (cis *CIStr) MemoryUsage() (sum int64) { // TableItemID is composed by table ID and column/index ID type TableItemID struct { - TableID int64 - ID int64 - IsIndex bool + TableID int64 + ID int64 + IsIndex bool + IsSyncLoadFailed bool +} + +// StatsLoadItem represents the load unit for statistics's memory loading. +type StatsLoadItem struct { + TableItemID + FullLoad bool } // PolicyRefInfo is the struct to refer the placement policy. @@ -1924,6 +1931,10 @@ func (p *PlacementSettings) String() string { writeSettingStringToBuilder(sb, "LEARNER_CONSTRAINTS", p.LearnerConstraints) } + if len(p.SurvivalPreferences) > 0 { + writeSettingStringToBuilder(sb, "SURVIVAL_PREFERENCES", p.SurvivalPreferences) + } + return sb.String() } diff --git a/pkg/planner/cardinality/cross_estimation.go b/pkg/planner/cardinality/cross_estimation.go index d045aef70ed93..59028f24e82b8 100644 --- a/pkg/planner/cardinality/cross_estimation.go +++ b/pkg/planner/cardinality/cross_estimation.go @@ -189,13 +189,13 @@ func getColumnRangeCounts(sctx context.PlanContext, colID int64, ranges []*range for i, ran := range ranges { if idxID >= 0 { idxHist := histColl.Indices[idxID] - if idxHist == nil || idxHist.IsInvalid(sctx, false) { + if statistics.IndexStatsIsInvalid(sctx, idxHist, histColl, idxID) { return nil, false } count, err = GetRowCountByIndexRanges(sctx, histColl, idxID, []*ranger.Range{ran}) } else { - colHist, ok := histColl.Columns[colID] - if !ok || colHist.IsInvalid(sctx, false) { + colHist := histColl.Columns[colID] + if statistics.ColumnStatsIsInvalid(colHist, sctx, histColl, colID) { return nil, false } count, err = GetRowCountByColumnRanges(sctx, histColl, colID, []*ranger.Range{ran}) diff --git a/pkg/planner/cardinality/pseudo.go b/pkg/planner/cardinality/pseudo.go index 18c7a6d1d11aa..3b0b1c3dfa45c 100644 --- a/pkg/planner/cardinality/pseudo.go +++ b/pkg/planner/cardinality/pseudo.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/context" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/ranger" @@ -40,7 +41,7 @@ func PseudoAvgCountPerValue(t *statistics.Table) float64 { return float64(t.RealtimeCount) / pseudoEqualRate } -func pseudoSelectivity(coll *statistics.HistColl, exprs []expression.Expression) float64 { +func pseudoSelectivity(sctx context.PlanContext, coll *statistics.HistColl, exprs []expression.Expression) float64 { minFactor := selectionFactor colExists := make(map[string]bool) for _, expr := range exprs { @@ -52,6 +53,7 @@ func pseudoSelectivity(coll *statistics.HistColl, exprs []expression.Expression) if colID == unknownColumnID { continue } + statistics.ColumnStatsIsInvalid((*statistics.Column)(nil), sctx, coll, colID) switch fun.FuncName.L { case ast.EQ, ast.NullEQ, ast.In: minFactor = math.Min(minFactor, 1.0/pseudoEqualRate) @@ -73,17 +75,19 @@ func pseudoSelectivity(coll *statistics.HistColl, exprs []expression.Expression) } // use the unique key info for _, idx := range coll.Indices { - if !idx.Info.Unique { - continue - } unique := true + firstMatch := false for _, col := range idx.Info.Columns { if !colExists[col.Name.L] { unique = false break } + firstMatch = true + } + if firstMatch { + statistics.IndexStatsIsInvalid(sctx, (*statistics.Index)(nil), coll, idx.ID) } - if unique { + if idx.Info.Unique && unique { return 1.0 / float64(coll.RealtimeCount) } } diff --git a/pkg/planner/cardinality/row_count_column.go b/pkg/planner/cardinality/row_count_column.go index a57b5f7873cea..6f11a95681860 100644 --- a/pkg/planner/cardinality/row_count_column.go +++ b/pkg/planner/cardinality/row_count_column.go @@ -49,7 +49,7 @@ func GetRowCountByColumnRanges(sctx context.PlanContext, coll *statistics.HistCo if c != nil && c.Info != nil { name = c.Info.Name.O } - if !ok || c.IsInvalid(sctx, coll.Pseudo) { + if statistics.ColumnStatsIsInvalid(c, sctx, coll, colID) { result, err = getPseudoRowCountByColumnRanges(sc.TypeCtx(), float64(coll.RealtimeCount), colRanges, 0) if err == nil && sc.EnableOptimizerCETrace && ok { ceTraceRange(sctx, coll.PhysicalID, []string{c.Info.Name.O}, colRanges, "Column Stats-Pseudo", uint64(result)) @@ -87,7 +87,7 @@ func GetRowCountByIntColumnRanges(sctx context.PlanContext, coll *statistics.His if c != nil && c.Info != nil { name = c.Info.Name.O } - if !ok || c.IsInvalid(sctx, coll.Pseudo) { + if statistics.ColumnStatsIsInvalid(c, sctx, coll, colID) { if len(intRanges) == 0 { return 0, nil } @@ -282,15 +282,17 @@ func GetColumnRowCount(sctx context.PlanContext, c *statistics.Column, ranges [] cnt = mathutil.Clamp(cnt, 0, c.TotalRowCount()) // If the current table row count has changed, we should scale the row count accordingly. - cnt *= c.GetIncreaseFactor(realtimeRowCount) + increaseFactor := c.GetIncreaseFactor(realtimeRowCount) + cnt *= increaseFactor - histNDV := c.NDV - if c.StatsVer == statistics.Version2 { - histNDV = histNDV - int64(c.TopN.Num()) - } // handling the out-of-range part if (c.OutOfRange(lowVal) && !lowVal.IsNull()) || c.OutOfRange(highVal) { - cnt += c.Histogram.OutOfRangeRowCount(sctx, &lowVal, &highVal, modifyCount, histNDV) + histNDV := c.NDV + // Exclude the TopN + if c.StatsVer == statistics.Version2 { + histNDV -= int64(c.TopN.Num()) + } + cnt += c.Histogram.OutOfRangeRowCount(sctx, &lowVal, &highVal, modifyCount, histNDV, increaseFactor) } if debugTrace { @@ -315,8 +317,8 @@ func betweenRowCountOnColumn(sctx context.PlanContext, c *statistics.Column, l, // ColumnGreaterRowCount estimates the row count where the column greater than value. func ColumnGreaterRowCount(sctx context.PlanContext, t *statistics.Table, value types.Datum, colID int64) float64 { - c, ok := t.Columns[colID] - if !ok || c.IsInvalid(sctx, t.Pseudo) { + c := t.Columns[colID] + if statistics.ColumnStatsIsInvalid(c, sctx, &t.HistColl, colID) { return float64(t.RealtimeCount) / pseudoLessRate } return c.GreaterRowCount(value) * c.GetIncreaseFactor(t.RealtimeCount) @@ -324,8 +326,8 @@ func ColumnGreaterRowCount(sctx context.PlanContext, t *statistics.Table, value // columnLessRowCount estimates the row count where the column less than value. Note that null values are not counted. func columnLessRowCount(sctx context.PlanContext, t *statistics.Table, value types.Datum, colID int64) float64 { - c, ok := t.Columns[colID] - if !ok || c.IsInvalid(sctx, t.Pseudo) { + c := t.Columns[colID] + if statistics.ColumnStatsIsInvalid(c, sctx, &t.HistColl, colID) { return float64(t.RealtimeCount) / pseudoLessRate } return c.LessRowCount(sctx, value) * c.GetIncreaseFactor(t.RealtimeCount) @@ -334,8 +336,8 @@ func columnLessRowCount(sctx context.PlanContext, t *statistics.Table, value typ // columnBetweenRowCount estimates the row count where column greater or equal to a and less than b. func columnBetweenRowCount(sctx context.PlanContext, t *statistics.Table, a, b types.Datum, colID int64) (float64, error) { sc := sctx.GetSessionVars().StmtCtx - c, ok := t.Columns[colID] - if !ok || c.IsInvalid(sctx, t.Pseudo) { + c := t.Columns[colID] + if statistics.ColumnStatsIsInvalid(c, sctx, &t.HistColl, colID) { return float64(t.RealtimeCount) / pseudoBetweenRate, nil } aEncoded, err := codec.EncodeKey(sc.TimeZone(), nil, a) @@ -357,8 +359,8 @@ func columnBetweenRowCount(sctx context.PlanContext, t *statistics.Table, a, b t // ColumnEqualRowCount estimates the row count where the column equals to value. func ColumnEqualRowCount(sctx context.PlanContext, t *statistics.Table, value types.Datum, colID int64) (float64, error) { - c, ok := t.Columns[colID] - if !ok || c.IsInvalid(sctx, t.Pseudo) { + c := t.Columns[colID] + if statistics.ColumnStatsIsInvalid(c, sctx, &t.HistColl, colID) { return float64(t.RealtimeCount) / pseudoEqualRate, nil } encodedVal, err := codec.EncodeKey(sctx.GetSessionVars().StmtCtx.TimeZone(), nil, value) diff --git a/pkg/planner/cardinality/row_count_index.go b/pkg/planner/cardinality/row_count_index.go index 738fa5580fe92..ab70ca6178f20 100644 --- a/pkg/planner/cardinality/row_count_index.go +++ b/pkg/planner/cardinality/row_count_index.go @@ -59,7 +59,7 @@ func GetRowCountByIndexRanges(sctx context.PlanContext, coll *statistics.HistCol } } recordUsedItemStatsStatus(sctx, idx, coll.PhysicalID, idxID) - if !ok || idx.IsInvalid(sctx, coll.Pseudo) { + if statistics.IndexStatsIsInvalid(sctx, idx, coll, idxID) { colsLen := -1 if idx != nil && idx.Info.Unique { colsLen = len(idx.Info.Columns) @@ -222,7 +222,7 @@ func getIndexRowCountForStatsV2(sctx context.PlanContext, idx *statistics.Index, defer debugtrace.LeaveContextCommon(sctx) } totalCount := float64(0) - isSingleCol := len(idx.Info.Columns) == 1 + isSingleColIdx := len(idx.Info.Columns) == 1 for _, indexRange := range indexRanges { var count float64 lb, err := codec.EncodeKey(sc.TimeZone(), nil, indexRange.LowVal...) @@ -278,7 +278,7 @@ func getIndexRowCountForStatsV2(sctx context.PlanContext, idx *statistics.Index, l := types.NewBytesDatum(lb) r := types.NewBytesDatum(rb) lowIsNull := bytes.Equal(lb, nullKeyBytes) - if isSingleCol && lowIsNull { + if isSingleColIdx && lowIsNull { count += float64(idx.Histogram.NullCount) } expBackoffSuccess := false @@ -325,15 +325,24 @@ func getIndexRowCountForStatsV2(sctx context.PlanContext, idx *statistics.Index, } // If the current table row count has changed, we should scale the row count accordingly. - count *= idx.GetIncreaseFactor(realtimeRowCount) + increaseFactor := idx.GetIncreaseFactor(realtimeRowCount) + count *= increaseFactor - histNDV := idx.NDV - if idx.StatsVer == statistics.Version2 { - histNDV = histNDV - int64(idx.TopN.Num()) - } // handling the out-of-range part - if (outOfRangeOnIndex(idx, l) && !(isSingleCol && lowIsNull)) || outOfRangeOnIndex(idx, r) { - count += idx.Histogram.OutOfRangeRowCount(sctx, &l, &r, modifyCount, histNDV) + if (outOfRangeOnIndex(idx, l) && !(isSingleColIdx && lowIsNull)) || outOfRangeOnIndex(idx, r) { + histNDV := idx.NDV + // Exclude the TopN in Stats Version 2 + if idx.StatsVer == statistics.Version2 { + c, ok := coll.Columns[idx.Histogram.ID] + // If this is single column of a multi-column index - use the column's NDV rather than index NDV + isSingleColRange := len(indexRange.LowVal) == len(indexRange.HighVal) && len(indexRange.LowVal) == 1 + if isSingleColRange && !isSingleColIdx && ok && c != nil && c.Histogram.NDV > 0 { + histNDV = c.Histogram.NDV - int64(c.TopN.Num()) + } else { + histNDV -= int64(idx.TopN.Num()) + } + } + count += idx.Histogram.OutOfRangeRowCount(sctx, &l, &r, modifyCount, histNDV, increaseFactor) } if debugTrace { @@ -435,7 +444,7 @@ func expBackoffEstimation(sctx context.PlanContext, idx *statistics.Index, coll err error foundStats bool ) - if col, ok := coll.Columns[colID]; ok && !col.IsInvalid(sctx, coll.Pseudo) { + if !statistics.ColumnStatsIsInvalid(coll.Columns[colID], sctx, coll, colID) { foundStats = true count, err = GetRowCountByColumnRanges(sctx, coll, colID, tmpRan) selectivity = count / float64(coll.RealtimeCount) @@ -449,7 +458,7 @@ func expBackoffEstimation(sctx context.PlanContext, idx *statistics.Index, coll continue } idxStats, ok := coll.Indices[idxID] - if !ok || idxStats.IsInvalid(sctx, coll.Pseudo) { + if !ok || statistics.IndexStatsIsInvalid(sctx, idxStats, coll, idxID) { continue } foundStats = true diff --git a/pkg/planner/cardinality/row_count_test.go b/pkg/planner/cardinality/row_count_test.go index de3ed97815b28..a1a6c399bc28f 100644 --- a/pkg/planner/cardinality/row_count_test.go +++ b/pkg/planner/cardinality/row_count_test.go @@ -33,8 +33,8 @@ func TestPseudoTable(t *testing.T) { State: model.StatePublic, } ti.Columns = append(ti.Columns, colInfo) - tbl := statistics.PseudoTable(ti, false) - require.Len(t, tbl.Columns, 1) + tbl := statistics.PseudoTable(ti, false, false) + require.Len(t, tbl.Columns, 0) require.Greater(t, tbl.RealtimeCount, int64(0)) sctx := mock.NewContext() count := columnLessRowCount(sctx, tbl, types.NewIntDatum(100), colInfo.ID) @@ -50,7 +50,7 @@ func TestPseudoTable(t *testing.T) { Hidden: true, State: model.StatePublic, }) - tbl = statistics.PseudoTable(ti, false) - // We added a hidden column. The pseudo table still only have one column. - require.Equal(t, len(tbl.Columns), 1) + tbl = statistics.PseudoTable(ti, false, false) + // We added a hidden column. The pseudo table still only have zero column. + require.Equal(t, len(tbl.Columns), 0) } diff --git a/pkg/planner/cardinality/selectivity.go b/pkg/planner/cardinality/selectivity.go index 65860a2faf49b..c4c65788facb2 100644 --- a/pkg/planner/cardinality/selectivity.go +++ b/pkg/planner/cardinality/selectivity.go @@ -78,7 +78,7 @@ func Selectivity( // TODO: If len(exprs) is bigger than 63, we could use bitset structure to replace the int64. // This will simplify some code and speed up if we use this rather than a boolean slice. if len(exprs) > 63 || (len(coll.Columns) == 0 && len(coll.Indices) == 0) { - ret = pseudoSelectivity(coll, exprs) + ret = pseudoSelectivity(ctx, coll, exprs) if sc.EnableOptimizerCETrace { ceTraceExpr(ctx, tableID, "Table Stats-Pseudo-Expression", expression.ComposeCNFCondition(ctx.GetExprCtx(), exprs...), ret*float64(coll.RealtimeCount)) @@ -104,7 +104,7 @@ func Selectivity( colHist := coll.Columns[c.UniqueID] var sel float64 - if colHist == nil || colHist.IsInvalid(ctx, coll.Pseudo) { + if statistics.ColumnStatsIsInvalid(colHist, ctx, coll, c.ID) { sel = 1.0 / pseudoEqualRate } else if colHist.Histogram.NDV > 0 { sel = 1 / float64(colHist.Histogram.NDV) @@ -119,12 +119,16 @@ func Selectivity( extractedCols := make([]*expression.Column, 0, len(coll.Columns)) extractedCols = expression.ExtractColumnsFromExpressions(extractedCols, remainedExprs, nil) - colIDs := maps.Keys(coll.Columns) - slices.Sort(colIDs) - for _, id := range colIDs { - colStats := coll.Columns[id] - col := expression.ColInfo2Col(extractedCols, colStats.Info) - if col != nil { + slices.SortFunc(extractedCols, func(a *expression.Column, b *expression.Column) int { + return cmp.Compare(a.ID, b.ID) + }) + extractedCols = slices.CompactFunc(extractedCols, func(a, b *expression.Column) bool { + return a.ID == b.ID + }) + for _, col := range extractedCols { + id := col.UniqueID + colStats := coll.Columns[col.UniqueID] + if colStats != nil { maskCovered, ranges, _, err := getMaskAndRanges(ctx, remainedExprs, ranger.ColumnRangeType, nil, nil, col) if err != nil { return 0, nil, errors.Trace(err) @@ -145,6 +149,10 @@ func Selectivity( return 0, nil, errors.Trace(err) } nodes[len(nodes)-1].Selectivity = cnt / float64(coll.RealtimeCount) + } else if !col.IsHidden { + // TODO: We are able to remove this path if we remove the async stats load. + statistics.ColumnStatsIsInvalid(nil, ctx, coll, col.ID) + recordUsedItemStatsStatus(ctx, (*statistics.Column)(nil), tableID, col.ID) } } id2Paths := make(map[int64]*planutil.AccessPath) @@ -736,7 +744,7 @@ func getMaskAndSelectivityForMVIndex( } // You can find more examples and explanations in comments for collectFilters4MVIndex() and // buildPartialPaths4MVIndex() in planner/core. - accessConds, _ := CollectFilters4MVIndex(ctx, exprs, cols) + accessConds, _, _ := CollectFilters4MVIndex(ctx, exprs, cols) paths, isIntersection, ok, err := BuildPartialPaths4MVIndex(ctx, accessConds, cols, coll.Indices[id].Info, coll) if err != nil || !ok { return 1.0, 0, false @@ -907,16 +915,15 @@ func GetSelectivityByFilter(sctx context.PlanContext, coll *statistics.HistColl, func findAvailableStatsForCol(sctx context.PlanContext, coll *statistics.HistColl, uniqueID int64) (isIndex bool, idx int64) { // try to find available stats in column stats - if colStats, ok := coll.Columns[uniqueID]; ok && colStats != nil && !colStats.IsInvalid(sctx, coll.Pseudo) && colStats.IsFullLoad() { + if colStats := coll.Columns[uniqueID]; !statistics.ColumnStatsIsInvalid(colStats, sctx, coll, uniqueID) && colStats.IsFullLoad() { return false, uniqueID } // try to find available stats in single column index stats (except for prefix index) for idxStatsIdx, cols := range coll.Idx2ColumnIDs { if len(cols) == 1 && cols[0] == uniqueID { - idxStats, ok := coll.Indices[idxStatsIdx] - if ok && + idxStats := coll.Indices[idxStatsIdx] + if !statistics.IndexStatsIsInvalid(sctx, idxStats, coll, idxStatsIdx) && idxStats.Info.Columns[0].Length == types.UnspecifiedLength && - !idxStats.IsInvalid(sctx, coll.Pseudo) && idxStats.IsFullLoad() { return true, idxStatsIdx } @@ -1050,29 +1057,28 @@ func crossValidationSelectivity( if i >= usedColsLen { break } - if col, ok := coll.Columns[colID]; ok { - if col.IsInvalid(sctx, coll.Pseudo) { - continue - } - // Since the column range is point range(LowVal is equal to HighVal), we need to set both LowExclude and HighExclude to false. - // Otherwise we would get 0.0 estRow from GetColumnRowCount. - rang := ranger.Range{ - LowVal: []types.Datum{idxPointRange.LowVal[i]}, - LowExclude: false, - HighVal: []types.Datum{idxPointRange.HighVal[i]}, - HighExclude: false, - Collators: []collate.Collator{idxPointRange.Collators[i]}, - } + col := coll.Columns[colID] + if statistics.ColumnStatsIsInvalid(col, sctx, coll, colID) { + continue + } + // Since the column range is point range(LowVal is equal to HighVal), we need to set both LowExclude and HighExclude to false. + // Otherwise we would get 0.0 estRow from GetColumnRowCount. + rang := ranger.Range{ + LowVal: []types.Datum{idxPointRange.LowVal[i]}, + LowExclude: false, + HighVal: []types.Datum{idxPointRange.HighVal[i]}, + HighExclude: false, + Collators: []collate.Collator{idxPointRange.Collators[i]}, + } - rowCount, err := GetColumnRowCount(sctx, col, []*ranger.Range{&rang}, coll.RealtimeCount, coll.ModifyCount, col.IsHandle) - if err != nil { - return 0, 0, err - } - crossValidationSelectivity = crossValidationSelectivity * (rowCount / totalRowCount) + rowCount, err := GetColumnRowCount(sctx, col, []*ranger.Range{&rang}, coll.RealtimeCount, coll.ModifyCount, col.IsHandle) + if err != nil { + return 0, 0, err + } + crossValidationSelectivity = crossValidationSelectivity * (rowCount / totalRowCount) - if rowCount < minRowCount { - minRowCount = rowCount - } + if rowCount < minRowCount { + minRowCount = rowCount } } return minRowCount, crossValidationSelectivity, nil @@ -1089,6 +1095,7 @@ var ( ) ( accessFilters, remainingFilters []expression.Expression, + accessTp int, ) BuildPartialPaths4MVIndex func( sctx context.PlanContext, diff --git a/pkg/planner/cardinality/selectivity_test.go b/pkg/planner/cardinality/selectivity_test.go index 21d6a31140dad..c41a956abfe83 100644 --- a/pkg/planner/cardinality/selectivity_test.go +++ b/pkg/planner/cardinality/selectivity_test.go @@ -973,6 +973,19 @@ func TestIndexJoinInnerRowCountUpperBound(t *testing.T) { StatsVer: 2, } } + idxValues := make([]types.Datum, 0, len(colValues)) + sc := stmtctx.NewStmtCtxWithTimeZone(time.UTC) + for _, colV := range colValues { + b, err := codec.EncodeKey(sc.TimeZone(), nil, colV) + require.NoError(t, err) + idxValues = append(idxValues, types.NewBytesDatum(b)) + } + mockStatsTbl.Indices[1] = &statistics.Index{ + Histogram: *mockStatsHistogram(1, idxValues, 1000, types.NewFieldType(mysql.TypeBlob)), + Info: tblInfo.Indices[0], + StatsLoadedStatus: statistics.NewStatsFullLoadStatus(), + StatsVer: 2, + } generateMapsForMockStatsTbl(mockStatsTbl) stat := h.GetTableStats(tblInfo) stat.HistColl = mockStatsTbl.HistColl diff --git a/pkg/planner/cardinality/testdata/cardinality_suite_out.json b/pkg/planner/cardinality/testdata/cardinality_suite_out.json index dfd9563c71c83..86b8d2a56aa50 100644 --- a/pkg/planner/cardinality/testdata/cardinality_suite_out.json +++ b/pkg/planner/cardinality/testdata/cardinality_suite_out.json @@ -24,7 +24,7 @@ { "Start": 800, "End": 900, - "Count": 723.504166655054 + "Count": 735.504166655054 }, { "Start": 900, @@ -79,7 +79,7 @@ { "Start": 800, "End": 1000, - "Count": 1181.696869573942 + "Count": 1193.696869573942 }, { "Start": 900, @@ -104,7 +104,7 @@ { "Start": 200, "End": 400, - "Count": 1190.2788209899081 + "Count": 1237.5288209899081 }, { "Start": 200, @@ -1521,7 +1521,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -1590,7 +1590,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -1679,7 +1679,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 3080 @@ -1705,7 +1705,7 @@ { "github.com/pingcap/tidb/pkg/planner/cardinality.crossValidationSelectivity": [ { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -1755,7 +1755,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -1892,7 +1892,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 3080 @@ -1918,7 +1918,7 @@ { "github.com/pingcap/tidb/pkg/planner/cardinality.crossValidationSelectivity": [ { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -2078,7 +2078,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -2147,7 +2147,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -2242,7 +2242,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 2980 @@ -2333,7 +2333,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 2980 @@ -2450,7 +2450,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -2585,7 +2585,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -2723,7 +2723,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 3080 @@ -2874,7 +2874,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 3080 @@ -2900,7 +2900,7 @@ { "github.com/pingcap/tidb/pkg/planner/cardinality.crossValidationSelectivity": [ { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -3146,7 +3146,7 @@ { "github.com/pingcap/tidb/pkg/planner/cardinality.crossValidationSelectivity": [ { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -3441,7 +3441,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -3581,7 +3581,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": true, "InValidForCollPseudo": false, "IsInvalid": false, @@ -3730,7 +3730,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 2980 @@ -3859,7 +3859,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": false, "IsInvalid": false, "TotalCount": 2980 diff --git a/pkg/planner/cardinality/trace.go b/pkg/planner/cardinality/trace.go index 07811f3c5acf3..ac5251c8e6425 100644 --- a/pkg/planner/cardinality/trace.go +++ b/pkg/planner/cardinality/trace.go @@ -169,7 +169,7 @@ func recordUsedItemStatsStatus(sctx context.PlanContext, stats any, tableID, id } // no need to record - if !missing && loadStatus.IsFullLoad() { + if !missing && loadStatus != nil && loadStatus.IsFullLoad() { return } @@ -198,7 +198,14 @@ func recordUsedItemStatsStatus(sctx context.PlanContext, stats any, tableID, id } if missing { - recordForColOrIdx[id] = "missing" + // Figure out whether it's really not existing. + if recordForTbl.ColAndIdxStatus != nil && recordForTbl.ColAndIdxStatus.(*statistics.ColAndIdxExistenceMap).HasAnalyzed(id, isIndex) { + // If this item has been analyzed but there's no its stats, we should mark it as uninitialized. + recordForColOrIdx[id] = statistics.StatsLoadedStatus{}.StatusToString() + } else { + // Otherwise, we mark it as missing. + recordForColOrIdx[id] = "missing" + } return } recordForColOrIdx[id] = loadStatus.StatusToString() diff --git a/pkg/planner/cardinality/trace_test.go b/pkg/planner/cardinality/trace_test.go index ef64ded5e8fa0..9fa9913cb11aa 100644 --- a/pkg/planner/cardinality/trace_test.go +++ b/pkg/planner/cardinality/trace_test.go @@ -233,7 +233,7 @@ func TestTraceDebugSelectivity(t *testing.T) { testdata.OnRecord(func() { out[i].ResultForV2 = res }) - require.Equal(t, out[i].ResultForV2, res, sql, "For ver2") + require.Equal(t, out[i].ResultForV2, res, sql, fmt.Sprintf("For ver2: test#%d", i)) } tk.MustExec("set tidb_analyze_version = 1") diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index facbe600ff4c5..e366f4b03360d 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "hashcode.go", "hint_utils.go", "indexmerge_path.go", + "indexmerge_unfinished_path.go", "initialize.go", "logical_plan_builder.go", "logical_plans.go", diff --git a/pkg/planner/core/casetest/binaryplan/binary_plan_test.go b/pkg/planner/core/casetest/binaryplan/binary_plan_test.go index 6d8ec897a8f22..4315cff0130eb 100644 --- a/pkg/planner/core/casetest/binaryplan/binary_plan_test.go +++ b/pkg/planner/core/casetest/binaryplan/binary_plan_test.go @@ -136,6 +136,6 @@ func TestBinaryPlanInExplainAndSlowLog(t *testing.T) { output[i].BinaryPlan = binary }) simplifyAndCheckBinaryPlan(t, binary) - require.Equal(t, output[i].BinaryPlan, binary) + require.Equal(t, output[i].BinaryPlan, binary, comment) } } diff --git a/pkg/planner/core/casetest/partition/BUILD.bazel b/pkg/planner/core/casetest/partition/BUILD.bazel index bb7ecd6f0beda..0472ac87bda43 100644 --- a/pkg/planner/core/casetest/partition/BUILD.bazel +++ b/pkg/planner/core/casetest/partition/BUILD.bazel @@ -10,7 +10,7 @@ go_test( ], data = glob(["testdata/**"]), flaky = True, - shard_count = 6, + shard_count = 7, deps = [ "//pkg/config", "//pkg/planner/core/internal", diff --git a/pkg/planner/core/casetest/partition/partition_pruner_test.go b/pkg/planner/core/casetest/partition/partition_pruner_test.go index 8aa63eb3bff2e..aafdeebe11fb8 100644 --- a/pkg/planner/core/casetest/partition/partition_pruner_test.go +++ b/pkg/planner/core/casetest/partition/partition_pruner_test.go @@ -212,3 +212,24 @@ func TestListColumnsPartitionPruner(t *testing.T) { } require.True(t, valid) } + +func TestPointGetIntHandleNotFirst(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table t ( + c int, + a int not null, + b int, + primary key (a) /*T![clustered_index] clustered */ + )`) + tk.MustExec(`insert into t values(1, 13, 1)`) + tk.MustQuery("select * from t WHERE `a` BETWEEN 13 AND 13").Check(testkit.Rows("1 13 1")) + tk.MustExec(`alter table t + partition by range (a) + (partition p0 values less than (10), + partition p1 values less than (maxvalue))`) + + tk.MustQuery("select * from t WHERE a BETWEEN 13 AND 13").Check(testkit.Rows("1 13 1")) + tk.MustQuery(`select * from t`).Check(testkit.Rows("1 13 1")) +} diff --git a/pkg/planner/core/casetest/planstats/plan_stats_test.go b/pkg/planner/core/casetest/planstats/plan_stats_test.go index 935d380caccc0..5337b1460b224 100644 --- a/pkg/planner/core/casetest/planstats/plan_stats_test.go +++ b/pkg/planner/core/casetest/planstats/plan_stats_test.go @@ -265,13 +265,13 @@ func TestPlanStatsLoadTimeout(t *testing.T) { tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) tableInfo := tbl.Meta() - neededColumn := model.TableItemID{TableID: tableInfo.ID, ID: tableInfo.Columns[0].ID, IsIndex: false} + neededColumn := model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tableInfo.ID, ID: tableInfo.Columns[0].ID, IsIndex: false}, FullLoad: true} resultCh := make(chan stmtctx.StatsLoadResult, 1) timeout := time.Duration(1<<63 - 1) task := &types.NeededItemTask{ - TableItemID: neededColumn, - ResultCh: resultCh, - ToTimeout: time.Now().Local().Add(timeout), + Item: neededColumn, + ResultCh: resultCh, + ToTimeout: time.Now().Local().Add(timeout), } dom.StatsHandle().AppendNeededItem(task, timeout) // make channel queue full sql := "select /*+ MAX_EXECUTION_TIME(1000) */ * from t where c>1" @@ -379,11 +379,11 @@ func TestCollectDependingVirtualCols(t *testing.T) { // prepare the input tbl := tblID2Tbl[tblName2TblID[testCase.TableName]] require.NotNil(t, tbl) - neededItems := make([]model.TableItemID, 0, len(testCase.InputColNames)) + neededItems := make([]model.StatsLoadItem, 0, len(testCase.InputColNames)) for _, colName := range testCase.InputColNames { col := tbl.Meta().FindPublicColumnByName(colName) require.NotNil(t, col) - neededItems = append(neededItems, model.TableItemID{TableID: tbl.Meta().ID, ID: col.ID}) + neededItems = append(neededItems, model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tbl.Meta().ID, ID: col.ID}, FullLoad: true}) } // call the function diff --git a/pkg/planner/core/collect_column_stats_usage.go b/pkg/planner/core/collect_column_stats_usage.go index f9c14fc6621a6..684ccc86deb4a 100644 --- a/pkg/planner/core/collect_column_stats_usage.go +++ b/pkg/planner/core/collect_column_stats_usage.go @@ -15,8 +15,13 @@ package core import ( + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/util/intset" "golang.org/x/exp/maps" ) @@ -41,8 +46,8 @@ type columnStatsUsageCollector struct { // we don't know `ndv(t.a, t.b)`(see (*LogicalAggregation).DeriveStats and getColsNDV for details). So when calculating the statistics // of column `e`, we may use the statistics of column `t.a` and `t.b`. colMap map[int64]map[model.TableItemID]struct{} - // histNeededCols records histogram-needed columns - histNeededCols map[model.TableItemID]struct{} + // histNeededCols records histogram-needed columns. The value field of the map indicates that whether we need to load the full stats of the time or not. + histNeededCols map[model.TableItemID]bool // cols is used to store columns collected from expressions and saves some allocation. cols []*expression.Column @@ -69,7 +74,7 @@ func newColumnStatsUsageCollector(collectMode uint64, enabledPlanCapture bool) * collector.colMap = make(map[int64]map[model.TableItemID]struct{}) } if collectMode&collectHistNeededColumns != 0 { - collector.histNeededCols = make(map[model.TableItemID]struct{}) + collector.histNeededCols = make(map[model.TableItemID]bool) } if enabledPlanCapture { collector.collectVisitedTable = true @@ -175,10 +180,33 @@ func (c *columnStatsUsageCollector) addHistNeededColumns(ds *DataSource) { tblID := ds.TableInfo().ID c.visitedtbls[tblID] = struct{}{} } + stats := domain.GetDomain(ds.SCtx()).StatsHandle() + tblStats := stats.GetPartitionStats(ds.tableInfo, ds.physicalTableID) + skipPseudoCheckForTest := false + failpoint.Inject("disablePseudoCheck", func() { + skipPseudoCheckForTest = true + }) + // Since we can not get the stats tbl, this table is not analyzed. So we don't need to consider load stats. + if tblStats.Pseudo && !skipPseudoCheckForTest { + return + } columns := expression.ExtractColumnsFromExpressions(c.cols[:0], ds.pushedDownConds, nil) + + colIDSet := intset.NewFastIntSet() + for _, col := range columns { tblColID := model.TableItemID{TableID: ds.physicalTableID, ID: col.ID, IsIndex: false} - c.histNeededCols[tblColID] = struct{}{} + colIDSet.Insert(int(col.ID)) + c.histNeededCols[tblColID] = true + } + for _, col := range ds.Columns { + if !colIDSet.Has(int(col.ID)) && !col.Hidden { + tblColID := model.TableItemID{TableID: ds.physicalTableID, ID: col.ID, IsIndex: false} + if _, ok := c.histNeededCols[tblColID]; ok { + continue + } + c.histNeededCols[tblColID] = false + } } } @@ -308,7 +336,7 @@ func (c *columnStatsUsageCollector) collectFromPlan(lp LogicalPlan) { // Third return value: ds.physicalTableID from all DataSource (always collected) func CollectColumnStatsUsage(lp LogicalPlan, predicate, histNeeded bool) ( []model.TableItemID, - []model.TableItemID, + []model.StatsLoadItem, *intset.FastIntSet, ) { var mode uint64 @@ -323,12 +351,93 @@ func CollectColumnStatsUsage(lp LogicalPlan, predicate, histNeeded bool) ( if collector.collectVisitedTable { recordTableRuntimeStats(lp.SCtx(), collector.visitedtbls) } - var predicateCols, histNeededCols []model.TableItemID + itemSet2slice := func(set map[model.TableItemID]bool) []model.StatsLoadItem { + ret := make([]model.StatsLoadItem, 0, len(set)) + for item, fullLoad := range set { + ret = append(ret, model.StatsLoadItem{TableItemID: item, FullLoad: fullLoad}) + } + return ret + } + is := lp.SCtx().GetInfoSchema().(infoschema.InfoSchema) + statsHandle := domain.GetDomain(lp.SCtx()).StatsHandle() + physTblIDsWithNeededCols := intset.NewFastIntSet() + for neededCol, fullLoad := range collector.histNeededCols { + if !fullLoad { + continue + } + physTblIDsWithNeededCols.Insert(int(neededCol.TableID)) + } + collector.visitedPhysTblIDs.ForEach(func(physicalTblID int) { + // 1. collect table metadata + tbl, _ := infoschema.FindTableByTblOrPartID(is, int64(physicalTblID)) + if tbl == nil { + return + } + + // 2. handle extra sync/async stats loading for the determinate mode + + // If we visited a table without getting any columns need stats (likely because there are no pushed down + // predicates), and we are in the determinate mode, we need to make sure we are able to get the "analyze row + // count" in getStatsTable(), which means any column/index stats are available. + if lp.SCtx().GetSessionVars().GetOptObjective() != variable.OptObjectiveDeterminate || + // If we already collected some columns that need trigger sync laoding on this table, we don't need to + // additionally do anything for determinate mode. + physTblIDsWithNeededCols.Has(physicalTblID) || + statsHandle == nil { + return + } + tblStats := statsHandle.GetTableStats(tbl.Meta()) + if tblStats == nil || tblStats.Pseudo { + return + } + var colToTriggerLoad *model.TableItemID + for _, col := range tbl.Cols() { + if col.State != model.StatePublic || (col.IsGenerated() && !col.GeneratedStored) || !tblStats.ColAndIdxExistenceMap.HasAnalyzed(col.ID, false) { + continue + } + if colStats := tblStats.Columns[col.ID]; colStats != nil { + // If any stats are already full loaded, we don't need to trigger stats loading on this table. + if colStats.IsFullLoad() { + colToTriggerLoad = nil + break + } + } + // Choose the first column we meet to trigger stats loading. + if colToTriggerLoad == nil { + colToTriggerLoad = &model.TableItemID{TableID: int64(physicalTblID), ID: col.ID, IsIndex: false} + } + } + if colToTriggerLoad == nil { + return + } + for _, idx := range tbl.Indices() { + if idx.Meta().State != model.StatePublic || idx.Meta().MVIndex { + continue + } + // If any stats are already full loaded, we don't need to trigger stats loading on this table. + if idxStats := tblStats.Indices[idx.Meta().ID]; idxStats != nil && idxStats.IsFullLoad() { + colToTriggerLoad = nil + break + } + } + if colToTriggerLoad == nil { + return + } + if histNeeded { + collector.histNeededCols[*colToTriggerLoad] = true + } else { + statistics.HistogramNeededItems.Insert(*colToTriggerLoad) + } + }) + var ( + predicateCols []model.TableItemID + histNeededCols []model.StatsLoadItem + ) if predicate { predicateCols = maps.Keys(collector.predicateCols) } if histNeeded { - histNeededCols = maps.Keys(collector.histNeededCols) + histNeededCols = itemSet2slice(collector.histNeededCols) } return predicateCols, histNeededCols, collector.visitedPhysTblIDs } diff --git a/pkg/planner/core/collect_column_stats_usage_test.go b/pkg/planner/core/collect_column_stats_usage_test.go index 198270eda0b15..4009287f4538b 100644 --- a/pkg/planner/core/collect_column_stats_usage_test.go +++ b/pkg/planner/core/collect_column_stats_usage_test.go @@ -66,13 +66,19 @@ func getColumnName(t *testing.T, is infoschema.InfoSchema, tblColID model.TableI return colName } -func checkColumnStatsUsage(t *testing.T, is infoschema.InfoSchema, lp LogicalPlan, histNeededOnly bool, expected []string, comment string) { - var tblColIDs []model.TableItemID - if histNeededOnly { - _, tblColIDs, _ = CollectColumnStatsUsage(lp, false, true) +func getStatsLoadItem(t *testing.T, is infoschema.InfoSchema, item model.StatsLoadItem, comment string) string { + str := getColumnName(t, is, item.TableItemID, comment) + if item.FullLoad { + str += " full" } else { - tblColIDs, _, _ = CollectColumnStatsUsage(lp, true, false) + str += " meta" } + return str +} + +func checkColumnStatsUsageForPredicates(t *testing.T, is infoschema.InfoSchema, lp LogicalPlan, expected []string, comment string) { + var tblColIDs []model.TableItemID + tblColIDs, _, _ = CollectColumnStatsUsage(lp, true, false) cols := make([]string, 0, len(tblColIDs)) for _, tblColID := range tblColIDs { col := getColumnName(t, is, tblColID, comment) @@ -82,6 +88,18 @@ func checkColumnStatsUsage(t *testing.T, is infoschema.InfoSchema, lp LogicalPla require.Equal(t, expected, cols, comment) } +func checkColumnStatsUsageForStatsLoad(t *testing.T, is infoschema.InfoSchema, lp LogicalPlan, expected []string, comment string) { + var loadItems []model.StatsLoadItem + _, loadItems, _ = CollectColumnStatsUsage(lp, false, true) + cols := make([]string, 0, len(loadItems)) + for _, item := range loadItems { + col := getStatsLoadItem(t, is, item, comment) + cols = append(cols, col) + } + sort.Strings(cols) + require.Equal(t, expected, cols, comment+", we get %v", cols) +} + func TestCollectPredicateColumns(t *testing.T) { tests := []struct { pruneMode string @@ -266,16 +284,18 @@ func TestCollectPredicateColumns(t *testing.T) { require.True(t, ok, comment) // We check predicate columns twice, before and after logical optimization. Some logical plan patterns may occur before // logical optimization while others may occur after logical optimization. - checkColumnStatsUsage(t, s.is, lp, false, tt.res, comment) + checkColumnStatsUsageForPredicates(t, s.is, lp, tt.res, comment) lp, err = logicalOptimize(ctx, builder.GetOptFlag(), lp) require.NoError(t, err, comment) - checkColumnStatsUsage(t, s.is, lp, false, tt.res, comment) + checkColumnStatsUsageForPredicates(t, s.is, lp, tt.res, comment) } } func TestCollectHistNeededColumns(t *testing.T) { failpoint.Enable("github.com/pingcap/tidb/pkg/planner/core/forceDynamicPrune", `return(true)`) defer failpoint.Disable("github.com/pingcap/tidb/pkg/planner/core/forceDynamicPrune") + failpoint.Enable("github.com/pingcap/tidb/pkg/planner/core/disablePseudoCheck", `return(true)`) + defer failpoint.Disable("github.com/pingcap/tidb/pkg/planner/core/disablePseudoCheck") tests := []struct { pruneMode string sql string @@ -283,45 +303,45 @@ func TestCollectHistNeededColumns(t *testing.T) { }{ { sql: "select * from t where a > 2", - res: []string{"t.a"}, + res: []string{"t.a full", "t.b meta", "t.c meta", "t.c_str meta", "t.d meta", "t.d_str meta", "t.e meta", "t.e_str meta", "t.f meta", "t.g meta", "t.h meta", "t.i_date meta"}, }, { sql: "select * from t where b in (2, 5) or c = 5", - res: []string{"t.b", "t.c"}, + res: []string{"t.a meta", "t.b full", "t.c full", "t.c_str meta", "t.d meta", "t.d_str meta", "t.e meta", "t.e_str meta", "t.f meta", "t.g meta", "t.h meta", "t.i_date meta"}, }, { sql: "select * from t where a + b > 1", - res: []string{"t.a", "t.b"}, + res: []string{"t.a full", "t.b full", "t.c meta", "t.c_str meta", "t.d meta", "t.d_str meta", "t.e meta", "t.e_str meta", "t.f meta", "t.g meta", "t.h meta", "t.i_date meta"}, }, { sql: "select b, count(a) from t where b > 1 group by b having count(a) > 2", - res: []string{"t.b"}, + res: []string{"t.a meta", "t.b full"}, }, { sql: "select * from t as x join t2 as y on x.b + y.b > 2 and x.c > 1 and y.a < 1", - res: []string{"t.c", "t2.a"}, + res: []string{"t.a meta", "t.b meta", "t.c full", "t.c_str meta", "t.d meta", "t.d_str meta", "t.e meta", "t.e_str meta", "t.f meta", "t.g meta", "t.h meta", "t.i_date meta", "t2.a full", "t2.b meta", "t2.c meta"}, }, { sql: "select * from t2 where t2.b > all(select b from t where t.c > 2)", - res: []string{"t.c"}, + res: []string{"t.b meta", "t.c full", "t2.a meta", "t2.b meta", "t2.c meta"}, }, { sql: "select * from t2 where t2.b > any(select b from t where t.c > 2)", - res: []string{"t.c"}, + res: []string{"t.b meta", "t.c full", "t2.a meta", "t2.b meta", "t2.c meta"}, }, { sql: "select * from t2 where t2.b in (select b from t where t.c > 2)", - res: []string{"t.c"}, + res: []string{"t.b meta", "t.c full", "t2.a meta", "t2.b meta", "t2.c meta"}, }, { pruneMode: "static", sql: "select * from pt1 where ptn < 20 and b > 1", - res: []string{"pt1.p1.b", "pt1.p1.ptn", "pt1.p2.b", "pt1.p2.ptn"}, + res: []string{"pt1.p1.a meta", "pt1.p1.b full", "pt1.p1.c meta", "pt1.p1.c_str meta", "pt1.p1.d meta", "pt1.p1.d_str meta", "pt1.p1.e meta", "pt1.p1.e_str meta", "pt1.p1.f meta", "pt1.p1.g meta", "pt1.p1.h meta", "pt1.p1.i_date meta", "pt1.p1.ptn full", "pt1.p2.a meta", "pt1.p2.b full", "pt1.p2.c meta", "pt1.p2.c_str meta", "pt1.p2.d meta", "pt1.p2.d_str meta", "pt1.p2.e meta", "pt1.p2.e_str meta", "pt1.p2.f meta", "pt1.p2.g meta", "pt1.p2.h meta", "pt1.p2.i_date meta", "pt1.p2.ptn full"}, }, { pruneMode: "dynamic", sql: "select * from pt1 where ptn < 20 and b > 1", - res: []string{"pt1.b", "pt1.ptn"}, + res: []string{"pt1.a meta", "pt1.b full", "pt1.c meta", "pt1.c_str meta", "pt1.d meta", "pt1.d_str meta", "pt1.e meta", "pt1.e_str meta", "pt1.f meta", "pt1.g meta", "pt1.h meta", "pt1.i_date meta", "pt1.ptn full"}, }, } @@ -353,6 +373,6 @@ func TestCollectHistNeededColumns(t *testing.T) { flags &= ^(flagJoinReOrder | flagPrunColumnsAgain) lp, err = logicalOptimize(ctx, flags, lp) require.NoError(t, err, comment) - checkColumnStatsUsage(t, s.is, lp, true, tt.res, comment) + checkColumnStatsUsageForStatsLoad(t, s.is, lp, tt.res, comment) } } diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index de1c7862b1cb7..62f179491a871 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -2574,7 +2574,11 @@ func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candida pointGetPlan.Handle = kv.IntHandle(candidate.path.Ranges[0].LowVal[0].GetInt64()) pointGetPlan.UnsignedHandle = mysql.HasUnsignedFlag(ds.handleCols.GetCol(0).RetType.GetFlag()) pointGetPlan.accessCols = ds.TblCols - pointGetPlan.HandleColOffset = ds.handleCols.GetCol(0).Index + hc, err := ds.handleCols.ResolveIndices(ds.schema) + if err != nil { + return invalidTask + } + pointGetPlan.HandleColOffset = hc.GetCol(0).Index // Add filter condition to table plan now. if len(candidate.path.TableFilters) > 0 { sel := PhysicalSelection{ diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index 5cf6e5553daa0..ea17974e063be 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -43,7 +43,7 @@ import ( func init() { cardinality.CollectFilters4MVIndex = collectFilters4MVIndex cardinality.BuildPartialPaths4MVIndex = buildPartialPaths4MVIndex - statistics.PrepareCols4MVIndex = PrepareCols4MVIndex + statistics.PrepareCols4MVIndex = PrepareIdxColsAndUnwrapArrayType } // generateIndexMergePath generates IndexMerge AccessPaths on this DataSource. @@ -135,7 +135,10 @@ func (ds *DataSource) generateIndexMergePath() error { return nil } -func (ds *DataSource) generateNormalIndexPartialPaths4DNF(dnfItems []expression.Expression, usedIndexCount int) (paths []*util.AccessPath, needSelection bool, usedMap []bool, err error) { +func (ds *DataSource) generateNormalIndexPartialPaths4DNF( + dnfItems []expression.Expression, + candidatePaths []*util.AccessPath, +) (paths []*util.AccessPath, needSelection bool, usedMap []bool) { paths = make([]*util.AccessPath, 0, len(dnfItems)) usedMap = make([]bool, len(dnfItems)) for offset, item := range dnfItems { @@ -152,12 +155,12 @@ func (ds *DataSource) generateNormalIndexPartialPaths4DNF(dnfItems []expression. needSelection = true } } - itemPaths := ds.accessPathsForConds(pushedDownCNFItems, usedIndexCount) + itemPaths := ds.accessPathsForConds(pushedDownCNFItems, candidatePaths) if len(itemPaths) == 0 { // for this dnf item, we couldn't generate an index merge partial path. // (1 member of (a)) or (3 member of (b)) or d=1; if one dnf item like d=1 here could walk index path, // the entire index merge is not valid anymore. - return nil, false, usedMap, nil + return nil, false, usedMap } // prune out global indexes. itemPaths = slices.DeleteFunc(itemPaths, func(path *util.AccessPath) bool { @@ -171,7 +174,7 @@ func (ds *DataSource) generateNormalIndexPartialPaths4DNF(dnfItems []expression. // for this dnf item, we couldn't generate an index merge partial path. // (1 member of (a)) or (3 member of (b)) or d=1; if one dnf item like d=1 here could walk index path, // the entire index merge is not valid anymore. - return nil, false, usedMap, nil + return nil, false, usedMap } // identify whether all pushedDownCNFItems are fully used. @@ -193,7 +196,7 @@ func (ds *DataSource) generateNormalIndexPartialPaths4DNF(dnfItems []expression. usedMap[offset] = true paths = append(paths, partialPath) } - return paths, needSelection, usedMap, nil + return paths, needSelection, usedMap } // getIndexMergeOrPath generates all possible IndexMergeOrPaths. @@ -243,7 +246,7 @@ func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) } } - itemPaths := ds.accessPathsForConds(pushedDownCNFItems, usedIndexCount) + itemPaths := ds.accessPathsForConds(pushedDownCNFItems, ds.possibleAccessPaths[:usedIndexCount]) if len(itemPaths) == 0 { partialAlternativePaths = nil break @@ -351,72 +354,75 @@ func (ds *DataSource) isSpecifiedInIndexMergeHints(name string) bool { } // accessPathsForConds generates all possible index paths for conditions. -func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, usedIndexCount int) []*util.AccessPath { - var results = make([]*util.AccessPath, 0, usedIndexCount) - for i := 0; i < usedIndexCount; i++ { - path := &util.AccessPath{} - if ds.possibleAccessPaths[i].IsTablePath() { +func (ds *DataSource) accessPathsForConds( + conditions []expression.Expression, + candidatePaths []*util.AccessPath, +) []*util.AccessPath { + var results = make([]*util.AccessPath, 0, len(candidatePaths)) + for _, path := range candidatePaths { + newPath := &util.AccessPath{} + if path.IsTablePath() { if !ds.isInIndexMergeHints("primary") { continue } if ds.tableInfo.IsCommonHandle { - path.IsCommonHandlePath = true - path.Index = ds.possibleAccessPaths[i].Index + newPath.IsCommonHandlePath = true + newPath.Index = path.Index } else { - path.IsIntHandlePath = true + newPath.IsIntHandlePath = true } - err := ds.deriveTablePathStats(path, conditions, true) + err := ds.deriveTablePathStats(newPath, conditions, true) if err != nil { logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) continue } var unsignedIntHandle bool - if path.IsIntHandlePath && ds.tableInfo.PKIsHandle { + if newPath.IsIntHandlePath && ds.tableInfo.PKIsHandle { if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil { unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) } } - // If the path contains a full range, ignore it. - if ranger.HasFullRange(path.Ranges, unsignedIntHandle) { + // If the newPath contains a full range, ignore it. + if ranger.HasFullRange(newPath.Ranges, unsignedIntHandle) { continue } // If we have point or empty range, just remove other possible paths. - if len(path.Ranges) == 0 || path.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) { + if len(newPath.Ranges) == 0 || newPath.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) { if len(results) == 0 { - results = append(results, path) + results = append(results, newPath) } else { - results[0] = path + results[0] = newPath results = results[:1] } break } } else { - path.Index = ds.possibleAccessPaths[i].Index - if !ds.isInIndexMergeHints(path.Index.Name.L) { + newPath.Index = path.Index + if !ds.isInIndexMergeHints(newPath.Index.Name.L) { continue } - err := ds.fillIndexPath(path, conditions) + err := ds.fillIndexPath(newPath, conditions) if err != nil { logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) continue } - ds.deriveIndexPathStats(path, conditions, true) - // If the path contains a full range, ignore it. - if ranger.HasFullRange(path.Ranges, false) { + ds.deriveIndexPathStats(newPath, conditions, true) + // If the newPath contains a full range, ignore it. + if ranger.HasFullRange(newPath.Ranges, false) { continue } // If we have empty range, or point range on unique index, just remove other possible paths. - if len(path.Ranges) == 0 || (path.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) && path.Index.Unique) { + if len(newPath.Ranges) == 0 || (newPath.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) && newPath.Index.Unique) { if len(results) == 0 { - results = append(results, path) + results = append(results, newPath) } else { - results[0] = path + results[0] = newPath results = results[:1] } break } } - results = append(results, path) + results = append(results, newPath) } return results } @@ -479,44 +485,6 @@ func (ds *DataSource) buildIndexMergeOrPath( return indexMergePath } -func (ds *DataSource) generateNormalIndexPartialPath4Or(dnfItems []expression.Expression, usedAccessMap []bool, normalPathCnt int) ([]*util.AccessPath, []bool, bool, error) { - remainedDNFItems := make([]expression.Expression, 0, len(dnfItems)) - for i, b := range usedAccessMap { - if !b { - remainedDNFItems = append(remainedDNFItems, dnfItems[i]) - } - } - noMVIndexPartialPath := false - if len(dnfItems) == len(remainedDNFItems) { - // there is no mv index paths generated, so for: (a<1) OR (a>2), no need to generated index merge. - noMVIndexPartialPath = true - } - paths, needSelection, usedMap, err := ds.generateNormalIndexPartialPaths4DNF(remainedDNFItems, normalPathCnt) - if err != nil { - return nil, usedAccessMap, false, err - } - // If all the partialPaths use the same index, we will not use the indexMerge. - singlePath := true - for i := len(paths) - 1; i >= 1; i-- { - if paths[i].Index != paths[i-1].Index { - singlePath = false - break - } - } - if singlePath && noMVIndexPartialPath { - return nil, usedAccessMap, false, nil - } - // collect the remain filter's used map. - cnt := 0 - for i, b := range usedAccessMap { - if !b { - usedAccessMap[i] = usedMap[cnt] - cnt++ - } - } - return paths, usedAccessMap, needSelection, nil -} - func (ds *DataSource) generateNormalIndexPartialPath4And(normalPathCnt int, usedAccessMap map[string]expression.Expression) []*util.AccessPath { if res := ds.generateIndexMergeAndPaths(normalPathCnt, usedAccessMap); res != nil { return res.PartialIndexPaths @@ -660,110 +628,6 @@ func (ds *DataSource) generateIndexMergeAndPaths(normalPathCnt int, usedAccessMa return indexMergePath } -/* -select * from t where ((1 member of (a) and b=1) or (2 member of (a) and b=2)) and (c > 10) - - IndexMerge(OR) - IndexRangeScan(a, b, [1 1, 1 1]) - IndexRangeScan(a, b, [2 2, 2 2]) - Selection(c > 10) - TableRowIdScan(t) - -Two limitations now: -1). Not support the embedded index merge case, which (DNF_Item1 OR DNF_Item2), for every DNF item, -try to map it into a simple normal index path or mv index path, other than an internal index merge path. -2). Every dnf item should exactly be used as full length index range or prefix index range, other than being not used. -*/ -func (ds *DataSource) generateMVIndexPartialPath4Or(normalPathCnt int, indexMergeDNFConds []expression.Expression) ([]*util.AccessPath, []bool, bool, error) { - // step1: collect all mv index paths - possibleMVIndexPaths := make([]*util.AccessPath, 0, len(ds.possibleAccessPaths)) - for idx := 0; idx < normalPathCnt; idx++ { - if !isMVIndexPath(ds.possibleAccessPaths[idx]) { - continue // not a MVIndex path - } - if !ds.isInIndexMergeHints(ds.possibleAccessPaths[idx].Index.Name.L) { - continue - } - possibleMVIndexPaths = append(possibleMVIndexPaths, ds.possibleAccessPaths[idx]) - } - // step2: mapping index merge conditions into possible mv index path - mvAndPartialPaths := make([]*util.AccessPath, 0, len(possibleMVIndexPaths)) - usedMap := make([]bool, len(indexMergeDNFConds)) - needSelection := false - - for offset, dnfCond := range indexMergeDNFConds { - var cnfConds []expression.Expression - sf, ok := dnfCond.(*expression.ScalarFunction) - if !ok { - continue - } - cnfConds = expression.SplitCNFItems(sf) - // for every dnf condition, find the most suitable mv index path. - // otherwise, for table(a json, b json, c int, idx(c,a), idx2(b,c)) - // condition: (1 member of (a) and c=1 and d=2) or (2 member of (b) and c=3 and d=2); - // will both pick(c,a) idx with range [1 1, 1 1] and [3,3], the latter can pick the most - // valuable index idx2 with range [2 3,2 3] - var ( - bestPaths []*util.AccessPath - bestCountAfterAccess float64 - bestNeedSelection bool - ) - for _, onePossibleMVIndexPath := range possibleMVIndexPaths { - idxCols, ok := PrepareCols4MVIndex(ds.table.Meta(), onePossibleMVIndexPath.Index, ds.TblCols) - if !ok { - continue - } - // for every cnfCond, try to map it into possible mv index path. - // remainingFilters is not cared here, because it will be all suspended on the table side. - accessFilters, remainingFilters := collectFilters4MVIndex(ds.SCtx(), cnfConds, idxCols) - if len(accessFilters) == 0 { - continue - } - paths, isIntersection, ok, err := buildPartialPaths4MVIndex(ds.SCtx(), accessFilters, idxCols, onePossibleMVIndexPath.Index, ds.tableStats.HistColl) - if err != nil { - logutil.BgLogger().Debug("build index merge partial mv index paths failed", zap.Error(err)) - return nil, nil, false, err - } - if !ok || len(paths) == 0 { - continue - } - // only under 2 cases we can fallthrough it. - // 1: the index merge only has one partial path. - // 2: index merge is UNION type. - canFallThrough := len(paths) == 1 || !isIntersection - if !canFallThrough { - continue - } - // UNION case, use the max count after access for simplicity. - maxCountAfterAccess := -1.0 - for _, p := range paths { - maxCountAfterAccess = math.Max(maxCountAfterAccess, p.CountAfterAccess) - } - // Note that: here every path is about mv index path. - // find the most valuable mv index path, which means it has the minimum countAfterAccess. - if len(bestPaths) == 0 { - bestPaths = paths - bestCountAfterAccess = maxCountAfterAccess - bestNeedSelection = len(remainingFilters) != 0 - } else if bestCountAfterAccess > maxCountAfterAccess { - bestPaths = paths - bestCountAfterAccess = maxCountAfterAccess - bestNeedSelection = len(remainingFilters) != 0 - } - } - if len(bestPaths) != 0 { - usedMap[offset] = true - // correctly find a dnf condition for this mv index path - mvAndPartialPaths = append(mvAndPartialPaths, bestPaths...) - if !needSelection && bestNeedSelection { - // collect one path's need selection flag. - needSelection = bestNeedSelection - } - } - } - return mvAndPartialPaths, usedMap, needSelection, nil -} - // generateMVIndexMergePartialPaths4And try to find mv index merge partial path from a collection of cnf conditions. func (ds *DataSource) generateMVIndexMergePartialPaths4And(normalPathCnt int, indexMergeConds []expression.Expression, histColl *statistics.HistColl) ([]*util.AccessPath, map[string]expression.Expression, error) { // step1: collect all mv index paths @@ -790,7 +654,12 @@ func (ds *DataSource) generateMVIndexMergePartialPaths4And(normalPathCnt int, in // mm is a map here used for de-duplicate partial paths which is derived from **same** accessFilters, not necessary to keep them both. mm := make(map[string]*record, 0) for idx := 0; idx < len(possibleMVIndexPaths); idx++ { - idxCols, ok := PrepareCols4MVIndex(ds.table.Meta(), possibleMVIndexPaths[idx].Index, ds.TblCols) + idxCols, ok := PrepareIdxColsAndUnwrapArrayType( + ds.table.Meta(), + possibleMVIndexPaths[idx].Index, + ds.TblCols, + true, + ) if !ok { continue } @@ -937,57 +806,28 @@ func (ds *DataSource) generateIndexMergeOnDNF4MVIndex(normalPathCnt int, filters continue } - idxCols, ok := PrepareCols4MVIndex(ds.table.Meta(), ds.possibleAccessPaths[idx].Index, ds.TblCols) - if !ok { - continue - } - for current, filter := range filters { sf, ok := filter.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { continue } - dnfFilters := expression.FlattenDNFConditions(sf) // [(1 member of (a) and b=1), (2 member of (a) and b=2)] - - // build partial paths for each dnf filter - cannotFit := false - var partialPaths []*util.AccessPath - for _, dnfFilter := range dnfFilters { - mvIndexFilters := []expression.Expression{dnfFilter} - if sf, ok := dnfFilter.(*expression.ScalarFunction); ok && sf.FuncName.L == ast.LogicAnd { - mvIndexFilters = expression.FlattenCNFConditions(sf) // (1 member of (a) and b=1) --> [(1 member of (a)), b=1] - } - - accessFilters, remainingFilters := collectFilters4MVIndex(ds.SCtx(), mvIndexFilters, idxCols) - if len(accessFilters) == 0 || len(remainingFilters) > 0 { // limitation 1 - cannotFit = true - break - } - paths, isIntersection, ok, err := buildPartialPaths4MVIndex(ds.SCtx(), accessFilters, idxCols, ds.possibleAccessPaths[idx].Index, ds.tableStats.HistColl) - if err != nil { - return nil, err - } - if isIntersection || !ok { // limitation 2 - cannotFit = true - break - } - partialPaths = append(partialPaths, paths...) - } - if cannotFit { - continue - } + dnfFilters := expression.SplitDNFItems(sf) // [(1 member of (a) and b=1), (2 member of (a) and b=2)] - var remainingFilters []expression.Expression - remainingFilters = append(remainingFilters, filters[:current]...) - remainingFilters = append(remainingFilters, filters[current+1:]...) - - indexMergePath := ds.buildPartialPathUp4MVIndex( - partialPaths, - false, - remainingFilters, - ds.tableStats.HistColl, + unfinishedIndexMergePath := generateUnfinishedIndexMergePathFromORList( + ds, + dnfFilters, + []*util.AccessPath{ds.possibleAccessPaths[idx]}, ) - mvIndexPaths = append(mvIndexPaths, indexMergePath) + finishedIndexMergePath := handleTopLevelANDListAndGenFinishedPath( + ds, + filters, + current, + []*util.AccessPath{ds.possibleAccessPaths[idx]}, + unfinishedIndexMergePath, + ) + if finishedIndexMergePath != nil { + mvIndexPaths = append(mvIndexPaths, finishedIndexMergePath) + } } } return @@ -1065,57 +905,67 @@ func (ds *DataSource) generateIndexMerge4ComposedIndex(normalPathCnt int, indexM return nil } - if len(indexMergeConds) == 1 { - // DNF path. - sf, ok := indexMergeConds[0].(*expression.ScalarFunction) - if !ok || sf.FuncName.L != ast.LogicOr { - // targeting: cond1 or cond2 or cond3 - return nil + // Collect access paths that satisfy the hints, and make sure there is at least one MV index path. + var mvIndexPathCnt int + candidateAccessPaths := make([]*util.AccessPath, 0, len(ds.possibleAccessPaths)) + for idx := 0; idx < normalPathCnt; idx++ { + if ds.possibleAccessPaths[idx].Index != nil && ds.possibleAccessPaths[idx].Index.Global { + continue } - dnfFilters := expression.FlattenDNFConditions(sf) - mvIndexPartialPaths, usedAccessMap, needSelection4MVIndex, err := ds.generateMVIndexPartialPath4Or(normalPathCnt, dnfFilters) - if err != nil { - return err + if (ds.possibleAccessPaths[idx].IsTablePath() && + !ds.isInIndexMergeHints("primary")) || + (!ds.possibleAccessPaths[idx].IsTablePath() && + !ds.isInIndexMergeHints(ds.possibleAccessPaths[idx].Index.Name.L)) { + continue } - if len(mvIndexPartialPaths) == 0 { - // no mv index partial paths to be composed of. - return nil + if isMVIndexPath(ds.possibleAccessPaths[idx]) { + mvIndexPathCnt++ } - normalIndexPartialPaths, usedAccessMap, needSelection4NormalIndex, err := ds.generateNormalIndexPartialPath4Or(dnfFilters, usedAccessMap, normalPathCnt) - if err != nil { - return err + candidateAccessPaths = append(candidateAccessPaths, ds.possibleAccessPaths[idx]) + } + if mvIndexPathCnt == 0 { + return nil + } + + for current, filter := range indexMergeConds { + // DNF path. + sf, ok := filter.(*expression.ScalarFunction) + if !ok || sf.FuncName.L != ast.LogicOr { + // targeting: cond1 or cond2 or cond3 + continue } - // since multi normal index merge path is handled before, here focus on multi mv index merge, or mv and normal mixed index merge - composed := (len(mvIndexPartialPaths) > 1) || (len(mvIndexPartialPaths) == 1 && len(normalIndexPartialPaths) >= 1) - if !composed { + dnfFilters := expression.SplitDNFItems(sf) + + unfinishedIndexMergePath := generateUnfinishedIndexMergePathFromORList( + ds, + dnfFilters, + candidateAccessPaths, + ) + finishedIndexMergePath := handleTopLevelANDListAndGenFinishedPath( + ds, + indexMergeConds, + current, + candidateAccessPaths, + unfinishedIndexMergePath, + ) + if finishedIndexMergePath == nil { return nil } - // if any cnf item is not used as index partial path, index merge is not valid anymore. - if slices.Contains(usedAccessMap, false) { - return nil + + var mvIndexPartialPathCnt, normalIndexPartialPathCnt int + for _, path := range finishedIndexMergePath.PartialIndexPaths { + if isMVIndexPath(path) { + mvIndexPartialPathCnt++ + } else { + normalIndexPartialPathCnt++ + } } - // todo: make this code as a portal of all index merge path. - // if we derive: - // 1: some mv index partial path, no normal index path, it means multi mv index merge. - // 2: some mv index partial path, some normal index path, it means hybrid index merge. - // 3: no mv index partial path, several normal index path, it means multi normal index merge. - combinedPartialPaths := append(normalIndexPartialPaths, mvIndexPartialPaths...) - if len(combinedPartialPaths) == 0 { + + // Keep the same behavior with previous implementation, we only handle the "composed" case here. + if mvIndexPartialPathCnt == 0 || (mvIndexPartialPathCnt == 1 && normalIndexPartialPathCnt == 0) { return nil } - // here we directly use the all index merge conditions as the table filers for simplicity. - // todo: make estimation more correct rather than pruning other index merge path. - var indexMergeTableFilters []expression.Expression - if needSelection4MVIndex || needSelection4NormalIndex { - indexMergeTableFilters = indexMergeConds - } - mvp := ds.buildPartialPathUp4MVIndex( - combinedPartialPaths, - false, - indexMergeTableFilters, - ds.tableStats.HistColl, - ) - ds.possibleAccessPaths = append(ds.possibleAccessPaths, mvp) + ds.possibleAccessPaths = append(ds.possibleAccessPaths, finishedIndexMergePath) return nil } // CNF path. @@ -1203,12 +1053,17 @@ func (ds *DataSource) generateIndexMerge4MVIndex(normalPathCnt int, filters []ex continue } - idxCols, ok := PrepareCols4MVIndex(ds.table.Meta(), ds.possibleAccessPaths[idx].Index, ds.TblCols) + idxCols, ok := PrepareIdxColsAndUnwrapArrayType( + ds.table.Meta(), + ds.possibleAccessPaths[idx].Index, + ds.TblCols, + true, + ) if !ok { continue } - accessFilters, remainingFilters := collectFilters4MVIndex(ds.SCtx(), filters, idxCols) + accessFilters, remainingFilters, _ := collectFilters4MVIndex(ds.SCtx(), filters, idxCols) if len(accessFilters) == 0 { // cannot use any filter on this MVIndex continue } @@ -1307,7 +1162,13 @@ func buildPartialPaths4MVIndex( virColVals = append(virColVals, v) case ast.JSONContains: // (json_contains(a->'$.zip', '[1, 2, 3]') isIntersection = true - virColVals, ok = jsonArrayExpr2Exprs(sctx.GetExprCtx(), ast.JSONContains, sf.GetArgs()[1], jsonType) + virColVals, ok = jsonArrayExpr2Exprs( + sctx.GetExprCtx(), + ast.JSONContains, + sf.GetArgs()[1], + jsonType, + true, + ) if !ok || len(virColVals) == 0 { // json_contains(JSON, '[]') is TRUE. If the row has an empty array, it'll not exist on multi-valued index, // but the `json_contains(array, '[]')` is still true, so also don't try to scan on the index. @@ -1323,7 +1184,13 @@ func buildPartialPaths4MVIndex( return nil, false, false, nil } var ok bool - virColVals, ok = jsonArrayExpr2Exprs(sctx.GetExprCtx(), ast.JSONOverlaps, sf.GetArgs()[1-jsonPathIdx], jsonType) + virColVals, ok = jsonArrayExpr2Exprs( + sctx.GetExprCtx(), + ast.JSONOverlaps, + sf.GetArgs()[1-jsonPathIdx], + jsonType, + true, + ) if !ok || len(virColVals) == 0 { // forbid empty array for safety return nil, false, false, nil } @@ -1390,15 +1257,22 @@ func buildPartialPath4MVIndex( return partialPath, true, nil } -// PrepareCols4MVIndex exported for test. -func PrepareCols4MVIndex( +// PrepareIdxColsAndUnwrapArrayType collects columns for an index and returns them as []*expression.Column. +// If any column of them is an array type, we will use it's underlying FieldType in the returned Column.RetType. +// If checkOnly1ArrayTypeCol is true, we will check if this index contains only one array type column. If not, it will +// return (nil, false). This check works as a sanity check for an MV index. +// Though this function is introduced for MV index, it can also be used for normal index if you pass false to +// checkOnly1ArrayTypeCol. +// This function is exported for test. +func PrepareIdxColsAndUnwrapArrayType( tableInfo *model.TableInfo, - mvIndex *model.IndexInfo, + idxInfo *model.IndexInfo, tblCols []*expression.Column, + checkOnly1ArrayTypeCol bool, ) (idxCols []*expression.Column, ok bool) { var virColNum = 0 - for i := range mvIndex.Columns { - colOffset := mvIndex.Columns[i].Offset + for i := range idxInfo.Columns { + colOffset := idxInfo.Columns[i].Offset colMeta := tableInfo.Cols()[colOffset] var col *expression.Column for _, c := range tblCols { @@ -1419,7 +1293,7 @@ func PrepareCols4MVIndex( } idxCols = append(idxCols, col) } - if virColNum != 1 { // assume only one vir-col in the MVIndex + if checkOnly1ArrayTypeCol && virColNum != 1 { // assume only one vir-col in the MVIndex return nil, false } return idxCols, true @@ -1428,7 +1302,12 @@ func PrepareCols4MVIndex( // collectFilters4MVIndex splits these filters into 2 parts where accessFilters can be used to access this index directly. // For idx(x, cast(a as array), z), `x=1 and (2 member of a) and z=1 and x+z>0` is split to: // accessFilters: `x=1 and (2 member of a) and z=1`, remaining: `x+z>0`. -func collectFilters4MVIndex(sctx context.PlanContext, filters []expression.Expression, idxCols []*expression.Column) (accessFilters, remainingFilters []expression.Expression) { +func collectFilters4MVIndex( + sctx context.PlanContext, + filters []expression.Expression, + idxCols []*expression.Column, +) (accessFilters, remainingFilters []expression.Expression, accessTp int) { + accessTp = unspecifiedFilterTp usedAsAccess := make([]bool, len(filters)) for _, col := range idxCols { found := false @@ -1436,10 +1315,14 @@ func collectFilters4MVIndex(sctx context.PlanContext, filters []expression.Expre if usedAsAccess[i] { continue } - if checkFilter4MVIndexColumn(sctx, f, col) { + if ok, tp := checkAccessFilter4IdxCol(sctx, f, col); ok { accessFilters = append(accessFilters, f) usedAsAccess[i] = true found = true + // access filter type on mv col overrides normal col for the return value of this function + if accessTp == unspecifiedFilterTp || accessTp == eqOnNonMVColTp { + accessTp = tp + } break } } @@ -1452,7 +1335,7 @@ func collectFilters4MVIndex(sctx context.PlanContext, filters []expression.Expre remainingFilters = append(remainingFilters, filters[i]) } } - return accessFilters, remainingFilters + return accessFilters, remainingFilters, accessTp } // CollectFilters4MVIndexMutations exported for unit test. @@ -1506,7 +1389,7 @@ func CollectFilters4MVIndexMutations(sctx PlanContext, filters []expression.Expr if usedAsAccess[i] { continue } - if checkFilter4MVIndexColumn(sctx, f, col) { + if ok, _ := checkAccessFilter4IdxCol(sctx, f, col); ok { if col.VirtualExpr != nil && col.VirtualExpr.GetType().IsArray() { // assert jsonColOffset should always be the same. // if the filter is from virtual expression, it means it is about the mv json col. @@ -1590,57 +1473,135 @@ func indexMergeContainSpecificIndex(path *util.AccessPath, indexSet map[int64]st return false } -// checkFilter4MVIndexColumn checks whether this filter can be used as an accessFilter to access the MVIndex column. -func checkFilter4MVIndexColumn(sctx PlanContext, filter expression.Expression, idxCol *expression.Column) bool { +const ( + unspecifiedFilterTp int = iota + eqOnNonMVColTp + multiValuesOROnMVColTp + multiValuesANDOnMVColTp + singleValueOnMVColTp +) + +// checkAccessFilter4IdxCol checks whether this filter can be used as an accessFilter to access the column of an index, +// and returns which type the access filter is, as defined above. +// Though this function is introduced for MV index, it can also be used for normal index +// If the return value ok is false, the type must be unspecifiedFilterTp. +func checkAccessFilter4IdxCol( + sctx PlanContext, + filter expression.Expression, + idxCol *expression.Column, +) ( + ok bool, + accessFilterTp int, +) { sf, ok := filter.(*expression.ScalarFunction) if !ok { - return false + return false, unspecifiedFilterTp } if idxCol.VirtualExpr != nil { // the virtual column on the MVIndex targetJSONPath, ok := unwrapJSONCast(idxCol.VirtualExpr) if !ok { - return false + return false, unspecifiedFilterTp } + var virColVals []expression.Expression + jsonType := idxCol.GetType().ArrayType() + var tp int switch sf.FuncName.L { case ast.JSONMemberOf: // (1 member of a) - return targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[1]) + if !targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[1]) { + return false, unspecifiedFilterTp + } + v, ok := unwrapJSONCast(sf.GetArgs()[0]) // cast(1 as json) --> 1 + if !ok { + return false, unspecifiedFilterTp + } + virColVals = append(virColVals, v) + tp = singleValueOnMVColTp case ast.JSONContains: // json_contains(a, '1') - return targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[0]) + if !targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[0]) { + return false, unspecifiedFilterTp + } + virColVals, ok = jsonArrayExpr2Exprs( + sctx.GetExprCtx(), + ast.JSONContains, + sf.GetArgs()[1], + jsonType, + false, + ) + if !ok || len(virColVals) == 0 { + return false, unspecifiedFilterTp + } + tp = multiValuesANDOnMVColTp case ast.JSONOverlaps: // json_overlaps(a, '1') or json_overlaps('1', a) - return targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[0]) || - targetJSONPath.Equal(sctx.GetExprCtx(), sf.GetArgs()[1]) + var jsonPathIdx int + if sf.GetArgs()[0].Equal(sctx.GetExprCtx(), targetJSONPath) { + jsonPathIdx = 0 // (json_overlaps(a->'$.zip', '[1, 2, 3]') + } else if sf.GetArgs()[1].Equal(sctx.GetExprCtx(), targetJSONPath) { + jsonPathIdx = 1 // (json_overlaps('[1, 2, 3]', a->'$.zip') + } else { + return false, unspecifiedFilterTp + } + var ok bool + virColVals, ok = jsonArrayExpr2Exprs( + sctx.GetExprCtx(), + ast.JSONOverlaps, + sf.GetArgs()[1-jsonPathIdx], + jsonType, + false, + ) + if !ok || len(virColVals) == 0 { // forbid empty array for safety + return false, unspecifiedFilterTp + } + tp = multiValuesOROnMVColTp default: - return false - } - } else { - if sf.FuncName.L != ast.EQ { // only support EQ now - return false + return false, unspecifiedFilterTp } - args := sf.GetArgs() - var argCol *expression.Column - var argConst *expression.Constant - if c, isCol := args[0].(*expression.Column); isCol { - if con, isCon := args[1].(*expression.Constant); isCon { - argCol, argConst = c, con - } - } else if c, isCol := args[1].(*expression.Column); isCol { - if con, isCon := args[0].(*expression.Constant); isCon { - argCol, argConst = c, con + for _, v := range virColVals { + if !isSafeTypeConversion4MVIndexRange(v.GetType(), idxCol.GetType()) { + return false, unspecifiedFilterTp } } - if argCol == nil || argConst == nil { - return false + // If json_contains or json_overlaps only contains one value, like json_overlaps(a,'[1]') or + // json_contains(a,'[1]'), we can just ignore the AND/OR semantic, and treat them like 1 member of (a). + if (tp == multiValuesOROnMVColTp || tp == multiValuesANDOnMVColTp) && len(virColVals) == 1 { + tp = singleValueOnMVColTp } - if argCol.Equal(sctx.GetExprCtx(), idxCol) { - return true + return true, tp + } + + // else: non virtual column + if sf.FuncName.L != ast.EQ { // only support EQ now + return false, unspecifiedFilterTp + } + args := sf.GetArgs() + var argCol *expression.Column + var argConst *expression.Constant + if c, isCol := args[0].(*expression.Column); isCol { + if con, isCon := args[1].(*expression.Constant); isCon { + argCol, argConst = c, con + } + } else if c, isCol := args[1].(*expression.Column); isCol { + if con, isCon := args[0].(*expression.Constant); isCon { + argCol, argConst = c, con } } - return false + if argCol == nil || argConst == nil { + return false, unspecifiedFilterTp + } + if argCol.Equal(sctx.GetExprCtx(), idxCol) { + return true, eqOnNonMVColTp + } + return false, unspecifiedFilterTp } // jsonArrayExpr2Exprs converts a JsonArray expression to expression list: cast('[1, 2, 3]' as JSON) --> []expr{1, 2, 3} -func jsonArrayExpr2Exprs(sctx expression.BuildContext, jsonFuncName string, jsonArrayExpr expression.Expression, targetType *types.FieldType) ([]expression.Expression, bool) { - if expression.MaybeOverOptimized4PlanCache(sctx, []expression.Expression{jsonArrayExpr}) { +func jsonArrayExpr2Exprs( + sctx expression.BuildContext, + jsonFuncName string, + jsonArrayExpr expression.Expression, + targetType *types.FieldType, + checkForSkipPlanCache bool, +) ([]expression.Expression, bool) { + if checkForSkipPlanCache && expression.MaybeOverOptimized4PlanCache(sctx, []expression.Expression{jsonArrayExpr}) { // skip plan cache and try to generate the best plan in this case. sctx.GetSessionVars().StmtCtx.SetSkipPlanCache(errors.NewNoStackError(jsonFuncName + " function with immutable parameters can affect index selection")) } diff --git a/pkg/planner/core/indexmerge_path_test.go b/pkg/planner/core/indexmerge_path_test.go index a8499a0b0a311..d2b2f74b3d5d9 100644 --- a/pkg/planner/core/indexmerge_path_test.go +++ b/pkg/planner/core/indexmerge_path_test.go @@ -40,7 +40,7 @@ func TestCollectFilters4MVIndexMutations(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, domains json null, images json null, KEY `a_domains_b` (a, (cast(`domains` as char(253) array)), b))") - sql := "SELECT * FROM t WHERE 15975127 member of (domains) AND 15975128 member of (domains) AND a = 1 AND b = 2" + sql := "SELECT * FROM t WHERE '15975127' member of (domains) AND '15975128' member of (domains) AND a = 1 AND b = 2" par := parser.New() par.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true}) @@ -72,7 +72,12 @@ func TestCollectFilters4MVIndexMutations(t *testing.T) { cnfs := ds.GetAllConds() tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) - idxCols, ok := core.PrepareCols4MVIndex(tbl.Meta(), tbl.Meta().FindIndexByName("a_domains_b"), ds.TblCols) + idxCols, ok := core.PrepareIdxColsAndUnwrapArrayType( + tbl.Meta(), + tbl.Meta().FindIndexByName("a_domains_b"), + ds.TblCols, + true, + ) require.True(t, ok) accessFilters, _, mvColOffset, mvFilterMutations := core.CollectFilters4MVIndexMutations(tk.Session().GetPlanCtx(), cnfs, idxCols) @@ -336,7 +341,7 @@ func TestPlanCacheMVIndex(t *testing.T) { check(false, `select * from ti where ? member of (domains) OR ? member of (signatures) OR json_overlaps(f_profile_ids, ?)`, fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), randV(`"[0,1]"`, `"[0,1,2]"`, `"[0]"`)) check(false, `select * from ti where ? member of (domains) OR ? member of (signatures) OR ? member of (f_profile_ids) OR json_contains(f_profile_ids, ?)`, fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("%v", rand.Intn(30)), randV(`"[0,1]"`, `"[0,1,2]"`, `"[0]"`)) check(false, `select * from ti WHERE ? member of (domains) OR ? member of (signatures) OR ? member of (long_link) OR json_contains(f_profile_ids, ?)`, fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), randV(`"[0,1]"`, `"[0,1,2]"`, `"[0]"`)) - check(false, `select * from ti WHERE ? member of (domains) AND ? member of (signatures) AND json_contains(f_profile_ids, ?)`, fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), randV(`'["0","1","2"]'`, `'["0"]'`, `'["1","2"]'`)) + check(false, `select * from ti WHERE ? member of (domains) AND ? member of (signatures) AND json_contains(f_profile_ids, ?)`, fmt.Sprintf("'%v'", rand.Intn(30)), fmt.Sprintf("'%v'", rand.Intn(30)), randV(`'[0,1,2]'`, `'[0]'`, `'[1,2]'`)) } } diff --git a/pkg/planner/core/indexmerge_unfinished_path.go b/pkg/planner/core/indexmerge_unfinished_path.go new file mode 100644 index 0000000000000..e9ad3eb878ac9 --- /dev/null +++ b/pkg/planner/core/indexmerge_unfinished_path.go @@ -0,0 +1,416 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "math" + "slices" + + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/planner/util" +) + +// Note that at this moment, the implementation related to unfinishedAccessPath only aims to handle OR list nested in +// AND list, which is like ... AND (... OR ... OR ...) AND ..., and build an MV IndexMerge access path. So some struct +// definition and code logic are specially designed for this case or only consider this case. +// This may be changed in the future. + +// unfinishedAccessPath is for collecting access filters for an access path. +// It maintains the information during iterating all filters. Importantly, it maintains incomplete access filters, which +// means they may not be able to build a valid range, but could build a valid range after collecting more access filters. +// After iterating all filters, we can check and build it into a valid util.AccessPath. +type unfinishedAccessPath struct { + index *model.IndexInfo + + accessFilters []expression.Expression + + // To avoid regression and keep the same behavior as the previous implementation, we collect access filters in two + // methods: + // + // 1. Use the same functions as the previous implementation to collect access filters. They are able to handle some + // complicated expressions, but the expressions must be able to build into a valid range at once (that's also what + // "finished" implies). + // In this case, idxColHasAccessFilter will be nil and initedAsFinished will be true. + // + // 2. Use the new logic, which is to collect access filters for each column respectively, gradually collect more + // access filters during iterating all filters and try to form a valid range at last. + // In this case, initedAsFinished will be false, and idxColHasAccessFilter will record if we have already collected + // a valid access filter for each column of the index. + idxColHasAccessFilter []bool + initedAsFinished bool + + // needKeepFilter means the OR list need to become a filter in the final Selection. + needKeepFilter bool + + // Similar to AccessPath.PartialIndexPaths, each element in the slice is expected to build into a partial AccessPath. + // Currently, it can only mean an OR type IndexMerge. + indexMergeORPartialPaths []unfinishedAccessPathList +} + +// unfinishedAccessPathList is for collecting access filters for a slice of candidate access paths. +// This type is useful because usually we have several candidate index/table paths. When we are iterating the +// expressions, we want to match them against all index/table paths and try to find access filter for every path. After +// iterating all expressions, we check if any of them can form a valid range so that we can build a valid AccessPath. +type unfinishedAccessPathList []*unfinishedAccessPath + +// generateUnfinishedIndexMergePathFromORList handles a list of filters connected by OR, collects access filters for +// each candidate access path, and returns an unfinishedAccessPath, which must be an index merge OR unfinished path, +// each partial path of which corresponds to one filter in the input orList. +/* +Example: + Input: + orList: 1 member of j->'$.a' OR 2 member of j->'$.b' + candidateAccessPaths: [idx1(a, j->'$.a' unsigned array), idx2(j->'$.b' unsigned array, a)] + Output: + unfinishedAccessPath{ + indexMergeORPartialPaths: [ + // Collect access filters for (1 member of j->'$.a') using two candidates respectively. + [unfinishedAccessPath{idx1,1 member of j->'$.a'}, nil] + // Collect access filters for (2 member of j->'$.b') using two candidates respectively. + [nil, unfinishedAccessPath{idx2,2 member of j->'$.b'}] + ] + } +*/ +func generateUnfinishedIndexMergePathFromORList( + ds *DataSource, + orList []expression.Expression, + candidateAccessPaths []*util.AccessPath, +) *unfinishedAccessPath { + if len(orList) < 2 { + return nil + } + unfinishedPartialPaths := make([]unfinishedAccessPathList, 0, len(orList)) + for _, singleFilter := range orList { + unfinishedPathList := initUnfinishedPathsFromExpr(ds, candidateAccessPaths, singleFilter) + if unfinishedPathList == nil { + return nil + } + unfinishedPartialPaths = append(unfinishedPartialPaths, unfinishedPathList) + } + return &unfinishedAccessPath{ + indexMergeORPartialPaths: unfinishedPartialPaths, + } +} + +// initUnfinishedPathsFromExpr tries to collect access filters from the input filter for each candidate access path, +// and returns them as a slice of unfinishedAccessPath, each of which corresponds to an input candidate access path. +// If we failed to collect access filters for one candidate access path, the corresponding element in the return slice +// will be nil. +// If we failed to collect access filters for all candidate access paths, this function will return nil. +/* +Example1 (consistent with the one in generateUnfinishedIndexMergePathFromORList()): + Input: + expr: 1 member of j->'$.a' + candidateAccessPaths: [idx1(a, j->'$.a' unsigned array), idx2(j->'$.b' unsigned array, a)] + Output: + [unfinishedAccessPath{idx1,1 member of j->'$.a'}, nil] + +Example2: + Input: + expr: a = 3 + candidateAccessPaths: [idx1(a, j->'$.a' unsigned array), idx2(j->'$.b' unsigned array, a)] + Output: + [unfinishedAccessPath{idx1,a=3}, unfinishedAccessPath{idx2,a=3}] +*/ +func initUnfinishedPathsFromExpr( + ds *DataSource, + candidateAccessPaths []*util.AccessPath, + expr expression.Expression, +) unfinishedAccessPathList { + retValues := make([]unfinishedAccessPath, len(candidateAccessPaths)) + ret := make([]*unfinishedAccessPath, 0, len(candidateAccessPaths)) + for i := range candidateAccessPaths { + ret = append(ret, &retValues[i]) + } + for i, path := range candidateAccessPaths { + ret[i].index = path.Index + // case 1: try to use the previous logic to handle non-mv index + if !isMVIndexPath(path) { + // generateNormalIndexPartialPaths4DNF is introduced for handle a slice of DNF items and a slice of + // candidate AccessPaths before, now we reuse it to handle single filter and single candidate AccessPath, + // so we need to wrap them in a slice here. + paths, needSelection, usedMap := ds.generateNormalIndexPartialPaths4DNF( + []expression.Expression{expr}, + []*util.AccessPath{path}, + ) + if len(usedMap) == 1 && usedMap[0] && len(paths) == 1 { + ret[i].initedAsFinished = true + ret[i].accessFilters = paths[0].AccessConds + ret[i].needKeepFilter = needSelection + continue + } + } + if path.IsTablePath() { + continue + } + idxCols, ok := PrepareIdxColsAndUnwrapArrayType(ds.table.Meta(), path.Index, ds.TblCols, false) + if !ok { + continue + } + cnfItems := expression.SplitCNFItems(expr) + + // case 2: try to use the previous logic to handle mv index + if isMVIndexPath(path) { + accessFilters, remainingFilters, tp := collectFilters4MVIndex(ds.SCtx(), cnfItems, idxCols) + if len(accessFilters) > 0 && (tp == multiValuesOROnMVColTp || tp == singleValueOnMVColTp) { + ret[i].initedAsFinished = true + ret[i].accessFilters = accessFilters + ret[i].needKeepFilter = len(remainingFilters) > 0 + continue + } + } + + // case 3: use the new logic if the previous logic didn't succeed to collect access filters that can build a + // valid range directly. + ret[i].idxColHasAccessFilter = make([]bool, len(idxCols)) + for j, col := range idxCols { + for _, cnfItem := range cnfItems { + if ok, tp := checkAccessFilter4IdxCol(ds.SCtx(), cnfItem, col); ok && + // Since we only handle the OR list nested in the AND list, and only generate IndexMerge OR path, + // we disable the multiValuesANDOnMVColTp case here. + (tp == eqOnNonMVColTp || tp == multiValuesOROnMVColTp || tp == singleValueOnMVColTp) { + ret[i].accessFilters = append(ret[i].accessFilters, cnfItem) + ret[i].idxColHasAccessFilter[j] = true + // Once we find one valid access filter for this column, we directly go to the next column without + // looking into other filters. + break + } + } + } + } + + validCnt := 0 + // remove useless paths + for i, path := range ret { + if !path.initedAsFinished && + !slices.Contains(path.idxColHasAccessFilter, true) { + ret[i] = nil + } else { + validCnt++ + } + } + if validCnt == 0 { + return nil + } + return ret +} + +// handleTopLevelANDListAndGenFinishedPath is expected to be used together with +// generateUnfinishedIndexMergePathFromORList() to handle the expression like ... AND (... OR ... OR ...) AND ... +// for mv index. +// It will try to collect possible access filters from other items in the top level AND list and try to merge them into +// the unfinishedAccessPath from generateUnfinishedIndexMergePathFromORList(), and try to build it into a valid +// util.AccessPath. +// The input candidateAccessPaths argument should be the same with generateUnfinishedIndexMergePathFromORList(). +func handleTopLevelANDListAndGenFinishedPath( + ds *DataSource, + allConds []expression.Expression, + orListIdxInAllConds int, + candidateAccessPaths []*util.AccessPath, + unfinishedIndexMergePath *unfinishedAccessPath, +) *util.AccessPath { + for i, cnfItem := range allConds { + // Skip the (... OR ... OR ...) in the list. + if i == orListIdxInAllConds { + continue + } + // Collect access filters from one AND item. + pathListFromANDItem := initUnfinishedPathsFromExpr(ds, candidateAccessPaths, cnfItem) + // Try to merge useful access filters in them into unfinishedIndexMergePath, which is from the nested OR list. + unfinishedIndexMergePath = mergeANDItemIntoUnfinishedIndexMergePath(unfinishedIndexMergePath, pathListFromANDItem) + } + if unfinishedIndexMergePath == nil { + return nil + } + return buildIntoAccessPath( + ds, + candidateAccessPaths, + unfinishedIndexMergePath, + allConds, + orListIdxInAllConds, + ) +} + +/* +Example (consistent with the one in generateUnfinishedIndexMergePathFromORList()): + + idx1: (a, j->'$.a' unsigned array) idx2: (j->'$.b' unsigned array, a) + Input: + indexMergePath: + unfinishedAccessPath{ indexMergeORPartialPaths:[ + [unfinishedAccessPath{idx1,1 member of j->'$.a'}, nil] + [nil, unfinishedAccessPath{idx2,2 member of j->'$.b'}] + ]} + pathListFromANDItem: + [unfinishedAccessPath{idx1,a=3}, unfinishedAccessPath{idx2,a=3}] + Output: + unfinishedAccessPath{ indexMergeORPartialPaths:[ + [unfinishedAccessPath{idx1,1 member of j->'$.a', a=3}, nil] + [nil, unfinishedAccessPath{idx2,2 member of j->'$.b', a=3}] + ]} +*/ +func mergeANDItemIntoUnfinishedIndexMergePath( + indexMergePath *unfinishedAccessPath, + pathListFromANDItem unfinishedAccessPathList, +) *unfinishedAccessPath { + // Currently, we only handle the case where indexMergePath is an index merge OR unfinished path and + // pathListFromANDItem is a normal unfinished path or nil + if indexMergePath == nil || len(indexMergePath.indexMergeORPartialPaths) == 0 { + return nil + } + // This means we failed to find any valid access filter from other expressions in the top level AND list. + // In this case, we ignore them and only rely on the nested OR list to try to build a IndexMerge OR path. + if pathListFromANDItem == nil { + return indexMergePath + } + for _, pathListForSinglePartialPath := range indexMergePath.indexMergeORPartialPaths { + if len(pathListForSinglePartialPath) != len(pathListFromANDItem) { + continue + } + for i, path := range pathListForSinglePartialPath { + if path == nil || pathListFromANDItem[i] == nil { + continue + } + // We don't do precise checks. As long as any columns have valid access filters, we collect the entire + // access filters from the AND item. + // We just collect as many possibly useful access filters as possible, buildIntoAccessPath() should handle + // them correctly. + if pathListFromANDItem[i].initedAsFinished || + slices.Contains(pathListFromANDItem[i].idxColHasAccessFilter, true) { + path.accessFilters = append(path.accessFilters, pathListFromANDItem[i].accessFilters...) + } + } + } + return indexMergePath +} + +func buildIntoAccessPath( + ds *DataSource, + originalPaths []*util.AccessPath, + indexMergePath *unfinishedAccessPath, + allConds []expression.Expression, + orListIdxInAllConds int, +) *util.AccessPath { + if indexMergePath == nil || len(indexMergePath.indexMergeORPartialPaths) == 0 { + return nil + } + var needSelectionGlobal bool + + // 1. Generate one or more partial access path for each partial unfinished path (access filter on mv index may + // produce several partial paths). + partialPaths := make([]*util.AccessPath, 0, len(indexMergePath.indexMergeORPartialPaths)) + + // for each partial path + for _, unfinishedPathList := range indexMergePath.indexMergeORPartialPaths { + var ( + bestPaths []*util.AccessPath + bestCountAfterAccess float64 + bestNeedSelection bool + ) + + // for each possible access path of this partial path + for i, unfinishedPath := range unfinishedPathList { + if unfinishedPath == nil { + continue + } + var paths []*util.AccessPath + var needSelection bool + if unfinishedPath.index != nil && unfinishedPath.index.MVIndex { + // case 1: mv index + idxCols, ok := PrepareIdxColsAndUnwrapArrayType( + ds.table.Meta(), + unfinishedPath.index, + ds.TblCols, + true, + ) + if !ok { + continue + } + accessFilters, remainingFilters, _ := collectFilters4MVIndex( + ds.SCtx(), + unfinishedPath.accessFilters, + idxCols, + ) + if len(accessFilters) == 0 { + continue + } + var isIntersection bool + var err error + paths, isIntersection, ok, err = buildPartialPaths4MVIndex( + ds.SCtx(), + accessFilters, + idxCols, + unfinishedPath.index, + ds.tableStats.HistColl, + ) + if err != nil || !ok || (isIntersection && len(paths) > 1) { + continue + } + needSelection = len(remainingFilters) > 0 || len(unfinishedPath.idxColHasAccessFilter) > 0 + } else { + // case 2: non-mv index + var usedMap []bool + // Reuse the previous implementation. The same usage as in initUnfinishedPathsFromExpr(). + paths, needSelection, usedMap = ds.generateNormalIndexPartialPaths4DNF( + []expression.Expression{ + expression.ComposeCNFCondition( + ds.SCtx().GetExprCtx(), + unfinishedPath.accessFilters..., + ), + }, + []*util.AccessPath{originalPaths[i]}, + ) + if len(paths) != 1 || slices.Contains(usedMap, false) { + continue + } + } + needSelection = needSelection || unfinishedPath.needKeepFilter + // If there are several partial paths, we use the max CountAfterAccess for comparison. + maxCountAfterAccess := -1.0 + for _, p := range paths { + maxCountAfterAccess = math.Max(maxCountAfterAccess, p.CountAfterAccess) + } + // Choose the best partial path for this partial path. + if len(bestPaths) == 0 { + bestPaths = paths + bestCountAfterAccess = maxCountAfterAccess + bestNeedSelection = needSelection + } else if bestCountAfterAccess > maxCountAfterAccess { + bestPaths = paths + bestCountAfterAccess = maxCountAfterAccess + bestNeedSelection = needSelection + } + } + if len(bestPaths) == 0 { + return nil + } + // Succeeded to get valid path(s) for this partial path. + partialPaths = append(partialPaths, bestPaths...) + needSelectionGlobal = needSelectionGlobal || bestNeedSelection + } + + // 2. Collect the final table filter + // We always put all filters in the top level AND list except for the OR list into the final table filters. + // Whether to put the OR list into the table filters also depends on the needSelectionGlobal. + tableFilter := allConds[:] + if !needSelectionGlobal { + tableFilter = slices.Delete(tableFilter, orListIdxInAllConds, orListIdxInAllConds+1) + } + + // 3. Build the final access path + ret := ds.buildPartialPathUp4MVIndex(partialPaths, false, tableFilter, ds.tableStats.HistColl) + return ret +} diff --git a/pkg/planner/core/integration_test.go b/pkg/planner/core/integration_test.go index 176681521543b..a1dffb4ac6031 100644 --- a/pkg/planner/core/integration_test.go +++ b/pkg/planner/core/integration_test.go @@ -2227,14 +2227,14 @@ func TestIssue48257(t *testing.T) { tk.MustExec("analyze table t1") tk.MustQuery("explain format = brief select * from t1").Check(testkit.Rows( "TableReader 1.00 root data:TableFullScan", - "└─TableFullScan 1.00 cop[tikv] table:t1 keep order:false", + "└─TableFullScan 1.00 cop[tikv] table:t1 keep order:false, stats:pseudo", )) tk.MustExec("insert into t1 value(1)") require.NoError(t, h.DumpStatsDeltaToKV(true)) require.NoError(t, h.Update(dom.InfoSchema())) tk.MustQuery("explain format = brief select * from t1").Check(testkit.Rows( "TableReader 2.00 root data:TableFullScan", - "└─TableFullScan 2.00 cop[tikv] table:t1 keep order:false", + "└─TableFullScan 2.00 cop[tikv] table:t1 keep order:false, stats:pseudo", )) tk.MustExec("set tidb_opt_objective='determinate'") tk.MustQuery("explain format = brief select * from t1").Check(testkit.Rows( diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index f9fe807686836..30553937d1a5a 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -62,6 +62,7 @@ import ( "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" "github.com/pingcap/tidb/pkg/util/hack" h "github.com/pingcap/tidb/pkg/util/hint" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/intset" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/plancodec" @@ -4464,7 +4465,7 @@ func getStatsTable(ctx PlanContext, tblInfo *model.TableInfo, pid int64) *statis } // 1. tidb-server started and statistics handle has not been initialized. if statsHandle == nil { - return statistics.PseudoTable(tblInfo, false) + return statistics.PseudoTable(tblInfo, false, true) } if pid == tblInfo.ID || ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { @@ -4473,6 +4474,7 @@ func getStatsTable(ctx PlanContext, tblInfo *model.TableInfo, pid int64) *statis usePartitionStats = true statsTbl = statsHandle.GetPartitionStats(tblInfo, pid) } + intest.Assert(statsTbl.ColAndIdxExistenceMap != nil, "The existence checking map must not be nil.") allowPseudoTblTriggerLoading := false // In OptObjectiveDeterminate mode, we need to ignore the real-time stats. @@ -4502,7 +4504,7 @@ func getStatsTable(ctx PlanContext, tblInfo *model.TableInfo, pid int64) *statis if statsTbl.RealtimeCount == 0 { countIs0 = true core_metrics.PseudoEstimationNotAvailable.Inc() - return statistics.PseudoTable(tblInfo, allowPseudoTblTriggerLoading) + return statistics.PseudoTable(tblInfo, allowPseudoTblTriggerLoading, true) } // 3. statistics is uninitialized or outdated. @@ -4788,7 +4790,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as h := domain.GetDomain(b.ctx).StatsHandle() tblStats := h.GetTableStats(tableInfo) isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() - globalStatsReady := tblStats.IsInitialized() + globalStatsReady := tblStats.IsAnalyzed() skipMissingPartition := b.ctx.GetSessionVars().SkipMissingPartitionStats // If we already enabled the tidb_skip_missing_partition_stats, the global stats can be treated as exist. allowDynamicWithoutStats := fixcontrol.GetBoolWithDefault(b.ctx.GetSessionVars().GetOptimizerFixControlMap(), fixcontrol.Fix44262, skipMissingPartition) diff --git a/pkg/planner/core/mock.go b/pkg/planner/core/mock.go index 0b63002c6c5e8..cbe24ce6b179a 100644 --- a/pkg/planner/core/mock.go +++ b/pkg/planner/core/mock.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/mock" ) @@ -412,6 +413,7 @@ func MockContext() *mock.Context { Client: &mock.Client{}, } ctx.GetSessionVars().CurrentDB = "test" + ctx.GetSessionVars().DivPrecisionIncrement = variable.DefDivPrecisionIncrement do := domain.NewMockDomain() if err := do.CreateStatsHandle(ctx, initStatsCtx); err != nil { panic(fmt.Sprintf("create mock context panic: %+v", err)) diff --git a/pkg/planner/core/plan_cache_test.go b/pkg/planner/core/plan_cache_test.go index f5b8670be73ed..543cc30198d54 100644 --- a/pkg/planner/core/plan_cache_test.go +++ b/pkg/planner/core/plan_cache_test.go @@ -1411,7 +1411,7 @@ func TestPlanCacheMVIndexRandomly(t *testing.T) { verifyPlanCacheForMVIndex(t, tk, true, true, `select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where (? member of (a) and c=? and d=?) or (? member of (b) and c=? and d=?)`, `int`, `int`, `int`, `int`, `int`, `int`) - verifyPlanCacheForMVIndex(t, tk, true, false, + verifyPlanCacheForMVIndex(t, tk, false, false, `select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where ( json_contains(a, ?) and c=? and d=?) or (? member of (b) and c=? and d=?)`, `json-signed`, `int`, `int`, `int`, `int`, `int`) verifyPlanCacheForMVIndex(t, tk, true, false, diff --git a/pkg/planner/core/plan_stats.go b/pkg/planner/core/plan_stats.go index 1c9ada6926283..9d87d2996be56 100644 --- a/pkg/planner/core/plan_stats.go +++ b/pkg/planner/core/plan_stats.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/table" - "github.com/pingcap/tidb/pkg/util/intset" "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -48,74 +47,13 @@ func (collectPredicateColumnsPoint) optimize(_ context.Context, plan LogicalPlan // Prepare the table metadata to avoid repeatedly fetching from the infoSchema below, and trigger extra sync/async // stats loading for the determinate mode. is := plan.SCtx().GetInfoSchema().(infoschema.InfoSchema) - statsHandle := domain.GetDomain(plan.SCtx()).StatsHandle() tblID2Tbl := make(map[int64]table.Table) - physTblIDsWithNeededCols := intset.NewFastIntSet() - for _, neededCol := range histNeededColumns { - physTblIDsWithNeededCols.Insert(int(neededCol.TableID)) - } visitedPhysTblIDs.ForEach(func(physicalTblID int) { - // 1. collect table metadata tbl, _ := infoschema.FindTableByTblOrPartID(is, int64(physicalTblID)) if tbl == nil { return } tblID2Tbl[int64(physicalTblID)] = tbl - - // 2. handle extra sync/async stats loading for the determinate mode - - // If we visited a table without getting any columns need stats (likely because there are no pushed down - // predicates), and we are in the determinate mode, we need to make sure we are able to get the "analyze row - // count" in getStatsTable(), which means any column/index stats are available. - if plan.SCtx().GetSessionVars().GetOptObjective() != variable.OptObjectiveDeterminate || - // If we already collected some columns that need trigger sync laoding on this table, we don't need to - // additionally do anything for determinate mode. - physTblIDsWithNeededCols.Has(physicalTblID) || - statsHandle == nil { - return - } - tblStats := statsHandle.GetTableStats(tbl.Meta()) - if tblStats == nil || tblStats.Pseudo { - return - } - var colToTriggerLoad *model.TableItemID - for _, col := range tbl.Cols() { - if col.State != model.StatePublic || (col.IsGenerated() && !col.GeneratedStored) { - continue - } - if colStats := tblStats.Columns[col.ID]; colStats != nil { - // Choose the first column we meet to trigger stats loading. - if colToTriggerLoad == nil && colStats.IsStatsInitialized() { - colToTriggerLoad = &model.TableItemID{TableID: int64(physicalTblID), ID: col.ID, IsIndex: false} - } - // If any stats are already full loaded, we don't need to trigger stats loading on this table. - if colStats.IsFullLoad() { - colToTriggerLoad = nil - break - } - } - } - if colToTriggerLoad == nil { - return - } - for _, idx := range tbl.Indices() { - if idx.Meta().State != model.StatePublic || idx.Meta().MVIndex { - continue - } - // If any stats are already full loaded, we don't need to trigger stats loading on this table. - if idxStats := tblStats.Indices[idx.Meta().ID]; idxStats != nil && idxStats.IsFullLoad() { - colToTriggerLoad = nil - break - } - } - if colToTriggerLoad == nil { - return - } - if histNeeded { - histNeededColumns = append(histNeededColumns, *colToTriggerLoad) - } else { - statistics.HistogramNeededItems.Insert(*colToTriggerLoad) - } }) // collect needed virtual columns from already needed columns @@ -156,7 +94,7 @@ func (syncWaitStatsLoadPoint) name() string { } // RequestLoadStats send load column/index stats requests to stats handle -func RequestLoadStats(ctx PlanContext, neededHistItems []model.TableItemID, syncWait int64) error { +func RequestLoadStats(ctx PlanContext, neededHistItems []model.StatsLoadItem, syncWait int64) error { maxExecutionTime := ctx.GetSessionVars().GetMaxExecutionTime() if maxExecutionTime > 0 && maxExecutionTime < uint64(syncWait) { syncWait = int64(maxExecutionTime) @@ -178,7 +116,7 @@ func RequestLoadStats(ctx PlanContext, neededHistItems []model.TableItemID, sync stmtCtx.AppendWarning(err) return nil } - logutil.BgLogger().Error("RequestLoadStats failed", zap.Error(err)) + logutil.BgLogger().Warn("RequestLoadStats failed", zap.Error(err)) return err } return nil @@ -234,8 +172,8 @@ func SyncWaitStatsLoad(plan LogicalPlan) error { // but d will not be collected. // It's because currently it's impossible that statistics related to indirectly depending columns are actually needed. // If we need to check indirect dependency some day, we can easily extend the logic here. -func CollectDependingVirtualCols(tblID2Tbl map[int64]table.Table, neededItems []model.TableItemID) []model.TableItemID { - generatedCols := make([]model.TableItemID, 0) +func CollectDependingVirtualCols(tblID2Tbl map[int64]table.Table, neededItems []model.StatsLoadItem) []model.StatsLoadItem { + generatedCols := make([]model.StatsLoadItem, 0) // group the neededItems by table id tblID2neededColIDs := make(map[int64][]int64, len(tblID2Tbl)) @@ -275,7 +213,7 @@ func CollectDependingVirtualCols(tblID2Tbl map[int64]table.Table, neededItems [] // then we think this virtual column is needed. for depCol := range col.Dependences { if _, ok := colNameSet[depCol]; ok { - generatedCols = append(generatedCols, model.TableItemID{TableID: tblID, ID: col.ID, IsIndex: false}) + generatedCols = append(generatedCols, model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tblID, ID: col.ID, IsIndex: false}, FullLoad: true}) break } } @@ -289,7 +227,7 @@ func CollectDependingVirtualCols(tblID2Tbl map[int64]table.Table, neededItems [] // composed up by A column, then we thought the idx_a should be collected // 2. The stats condition of idx_a can't meet IsFullLoad, which means its stats was evicted previously func collectSyncIndices(ctx PlanContext, - histNeededColumns []model.TableItemID, + histNeededColumns []model.StatsLoadItem, tblID2Tbl map[int64]table.Table, ) map[model.TableItemID]struct{} { histNeededIndices := make(map[model.TableItemID]struct{}) @@ -317,19 +255,21 @@ func collectSyncIndices(ctx PlanContext, if tblStats == nil || tblStats.Pseudo { continue } - idxStats, ok := tblStats.Indices[idx.Meta().ID] - if ok && idxStats.IsStatsInitialized() && !idxStats.IsFullLoad() { - histNeededIndices[model.TableItemID{TableID: column.TableID, ID: idxID, IsIndex: true}] = struct{}{} + _, loadNeeded := tblStats.IndexIsLoadNeeded(idxID) + if !loadNeeded { + continue } + histNeededIndices[model.TableItemID{TableID: column.TableID, ID: idxID, IsIndex: true}] = struct{}{} } } } return histNeededIndices } -func collectHistNeededItems(histNeededColumns []model.TableItemID, histNeededIndices map[model.TableItemID]struct{}) (histNeededItems []model.TableItemID) { +func collectHistNeededItems(histNeededColumns []model.StatsLoadItem, histNeededIndices map[model.TableItemID]struct{}) (histNeededItems []model.StatsLoadItem) { + histNeededItems = make([]model.StatsLoadItem, 0, len(histNeededColumns)+len(histNeededIndices)) for idx := range histNeededIndices { - histNeededItems = append(histNeededItems, idx) + histNeededItems = append(histNeededItems, model.StatsLoadItem{TableItemID: idx, FullLoad: true}) } histNeededItems = append(histNeededItems, histNeededColumns...) return diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 0bedfec4c1dfc..c07597c98b2cf 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -847,10 +847,10 @@ func (b *PlanBuilder) buildCreateBindPlanFromPlanDigest(v *ast.CreateBindingStmt if err != nil { return nil, errors.Errorf("binding failed: %v", err) } - if err = hint.CheckBindingFromHistoryBindable(originNode, bindableStmt.PlanHint); err != nil { - return nil, err + if complete, reason := hint.CheckBindingFromHistoryComplete(originNode, bindableStmt.PlanHint); !complete { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError(reason)) } - bindSQL := bindinfo.GenerateBindingSQL(context.TODO(), originNode, bindableStmt.PlanHint, true, bindableStmt.Schema) + bindSQL := bindinfo.GenerateBindingSQL(originNode, bindableStmt.PlanHint, true, bindableStmt.Schema) var hintNode ast.StmtNode hintNode, err = parser4binding.ParseOneStmt(bindSQL, bindableStmt.Charset, bindableStmt.Collation) if err != nil { @@ -1503,10 +1503,10 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName m Ranges: ranger.FullRange(), physicalTableID: physicalID, isPartition: isPartition, - tblColHists: &(statistics.PseudoTable(tblInfo, false)).HistColl, + tblColHists: &(statistics.PseudoTable(tblInfo, false, false)).HistColl, }.Init(b.ctx, b.getSelectOffset()) // There is no alternative plan choices, so just use pseudo stats to avoid panic. - is.SetStats(&property.StatsInfo{HistColl: &(statistics.PseudoTable(tblInfo, false)).HistColl}) + is.SetStats(&property.StatsInfo{HistColl: &(statistics.PseudoTable(tblInfo, false, false)).HistColl}) if hasCommonCols { for _, c := range commonInfos { is.Columns = append(is.Columns, c.ColumnInfo) @@ -1522,7 +1522,7 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName m DBName: dbName, physicalTableID: physicalID, isPartition: isPartition, - tblColHists: &(statistics.PseudoTable(tblInfo, false)).HistColl, + tblColHists: &(statistics.PseudoTable(tblInfo, false, false)).HistColl, }.Init(b.ctx, b.getSelectOffset()) ts.SetSchema(idxColSchema) ts.Columns = ExpandVirtualColumn(ts.Columns, ts.schema, ts.Table.Columns) diff --git a/pkg/planner/core/point_get_plan.go b/pkg/planner/core/point_get_plan.go index f5e7390a2d3be..550c8b989d209 100644 --- a/pkg/planner/core/point_get_plan.go +++ b/pkg/planner/core/point_get_plan.go @@ -934,7 +934,8 @@ func getLockWaitTime(ctx PlanContext, lockInfo *ast.SelectLockInfo) (lock bool, // autocommit to 0. If autocommit is enabled, the rows matching the specification are not locked. // See https://dev.mysql.com/doc/refman/5.7/en/innodb-locking-reads.html sessVars := ctx.GetSessionVars() - if !sessVars.IsAutocommit() || sessVars.InTxn() || config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() { + if !sessVars.IsAutocommit() || sessVars.InTxn() || (config.GetGlobalConfig(). + PessimisticTxn.PessimisticAutoCommit.Load() && !sessVars.BulkDMLEnabled) { lock = true waitTime = sessVars.LockWaitTimeout if lockInfo.LockType == ast.SelectLockForUpdateWaitN { diff --git a/pkg/planner/core/rule_predicate_push_down.go b/pkg/planner/core/rule_predicate_push_down.go index 810f332772542..9f1db21b805ef 100644 --- a/pkg/planner/core/rule_predicate_push_down.go +++ b/pkg/planner/core/rule_predicate_push_down.go @@ -395,14 +395,6 @@ func simplifyOuterJoin(p *LogicalJoin, predicates []expression.Expression) { innerTable, outerTable = outerTable, innerTable } - // first simplify embedded outer join. - if innerPlan, ok := innerTable.(*LogicalJoin); ok { - simplifyOuterJoin(innerPlan, predicates) - } - if outerPlan, ok := outerTable.(*LogicalJoin); ok { - simplifyOuterJoin(outerPlan, predicates) - } - if p.JoinType == InnerJoin { return } diff --git a/pkg/planner/core/stats.go b/pkg/planner/core/stats.go index be2c3965ed806..6e08615b7d787 100644 --- a/pkg/planner/core/stats.go +++ b/pkg/planner/core/stats.go @@ -70,7 +70,7 @@ func (p *LogicalMemTable) DeriveStats(_ []*property.StatsInfo, selfSchema *expre if p.StatsInfo() != nil { return p.StatsInfo(), nil } - statsTable := statistics.PseudoTable(p.TableInfo, false) + statsTable := statistics.PseudoTable(p.TableInfo, false, false) stats := &property.StatsInfo{ RowCount: float64(statsTable.RealtimeCount), ColNDVs: make(map[int64]float64, len(p.TableInfo.Columns)), @@ -263,11 +263,12 @@ func (ds *DataSource) initStats(colGroups [][]*expression.Column) { statsRecord := ds.SCtx().GetSessionVars().StmtCtx.GetUsedStatsInfo(true) name, tblInfo := getTblInfoForUsedStatsByPhysicalID(ds.SCtx(), ds.physicalTableID) statsRecord.RecordUsedInfo(ds.physicalTableID, &stmtctx.UsedStatsInfoForTable{ - Name: name, - TblInfo: tblInfo, - Version: tableStats.StatsVersion, - RealtimeCount: tableStats.HistColl.RealtimeCount, - ModifyCount: tableStats.HistColl.ModifyCount, + Name: name, + TblInfo: tblInfo, + Version: tableStats.StatsVersion, + RealtimeCount: tableStats.HistColl.RealtimeCount, + ModifyCount: tableStats.HistColl.ModifyCount, + ColAndIdxStatus: ds.statisticTable.ColAndIdxExistenceMap, }) for _, col := range ds.schema.Columns { diff --git a/pkg/planner/memo/BUILD.bazel b/pkg/planner/memo/BUILD.bazel index dd538beb3c894..d78fbd7b42549 100644 --- a/pkg/planner/memo/BUILD.bazel +++ b/pkg/planner/memo/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "group_expr.go", "implementation.go", "pattern.go", + "task.go", ], importpath = "github.com/pingcap/tidb/pkg/planner/memo", visibility = ["//visibility:public"], @@ -27,10 +28,11 @@ go_test( "group_test.go", "main_test.go", "pattern_test.go", + "task_test.go", ], embed = [":memo"], flaky = True, - shard_count = 22, + shard_count = 24, deps = [ "//pkg/domain", "//pkg/expression", diff --git a/pkg/planner/memo/task.go b/pkg/planner/memo/task.go new file mode 100644 index 0000000000000..4380c69ddaf67 --- /dev/null +++ b/pkg/planner/memo/task.go @@ -0,0 +1,77 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "sync" +) + +// Task is an interface defined for all type of optimizing work: exploring, implementing, deriving-stats, join-reordering and so on. +type Task interface { + // task self executing logic + execute() error + // task self description string. + desc() string +} + +// TaskStackPool is initialized for memory saving by reusing taskStack. +var TaskStackPool = sync.Pool{ + New: func() any { + return newTaskStack() + }, +} + +// TaskStack is used to store the optimizing tasks created before or during the optimizing process. +type TaskStack struct { + tasks []Task +} + +func newTaskStack() *TaskStack { + return &TaskStack{ + tasks: make([]Task, 0, 4), + } +} + +// Destroy indicates that when stack itself is useless like in the end of optimizing phase, we can destroy ourselves. +func (ts *TaskStack) Destroy() { + // when a taskStack itself is useless, we can destroy itself actively. + clear(ts.tasks) + TaskStackPool.Put(ts) +} + +// Len indicates the length of current stack. +func (ts *TaskStack) Len() int { + return len(ts.tasks) +} + +// Pop indicates to pop one task out of the stack. +func (ts *TaskStack) Pop() Task { + if !ts.Empty() { + tmp := ts.tasks[len(ts.tasks)-1] + ts.tasks = ts.tasks[:len(ts.tasks)-1] + return tmp + } + return nil +} + +// Push indicates to push one task into the stack. +func (ts *TaskStack) Push(one Task) { + ts.tasks = append(ts.tasks, one) +} + +// Empty indicates whether taskStack is empty. +func (ts *TaskStack) Empty() bool { + return ts.Len() == 0 +} diff --git a/pkg/planner/memo/task_test.go b/pkg/planner/memo/task_test.go new file mode 100644 index 0000000000000..d14d39ccb4661 --- /dev/null +++ b/pkg/planner/memo/task_test.go @@ -0,0 +1,95 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "strconv" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" +) + +type TestTaskImpl struct { + a int64 +} + +func (t *TestTaskImpl) execute() error { + return nil +} +func (t *TestTaskImpl) desc() string { + return strconv.Itoa(int(t.a)) +} + +func TestTaskStack(t *testing.T) { + newSS := newTaskStack() + // size of pointer to TaskStack{} + require.Equal(t, int64(unsafe.Sizeof(newSS)), int64(8)) + // size of pointer to TaskStack.[]Task, cap + len + addr + require.Equal(t, int64(unsafe.Sizeof(newSS.tasks)), int64(24)) + // size of pointer to TaskStack's first element Task[0] + newSS.Push(nil) + newSS.Push(&TestTaskImpl{a: 1}) + newSS.Push(nil) + v := unsafe.Sizeof(newSS.tasks[0]) + require.Equal(t, int64(v), int64(16)) + v = unsafe.Sizeof(newSS.tasks[1]) + require.Equal(t, int64(v), int64(16)) +} + +func TestTaskFunctionality(t *testing.T) { + taskTaskPool := TaskStackPool.Get() + require.Equal(t, len(taskTaskPool.(*TaskStack).tasks), 0) + require.Equal(t, cap(taskTaskPool.(*TaskStack).tasks), 4) + taskStack := taskTaskPool.(*TaskStack) + taskStack.Push(&TestTaskImpl{a: 1}) + taskStack.Push(&TestTaskImpl{a: 2}) + one := taskStack.Pop() + require.Equal(t, one.desc(), "2") + one = taskStack.Pop() + require.Equal(t, one.desc(), "1") + // empty, pop nil. + one = taskStack.Pop() + require.Nil(t, one) + + taskStack.Push(&TestTaskImpl{a: 3}) + taskStack.Push(&TestTaskImpl{a: 4}) + taskStack.Push(&TestTaskImpl{a: 5}) + taskStack.Push(&TestTaskImpl{a: 6}) + // no clean, put it back + TaskStackPool.Put(taskTaskPool) + + // require again. + taskTaskPool = TaskStackPool.Get() + require.Equal(t, len(taskTaskPool.(*TaskStack).tasks), 4) + require.Equal(t, cap(taskTaskPool.(*TaskStack).tasks), 4) + // clean the stack + one = taskStack.Pop() + require.Equal(t, one.desc(), "6") + one = taskStack.Pop() + require.Equal(t, one.desc(), "5") + one = taskStack.Pop() + require.Equal(t, one.desc(), "4") + one = taskStack.Pop() + require.Equal(t, one.desc(), "3") + one = taskStack.Pop() + require.Nil(t, one) + + // self destroy. + taskStack.Destroy() + taskTaskPool = TaskStackPool.Get() + require.Equal(t, len(taskTaskPool.(*TaskStack).tasks), 0) + require.Equal(t, cap(taskTaskPool.(*TaskStack).tasks), 4) +} diff --git a/pkg/planner/optimize.go b/pkg/planner/optimize.go index 3eb68aa0f1ad4..915fc537fcbc9 100644 --- a/pkg/planner/optimize.go +++ b/pkg/planner/optimize.go @@ -280,7 +280,6 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in if sessVars.StmtCtx.EnableOptimizerDebugTrace { core.DebugTraceTryBinding(pctx, binding.Hint) } - metrics.BindUsageCounter.WithLabelValues(scope).Inc() hint.BindHint(stmtNode, binding.Hint) curStmtHints, _, curWarns := hint.ParseStmtHints(binding.Hint.GetFirstTableHints(), setVarHintChecker, byte(kv.ReplicaReadFollower)) sessVars.StmtCtx.StmtHints = curStmtHints @@ -579,7 +578,7 @@ func setVarHintChecker(varName, hint string) (ok bool, warning error) { if sysVar == nil { // no such a variable return false, plannererrors.ErrUnresolvedHintName.FastGenByArgs(varName, hint) } - if !sysVar.IsHintUpdatableVerfied { + if !sysVar.IsHintUpdatableVerified { warning = plannererrors.ErrNotHintUpdatable.FastGenByArgs(varName) } return true, warning diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel index ef63c250b22e9..c6b7ad1e527f7 100644 --- a/pkg/server/BUILD.bazel +++ b/pkg/server/BUILD.bazel @@ -190,6 +190,7 @@ go_test( "//pkg/util/chunk", "//pkg/util/context", "//pkg/util/dbterror/exeerrors", + "//pkg/util/plancodec", "//pkg/util/replayer", "//pkg/util/sqlkiller", "//pkg/util/syncutil", diff --git a/pkg/server/conn.go b/pkg/server/conn.go index 01b01e7d85699..7988dac574198 100644 --- a/pkg/server/conn.go +++ b/pkg/server/conn.go @@ -1160,7 +1160,7 @@ func (cc *clientConn) Run(ctx context.Context) { zap.Stringer("sql", getLastStmtInConn{cc}), zap.String("txn_mode", txnMode), zap.Uint64("timestamp", startTS), - zap.String("err", errStrForLog(err, cc.ctx.GetSessionVars().EnableRedactNew)), + zap.String("err", errStrForLog(err, cc.ctx.GetSessionVars().EnableRedactLog)), ) } err1 := cc.writeError(ctx, err) @@ -1173,7 +1173,7 @@ func (cc *clientConn) Run(ctx context.Context) { } func errStrForLog(err error, redactMode string) string { - if redactMode == "ON" { + if redactMode == errors.RedactLogEnable { // currently, only ErrParse is considered when enableRedactLog because it may contain sensitive information like // password or accesskey if parser.ErrParse.Equal(err) { @@ -2613,7 +2613,7 @@ func (cc getLastStmtInConn) String() string { return "ListFields " + string(data) case mysql.ComQuery, mysql.ComStmtPrepare: sql := string(hack.String(data)) - sql = parser.Normalize(sql, cc.ctx.GetSessionVars().EnableRedactNew) + sql = parser.Normalize(sql, cc.ctx.GetSessionVars().EnableRedactLog) return executor.FormatSQL(sql).String() case mysql.ComStmtExecute, mysql.ComStmtFetch: stmtID := binary.LittleEndian.Uint32(data[0:4]) @@ -2645,7 +2645,7 @@ func (cc getLastStmtInConn) PProfLabel() string { case mysql.ComStmtReset: return "ResetStmt" case mysql.ComQuery, mysql.ComStmtPrepare: - return parser.Normalize(executor.FormatSQL(string(hack.String(data))).String(), "ON") + return parser.Normalize(executor.FormatSQL(string(hack.String(data))).String(), errors.RedactLogEnable) case mysql.ComStmtExecute, mysql.ComStmtFetch: stmtID := binary.LittleEndian.Uint32(data[0:4]) return executor.FormatSQL(cc.preparedStmt2StringNoArgs(stmtID)).String() diff --git a/pkg/server/conn_stmt.go b/pkg/server/conn_stmt.go index 4b234a0594e45..fbf690b135cd9 100644 --- a/pkg/server/conn_stmt.go +++ b/pkg/server/conn_stmt.go @@ -581,9 +581,9 @@ func (cc *clientConn) preparedStmt2String(stmtID uint32) string { if sv == nil { return "" } - sql := parser.Normalize(cc.preparedStmt2StringNoArgs(stmtID), sv.EnableRedactNew) - if m := sv.EnableRedactNew; m != "ON" { - sql += redact.String(sv.EnableRedactNew, sv.PlanCacheParams.String()) + sql := parser.Normalize(cc.preparedStmt2StringNoArgs(stmtID), sv.EnableRedactLog) + if m := sv.EnableRedactLog; m != errors.RedactLogEnable { + sql += redact.String(sv.EnableRedactLog, sv.PlanCacheParams.String()) } return sql } diff --git a/pkg/server/conn_test.go b/pkg/server/conn_test.go index d4353af605020..25161e4536260 100644 --- a/pkg/server/conn_test.go +++ b/pkg/server/conn_test.go @@ -52,6 +52,7 @@ import ( "github.com/pingcap/tidb/pkg/util/arena" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" + "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/sqlkiller" "github.com/stretchr/testify/require" tikverr "github.com/tikv/client-go/v2/error" @@ -720,6 +721,16 @@ func TestConnExecutionTimeout(t *testing.T) { tk.MustQuery("select SLEEP(1);").Check(testkit.Rows("0")) err := tk.QueryToErr("select * FROM testTable2 WHERE SLEEP(1);") require.Equal(t, "[executor:3024]Query execution was interrupted, maximum statement execution time exceeded", err.Error()) + // Test executor stats when execution time exceeded. + tk.MustExec("set @@tidb_slow_log_threshold=300") + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/store/mockstore/unistore/unistoreRPCSlowByInjestSleep", `return(150)`)) + err = tk.QueryToErr("select /*+ max_execution_time(600), set_var(tikv_client_read_timeout=100) */ * from testTable2") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/mockstore/unistore/unistoreRPCSlowByInjestSleep")) + require.Error(t, err) + require.Equal(t, "[executor:3024]Query execution was interrupted, maximum statement execution time exceeded", err.Error()) + planInfo, err := plancodec.DecodePlan(tk.Session().GetSessionVars().StmtCtx.GetEncodedPlan()) + require.NoError(t, err) + require.Regexp(t, "TableReader.*cop_task: {num: .*, rpc_num: .*, rpc_time: .*", planInfo) // Killed because of max execution time, reset Killed to 0. tk.Session().GetSessionVars().SQLKiller.SendKillSignal(sqlkiller.MaxExecTimeExceeded) diff --git a/pkg/server/testdata/optimizer_suite_out.json b/pkg/server/testdata/optimizer_suite_out.json index 90839db93dbd9..d538e402e73ae 100644 --- a/pkg/server/testdata/optimizer_suite_out.json +++ b/pkg/server/testdata/optimizer_suite_out.json @@ -106,7 +106,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": true, "IsInvalid": true, "TotalCount": 0 @@ -137,7 +137,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": false, "InValidForCollPseudo": true, "IsInvalid": true, @@ -160,7 +160,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": true, "IsInvalid": true, "TotalCount": 0 @@ -349,7 +349,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": false, "InValidForCollPseudo": true, "IsInvalid": true, @@ -372,7 +372,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": true, "IsInvalid": true, "TotalCount": 0 @@ -532,7 +532,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": true, "IsInvalid": true, "TotalCount": 0 @@ -564,7 +564,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Column).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.ColumnStatsIsInvalid": { "EssentialLoaded": false, "InValidForCollPseudo": true, "IsInvalid": true, @@ -589,7 +589,7 @@ ] }, { - "github.com/pingcap/tidb/pkg/statistics.(*Index).IsInvalid": { + "github.com/pingcap/tidb/pkg/statistics.IndexStatsIsInvalid": { "CollPseudo": true, "IsInvalid": true, "TotalCount": 0 diff --git a/pkg/server/tests/commontest/tidb_test.go b/pkg/server/tests/commontest/tidb_test.go index dccefb05c262f..617274d6d39dc 100644 --- a/pkg/server/tests/commontest/tidb_test.go +++ b/pkg/server/tests/commontest/tidb_test.go @@ -2959,6 +2959,7 @@ func TestProxyProtocolWithIpNoFallbackable(t *testing.T) { ts := servertestkit.CreateTidbTestSuite(t) // Prepare Server + server2.RunInGoTestChan = make(chan struct{}) server, err := server2.NewServer(cfg, ts.Tidbdrv) require.NoError(t, err) server.SetDomain(ts.Domain) @@ -2966,7 +2967,7 @@ func TestProxyProtocolWithIpNoFallbackable(t *testing.T) { err := server.Run(nil) require.NoError(t, err) }() - time.Sleep(time.Millisecond * 1000) + <-server2.RunInGoTestChan defer func() { server.Close() }() diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 55b5b585f1eb9..7e844cbe0ebb9 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/disttask/framework/proto" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/expression" @@ -1045,48 +1046,47 @@ const ( // enlarge `VARIABLE_VALUE` of `mysql.global_variables` from `varchar(1024)` to `varchar(16383)`. version179 = 179 - // version 180 + // ... + // [version180, version189] is the version range reserved for patches of 7.5.x + // ... + + // version 190 // add priority/create_time/end_time to `mysql.tidb_global_task`/`mysql.tidb_global_task_history` // add concurrency/create_time/end_time/digest to `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` // add idx_exec_id(exec_id), uk_digest to `mysql.tidb_background_subtask` // add cpu_count to mysql.dist_framework_meta - version180 = 180 + // modify `mysql.dist_framework_meta` host from VARCHAR(100) to VARCHAR(261) + // modify `mysql.tidb_background_subtask`/`mysql.tidb_background_subtask_history` exec_id from varchar(256) to VARCHAR(261) + // modify `mysql.tidb_global_task`/`mysql.tidb_global_task_history` dispatcher_id from varchar(256) to VARCHAR(261) + version190 = 190 - // version 181 + // version 191 // set tidb_txn_mode to Optimistic when tidb_txn_mode is not set. - version181 = 181 + version191 = 191 - // version 182 + // version 192 // add new system table `mysql.request_unit_by_group`, which is used for // historical RU consumption by resource group per day. - version182 = 182 + version192 = 192 - // version 183 + // version 193 // replace `mysql.tidb_mdl_view` table - version183 = 183 + version193 = 193 - // version 184 + // version 194 // remove `mysql.load_data_jobs` table - version184 = 184 + version194 = 194 - // version 185 + // version 195 // drop `mysql.schema_index_usage` table // create `sys` schema // create `sys.schema_unused_indexes` table - version185 = 185 - - // version 186 - // modify `mysql.dist_framework_meta` host from VARCHAR(100) to VARCHAR(261) - // modify `mysql.tidb_background_subtask` exec_id from varchar(256) to VARCHAR(261) - // modify `mysql.tidb_background_subtask_history` exec_id from varchar(256) to VARCHAR(261) - // modify `mysql.tidb_global_task` dispatcher_id from varchar(256) to VARCHAR(261) - // modify `mysql.tidb_global_task_history` dispatcher_id from varchar(256) to VARCHAR(261) - version186 = 186 + version195 = 195 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version186 +var currentBootstrapVersion int64 = version195 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 @@ -1241,13 +1241,12 @@ var ( upgradeToVer177, upgradeToVer178, upgradeToVer179, - upgradeToVer180, - upgradeToVer181, - upgradeToVer182, - upgradeToVer183, - upgradeToVer184, - upgradeToVer185, - upgradeToVer186, + upgradeToVer190, + upgradeToVer191, + upgradeToVer192, + upgradeToVer193, + upgradeToVer194, + upgradeToVer195, } ) @@ -1311,6 +1310,47 @@ var ( SupportUpgradeHTTPOpVer int64 = version174 ) +func checkDistTask(s sessiontypes.Session) { + ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBootstrap) + rs, err := s.ExecuteInternal(ctx, "SELECT HIGH_PRIORITY variable_value from mysql.global_variables where variable_name = %?;", variable.TiDBEnableDistTask) + if err != nil { + logutil.BgLogger().Fatal("check dist task failed, getting tidb_enable_dist_task failed", zap.Error(err)) + } + defer terror.Call(rs.Close) + req := rs.NewChunk(nil) + err = rs.Next(ctx, req) + if err != nil { + logutil.BgLogger().Fatal("check dist task failed, getting tidb_enable_dist_task failed", zap.Error(err)) + } + if req.NumRows() == 0 { + // Not set yet. + return + } else if req.GetRow(0).GetString(0) == variable.On { + logutil.BgLogger().Fatal("check dist task failed, tidb_enable_dist_task is enabled", zap.Error(err)) + } + + // Even if the variable is set to `off`, we still need to check the tidb_global_task. + rs2, err := s.ExecuteInternal(ctx, `SELECT id FROM %n.%n WHERE state not in (%?, %?, %?)`, + mysql.SystemDB, + "tidb_global_task", + proto.TaskStateSucceed, + proto.TaskStateFailed, + proto.TaskStateReverted, + ) + if err != nil { + logutil.BgLogger().Fatal("check dist task failed, reading tidb_global_task failed", zap.Error(err)) + } + defer terror.Call(rs2.Close) + req = rs2.NewChunk(nil) + err = rs2.Next(ctx, req) + if err != nil { + logutil.BgLogger().Fatal("check dist task failed, reading tidb_global_task failed", zap.Error(err)) + } + if req.NumRows() > 0 { + logutil.BgLogger().Fatal("check dist task failed, some distributed tasks is still running", zap.Error(err)) + } +} + // upgrade function will do some upgrade works, when the system is bootstrapped by low version TiDB server // For example, add new system variables into mysql.global_variables table. func upgrade(s sessiontypes.Session) { @@ -1320,6 +1360,8 @@ func upgrade(s sessiontypes.Session) { // It is already bootstrapped/upgraded by a higher version TiDB server. return } + + checkDistTask(s) printClusterState(s, ver) // Only upgrade from under version92 and this TiDB is not owner set. @@ -2956,8 +2998,8 @@ func upgradeToVer179(s sessiontypes.Session, ver int64) { doReentrantDDL(s, "ALTER TABLE mysql.global_variables MODIFY COLUMN `VARIABLE_VALUE` varchar(16383)") } -func upgradeToVer180(s sessiontypes.Session, ver int64) { - if ver >= version180 { +func upgradeToVer190(s sessiontypes.Session, ver int64) { + if ver >= version190 { return } doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN `priority` INT DEFAULT 1 AFTER `state`", infoschema.ErrColumnExists) @@ -2980,10 +3022,16 @@ func upgradeToVer180(s sessiontypes.Session, ver int64) { doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask ADD UNIQUE INDEX uk_task_key_step_ordinal(task_key, step, ordinal)", dbterror.ErrDupKeyName) doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta ADD COLUMN `cpu_count` INT DEFAULT 0 AFTER `role`", infoschema.ErrColumnExists) + + doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta MODIFY COLUMN `host` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask MODIFY COLUMN `exec_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history MODIFY COLUMN `exec_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task MODIFY COLUMN `dispatcher_id` VARCHAR(261)") + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history MODIFY COLUMN `dispatcher_id` VARCHAR(261)") } -func upgradeToVer181(s sessiontypes.Session, ver int64) { - if ver >= version181 { +func upgradeToVer191(s sessiontypes.Session, ver int64) { + if ver >= version191 { return } sql := fmt.Sprintf("INSERT HIGH_PRIORITY IGNORE INTO %s.%s VALUES('%s', '%s')", @@ -2992,47 +3040,35 @@ func upgradeToVer181(s sessiontypes.Session, ver int64) { mustExecute(s, sql) } -func upgradeToVer182(s sessiontypes.Session, ver int64) { - if ver >= version182 { +func upgradeToVer192(s sessiontypes.Session, ver int64) { + if ver >= version192 { return } doReentrantDDL(s, CreateRequestUnitByGroupTable) } -func upgradeToVer183(s sessiontypes.Session, ver int64) { - if ver >= version183 { +func upgradeToVer193(s sessiontypes.Session, ver int64) { + if ver >= version193 { return } doReentrantDDL(s, CreateMDLView) } -func upgradeToVer184(s sessiontypes.Session, ver int64) { - if ver >= version184 { +func upgradeToVer194(s sessiontypes.Session, ver int64) { + if ver >= version194 { return } mustExecute(s, "DROP TABLE IF EXISTS mysql.load_data_jobs") } -func upgradeToVer185(s sessiontypes.Session, ver int64) { - if ver >= version185 { +func upgradeToVer195(s sessiontypes.Session, ver int64) { + if ver >= version195 { return } doReentrantDDL(s, DropMySQLIndexUsageTable) } -func upgradeToVer186(s sessiontypes.Session, ver int64) { - if ver >= version186 { - return - } - - doReentrantDDL(s, "ALTER TABLE mysql.dist_framework_meta MODIFY COLUMN `host` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask MODIFY COLUMN `exec_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_background_subtask_history MODIFY COLUMN `exec_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task MODIFY COLUMN `dispatcher_id` VARCHAR(261)") - doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history MODIFY COLUMN `dispatcher_id` VARCHAR(261)") -} - func writeOOMAction(s sessiontypes.Session) { comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+" mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index 66dd4fafb6c83..eb3e5a32a77c0 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/failpoint" + "github.com/pingcap/log" "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/domain" @@ -36,6 +37,8 @@ import ( "github.com/pingcap/tidb/pkg/store/mockstore" tbctximpl "github.com/pingcap/tidb/pkg/table/contextimpl" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) // This test file have many problem. @@ -2185,3 +2188,79 @@ func TestTiDBUpgradeToVer179(t *testing.T) { dom.Close() } + +func testTiDBUpgradeWithDistTask(t *testing.T, injectQuery string, fatal bool) { + store, _ := CreateStoreAndBootstrap(t) + defer func() { + require.NoError(t, store.Close()) + }() + ver178 := version178 + seV178 := CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMeta(txn) + err = m.FinishBootstrap(int64(ver178)) + require.NoError(t, err) + MustExec(t, seV178, fmt.Sprintf("update mysql.tidb set variable_value=%d where variable_name='tidb_server_version'", ver178)) + MustExec(t, seV178, injectQuery) + err = txn.Commit(context.Background()) + require.NoError(t, err) + + unsetStoreBootstrapped(store.UUID()) + ver, err := getBootstrapVersion(seV178) + require.NoError(t, err) + require.Equal(t, int64(ver178), ver) + + conf := new(log.Config) + lg, p, e := log.InitLogger(conf, zap.WithFatalHook(zapcore.WriteThenPanic)) + require.NoError(t, e) + rs := log.ReplaceGlobals(lg, p) + defer func() { + rs() + }() + + var dom *domain.Domain + + fatal2panic := false + fc := func() { + defer func() { + if err := recover(); err != nil { + fatal2panic = true + } + }() + dom, _ = BootstrapSession(store) + } + fc() + + if fatal { + dom = domain.GetDomain(seV178) + } + dom.Close() + require.Equal(t, fatal, fatal2panic) +} + +func TestTiDBUpgradeWithDistTaskEnable(t *testing.T) { + t.Run("test enable dist task", func(t *testing.T) { testTiDBUpgradeWithDistTask(t, "set global tidb_enable_dist_task = 1", true) }) + t.Run("test disable dist task", func(t *testing.T) { testTiDBUpgradeWithDistTask(t, "set global tidb_enable_dist_task = 0", false) }) +} + +func TestTiDBUpgradeWithDistTaskRunning(t *testing.T) { + t.Run("test dist task running", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'running'", true) + }) + t.Run("test dist task succeed", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'succeed'", false) + }) + t.Run("test dist task failed", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'failed'", false) + }) + t.Run("test dist task reverted", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'reverted'", false) + }) + t.Run("test dist task paused", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'paused'", true) + }) + t.Run("test dist task other", func(t *testing.T) { + testTiDBUpgradeWithDistTask(t, "insert into mysql.tidb_global_task set id = 1, task_key = 'aaa', type= 'aaa', state = 'other'", true) + }) +} diff --git a/pkg/session/nontransactional.go b/pkg/session/nontransactional.go index c1a5787c18b56..3cd89ce8b3ea2 100644 --- a/pkg/session/nontransactional.go +++ b/pkg/session/nontransactional.go @@ -120,7 +120,7 @@ func HandleNonTransactionalDML(ctx context.Context, stmt *ast.NonTransactionalDM if stmt.DryRun == ast.DryRunSplitDml { return buildDryRunResults(stmt.DryRun, splitStmts, se.GetSessionVars().BatchSize.MaxChunkSize) } - return buildExecuteResults(ctx, jobs, se.GetSessionVars().BatchSize.MaxChunkSize, se.GetSessionVars().EnableRedactNew) + return buildExecuteResults(ctx, jobs, se.GetSessionVars().BatchSize.MaxChunkSize, se.GetSessionVars().EnableRedactLog) } // we require: @@ -280,7 +280,7 @@ func runJobs(ctx context.Context, jobs []job, stmt *ast.NonTransactionalDMLStmt, failedJobs := make([]string, 0) for _, job := range jobs { if job.err != nil { - failedJobs = append(failedJobs, fmt.Sprintf("job:%s, error: %s", job.String(se.GetSessionVars().EnableRedactNew), job.err.Error())) + failedJobs = append(failedJobs, fmt.Sprintf("job:%s, error: %s", job.String(se.GetSessionVars().EnableRedactLog), job.err.Error())) } } if len(failedJobs) == 0 { @@ -324,7 +324,7 @@ func runJobs(ctx context.Context, jobs []job, stmt *ast.NonTransactionalDMLStmt, return nil, errors.Annotate(jobs[i].err, "Early return: error occurred in the first job. All jobs are canceled") } if jobs[i].err != nil && !se.GetSessionVars().NonTransactionalIgnoreError { - return nil, ErrNonTransactionalJobFailure.GenWithStackByArgs(jobs[i].jobID, len(jobs), jobs[i].start.String(), jobs[i].end.String(), jobs[i].String(se.GetSessionVars().EnableRedactNew), jobs[i].err.Error()) + return nil, ErrNonTransactionalJobFailure.GenWithStackByArgs(jobs[i].jobID, len(jobs), jobs[i].start.String(), jobs[i].end.String(), jobs[i].String(se.GetSessionVars().EnableRedactLog), jobs[i].err.Error()) } } return splitStmts, nil @@ -410,8 +410,8 @@ func doOneJob(ctx context.Context, job *job, totalJobCount int, options statemen job.sql = dmlSQL logutil.Logger(ctx).Info("start a Non-transactional DML", - zap.String("job", job.String(se.GetSessionVars().EnableRedactNew)), zap.Int("totalJobCount", totalJobCount)) - dmlSQLInLog := parser.Normalize(dmlSQL, se.GetSessionVars().EnableRedactNew) + zap.String("job", job.String(se.GetSessionVars().EnableRedactLog)), zap.Int("totalJobCount", totalJobCount)) + dmlSQLInLog := parser.Normalize(dmlSQL, se.GetSessionVars().EnableRedactLog) options.stmt.DMLStmt.SetText(nil, fmt.Sprintf("/* job %v/%v */ %s", job.jobID, totalJobCount, dmlSQL)) rs, err := se.ExecuteStmt(ctx, options.stmt.DMLStmt) diff --git a/pkg/session/session.go b/pkg/session/session.go index 7b63c4a1666cb..b9b242212ffb0 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -570,6 +570,9 @@ func (s *session) doCommit(ctx context.Context) error { if s.GetSessionVars().TxnCtx != nil { needCheckSchema = !s.GetSessionVars().TxnCtx.EnableMDL } + if s.txn.IsPipelined() && !s.GetSessionVars().TxnCtx.EnableMDL { + return errors.New("cannot commit pipelined transaction without Metadata Lock: MDL is OFF") + } s.txn.SetOption(kv.SchemaChecker, domain.NewSchemaChecker(domain.GetDomain(s), s.GetInfoSchema().SchemaMetaVersion(), physicalTableIDs, needCheckSchema)) s.txn.SetOption(kv.InfoSchema, s.sessionVars.TxnCtx.InfoSchema) s.txn.SetOption(kv.CommitHook, func(info string, _ error) { s.sessionVars.LastTxnInfo = info }) @@ -685,7 +688,8 @@ func (s *session) handleAssertionFailure(ctx context.Context, err error) error { assertionFailure.ExistingStartTs, assertionFailure.ExistingCommitTs, ) - if s.GetSessionVars().EnableRedactLog { + rmode := s.GetSessionVars().EnableRedactLog + if rmode == errors.RedactLogEnable { return newErr } @@ -727,7 +731,7 @@ func (s *session) handleAssertionFailure(ctx context.Context, err error) error { } if store, ok := s.store.(helper.Storage); ok { content := consistency.GetMvccByKey(store, key, decodeFunc) - logutil.Logger(ctx).Error("assertion failed", zap.String("message", newErr.Error()), zap.String("mvcc history", content)) + logutil.Logger(ctx).Error("assertion failed", zap.String("message", redact.String(rmode, newErr.Error())), zap.String("mvcc history", redact.String(rmode, content))) } return newErr } @@ -1173,8 +1177,8 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) { // We do not have to log the query every time. // We print the queries at the first try only. sql := sqlForLog(st.GetTextToLog(false)) - if sessVars.EnableRedactNew != "ON" { - sql += redact.String(sessVars.EnableRedactNew, sessVars.PlanCacheParams.String()) + if sessVars.EnableRedactLog != errors.RedactLogEnable { + sql += redact.String(sessVars.EnableRedactLog, sessVars.PlanCacheParams.String()) } logutil.Logger(ctx).Warn("retrying", zap.Int64("schemaVersion", schemaVersion), @@ -1527,7 +1531,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu TableIDs: s.sessionVars.StmtCtx.TableIDs, IndexNames: s.sessionVars.StmtCtx.IndexNames, MaxExecutionTime: maxExecutionTime, - RedactSQL: s.sessionVars.EnableRedactNew, + RedactSQL: s.sessionVars.EnableRedactLog, ResourceGroupName: s.sessionVars.StmtCtx.ResourceGroupName, SessionAlias: s.sessionVars.SessionAlias, } @@ -1669,7 +1673,7 @@ func (s *session) Parse(ctx context.Context, sql string) ([]ast.StmtNode, error) // Only print log message when this SQL is from the user. // Mute the warning for internal SQLs. if !s.sessionVars.InRestrictedSQL { - logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactNew, sql))) + logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactLog, sql))) s.sessionVars.StmtCtx.AppendError(err) } return nil, err @@ -1719,7 +1723,7 @@ func (s *session) ParseWithParams(ctx context.Context, sql string, args ...any) if err != nil { s.rollbackOnError(ctx) logSQL := sql[:min(500, len(sql))] - logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactNew, logSQL))) + logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactLog, logSQL))) return nil, util.SyntaxError(err) } durParse := time.Since(parseStartTime) @@ -2207,7 +2211,7 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex if !s.sessionVars.InRestrictedSQL { if !variable.ErrUnknownSystemVar.Equal(err) { sql := stmtNode.Text() - sql = parser.Normalize(sql, s.sessionVars.EnableRedactNew) + sql = parser.Normalize(sql, s.sessionVars.EnableRedactLog) logutil.Logger(ctx).Warn("compile SQL failed", zap.Error(err), zap.String("SQL", sql)) } @@ -3782,7 +3786,8 @@ func (s *session) PrepareTxnCtx(ctx context.Context) error { } txnMode := ast.Optimistic - if !s.sessionVars.IsAutocommit() || config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() { + if !s.sessionVars.IsAutocommit() || (config.GetGlobalConfig().PessimisticTxn. + PessimisticAutoCommit.Load() && !s.GetSessionVars().BulkDMLEnabled) { if s.sessionVars.TxnMode == ast.Pessimistic { txnMode = ast.Pessimistic } @@ -3818,7 +3823,7 @@ func (s *session) PrepareTSFuture(ctx context.Context, future oracle.Future, sco future: future, store: s.store, txnScope: scope, - pipelined: s.isPipelinedDML(), + pipelined: s.usePipelinedDmlOrWarn(), }) return nil } @@ -3934,10 +3939,10 @@ func logGeneralQuery(execStmt *executor.ExecStmt, s *session, isPrepared bool) { } query = executor.QueryReplacer.Replace(query) - if vars.EnableRedactNew != "ON" { - query += redact.String(vars.EnableRedactNew, vars.PlanCacheParams.String()) + if vars.EnableRedactLog != errors.RedactLogEnable { + query += redact.String(vars.EnableRedactLog, vars.PlanCacheParams.String()) } - logutil.BgLogger().Info("GENERAL_LOG", + logutil.GeneralLogger.Info("GENERAL_LOG", zap.Uint64("conn", vars.ConnectionID), zap.String("session_alias", vars.SessionAlias), zap.String("user", vars.User.LoginString()), @@ -4291,8 +4296,8 @@ func (s *session) NewStmtIndexUsageCollector() *indexusage.StmtIndexUsageCollect return indexusage.NewStmtIndexUsageCollector(s.idxUsageCollector) } -// isPipelinedDML returns the current statement can be executed as a pipelined DML. -func (s *session) isPipelinedDML() bool { +// usePipelinedDmlOrWarn returns the current statement can be executed as a pipelined DML. +func (s *session) usePipelinedDmlOrWarn() bool { if !s.sessionVars.BulkDMLEnabled { return false } @@ -4300,15 +4305,110 @@ func (s *session) isPipelinedDML() bool { if stmtCtx == nil { return false } - if !stmtCtx.InInsertStmt && !stmtCtx.InDeleteStmt && !stmtCtx.InUpdateStmt { - // not a DML + if stmtCtx.IsReadOnly { + return false + } + vars := s.GetSessionVars() + if !vars.TxnCtx.EnableMDL { + stmtCtx.AppendWarning( + errors.New( + "Pipelined DML can not be used without Metadata Lock. Fallback to standard mode", + ), + ) + return false + } + if (vars.BatchCommit || vars.BatchInsert || vars.BatchDelete) && vars.DMLBatchSize > 0 && variable.EnableBatchDML.Load() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used with the deprecated Batch DML. Fallback to standard mode")) + return false + } + if vars.BinlogClient != nil { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used with Binlog: BinlogClient != nil. Fallback to standard mode")) + return false + } + if !(stmtCtx.InInsertStmt || stmtCtx.InDeleteStmt || stmtCtx.InUpdateStmt) { + if !stmtCtx.IsReadOnly { + stmtCtx.AppendWarning(errors.New("Pipelined DML can only be used for auto-commit INSERT, REPLACE, UPDATE or DELETE. Fallback to standard mode")) + } return false } if s.isInternal() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used for internal SQL. Fallback to standard mode")) + return false + } + if vars.InTxn() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used in transaction. Fallback to standard mode")) + return false + } + if !vars.IsAutocommit() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can only be used in autocommit mode. Fallback to standard mode")) + return false + } + if s.GetSessionVars().ConstraintCheckInPlace { + // we enforce that pipelined DML must lazily check key. + return false + } + is, ok := s.GetDomainInfoSchema().(infoschema.InfoSchema) + if !ok { + stmtCtx.AppendWarning(errors.New("Pipelined DML failed to get latest InfoSchema. Fallback to standard mode")) return false } - return s.sessionVars.IsAutocommit() && !s.sessionVars.InTxn() && - !config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() && s.sessionVars.BinlogClient == nil + for _, t := range stmtCtx.Tables { + // get table schema from current infoschema + tbl, err := is.TableByName(model.NewCIStr(t.DB), model.NewCIStr(t.Table)) + if err != nil { + stmtCtx.AppendWarning(errors.New("Pipelined DML failed to get table schema. Fallback to standard mode")) + return false + } + if tbl.Meta().IsView() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used on view. Fallback to standard mode")) + return false + } + if tbl.Meta().IsSequence() { + stmtCtx.AppendWarning(errors.New("Pipelined DML can not be used on sequence. Fallback to standard mode")) + return false + } + if len(tbl.Meta().ForeignKeys) > 0 && vars.ForeignKeyChecks { + stmtCtx.AppendWarning( + errors.New( + "Pipelined DML can not be used on table with foreign keys when foreign_key_checks = ON. Fallback to standard mode", + ), + ) + return false + } + referredFKs := is.GetTableReferredForeignKeys(t.DB, t.Table) + if len(referredFKs) > 0 { + return false + } + if tbl.Meta().TempTableType != model.TempTableNone { + stmtCtx.AppendWarning( + errors.New( + "Pipelined DML can not be used on temporary tables. " + + "Fallback to standard mode", + ), + ) + return false + } + if tbl.Meta().TableCacheStatusType != model.TableCacheStatusDisable { + stmtCtx.AppendWarning( + errors.New( + "Pipelined DML can not be used on cached tables. " + + "Fallback to standard mode", + ), + ) + return false + } + } + + // tidb_dml_type=bulk will invalidate the config pessimistic-auto-commit. + // The behavior is as if the config is set to false. But we generate a warning for it. + if config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() { + stmtCtx.AppendWarning( + errors.New( + "pessimistic-auto-commit config is ignored in favor of Pipelined DML", + ), + ) + } + return true } // RemoveLockDDLJobs removes the DDL jobs which doesn't get the metadata lock from job2ver. diff --git a/pkg/session/txn.go b/pkg/session/txn.go index fbf7920a941bd..7f44220780fc6 100644 --- a/pkg/session/txn.go +++ b/pkg/session/txn.go @@ -113,7 +113,9 @@ func (txn *LazyTxn) initStmtBuf() { } buf := txn.Transaction.GetMemBuffer() txn.initCnt = buf.Len() - txn.stagingHandle = buf.Staging() + if !txn.IsPipelined() { + txn.stagingHandle = buf.Staging() + } } // countHint is estimated count of mutations. @@ -139,7 +141,9 @@ func (txn *LazyTxn) flushStmtBuf() { } } - buf.Release(txn.stagingHandle) + if !txn.IsPipelined() { + buf.Release(txn.stagingHandle) + } txn.initCnt = buf.Len() } @@ -148,7 +152,9 @@ func (txn *LazyTxn) cleanupStmtBuf() { return } buf := txn.Transaction.GetMemBuffer() - buf.Cleanup(txn.stagingHandle) + if !txn.IsPipelined() { + buf.Cleanup(txn.stagingHandle) + } txn.initCnt = buf.Len() txn.mu.Lock() @@ -320,7 +326,7 @@ func (txn *LazyTxn) changePendingToValid(ctx context.Context, sctx sessionctx.Co } func (txn *LazyTxn) changeToInvalid() { - if txn.stagingHandle != kv.InvalidStagingHandle { + if txn.stagingHandle != kv.InvalidStagingHandle && !txn.IsPipelined() { txn.Transaction.GetMemBuffer().Cleanup(txn.stagingHandle) } txn.stagingHandle = kv.InvalidStagingHandle diff --git a/pkg/session/txnmanager.go b/pkg/session/txnmanager.go index c28040c071057..262ac6a90b4da 100644 --- a/pkg/session/txnmanager.go +++ b/pkg/session/txnmanager.go @@ -208,7 +208,7 @@ func (m *txnManager) OnStmtStart(ctx context.Context, node ast.StmtNode) error { var sql string if node != nil { sql = node.OriginalText() - sql = parser.Normalize(sql, m.sctx.GetSessionVars().EnableRedactNew) + sql = parser.Normalize(sql, m.sctx.GetSessionVars().EnableRedactLog) } m.recordEvent(sql) return m.ctxProvider.OnStmtStart(ctx, m.stmtNode) @@ -305,6 +305,14 @@ func (m *txnManager) OnLocalTemporaryTableCreated() { } func (m *txnManager) AdviseWarmup() error { + if m.sctx.GetSessionVars().BulkDMLEnabled { + // We don't want to validate the feasibility of pipelined DML here. + // We'd like to check it later after optimization so that optimizer info can be used. + // And it does not make much sense to save such a little time for pipelined-dml as it's + // for bulk processing. + return nil + } + if m.ctxProvider != nil { return m.ctxProvider.AdviseWarmup() } diff --git a/pkg/sessionctx/sessionstates/BUILD.bazel b/pkg/sessionctx/sessionstates/BUILD.bazel index c24a4a626d58c..7f840701053a2 100644 --- a/pkg/sessionctx/sessionstates/BUILD.bazel +++ b/pkg/sessionctx/sessionstates/BUILD.bazel @@ -31,7 +31,7 @@ go_test( ], embed = [":sessionstates"], flaky = True, - shard_count = 16, + shard_count = 17, deps = [ "//pkg/config", "//pkg/errno", diff --git a/pkg/sessionctx/sessionstates/session_states_test.go b/pkg/sessionctx/sessionstates/session_states_test.go index f22e286796ae8..2caea78e5575b 100644 --- a/pkg/sessionctx/sessionstates/session_states_test.go +++ b/pkg/sessionctx/sessionstates/session_states_test.go @@ -1386,6 +1386,45 @@ func TestSQLBinding(t *testing.T) { } } +func TestSQLBindingCompatibility(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("create table test.t1(id int primary key, name varchar(10), key(name))") + + tests := []struct { + bindingStr string + expectedResult []string + }{ + // db is empty + { + bindingStr: "{\"bindings\": \"[{\\\\\"OriginalSQL\\\\\":\\\\\"select * from `test` . `t1`\\\\\",\\\\\"Db\\\\\":\\\\\"\\\\\",\\\\\"Bindings\\\\\":[{\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`name`)\\\\\",\\\\\"Status\\\\\":\\\\\"enabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}]}]\"}", + expectedResult: []string{"select * from `test` . `t1` SELECT * FROM `test`.`t1` USE INDEX (`name`) enabled 2024-03-18 16:38:14.270 2024-03-18 16:38:14.270 utf8mb4 utf8mb4_0900_ai_ci manual 4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0 "}, + }, + // db is not empty + { + bindingStr: "{\"bindings\": \"[{\\\\\"OriginalSQL\\\\\":\\\\\"select * from `t1`\\\\\",\\\\\"Db\\\\\":\\\\\"test\\\\\",\\\\\"Bindings\\\\\":[{\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`name`)\\\\\",\\\\\"Status\\\\\":\\\\\"enabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}]}]\"}", + expectedResult: []string{"select * from `t1` SELECT * FROM `test`.`t1` USE INDEX (`name`) test enabled 2024-03-18 16:38:14.270 2024-03-18 16:38:14.270 utf8mb4 utf8mb4_0900_ai_ci manual 4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0 "}, + }, + // 2 bindings in 2 arrays + { + bindingStr: "{\"bindings\": \"[{\\\\\"OriginalSQL\\\\\":\\\\\"select * from `t1`\\\\\",\\\\\"Db\\\\\":\\\\\"test\\\\\",\\\\\"Bindings\\\\\":[{\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`name`)\\\\\",\\\\\"Status\\\\\":\\\\\"enabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}]}, {\\\\\"OriginalSQL\\\\\":\\\\\"select * from `test` . `t1`\\\\\",\\\\\"Db\\\\\":\\\\\"\\\\\",\\\\\"Bindings\\\\\":[{\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`name`)\\\\\",\\\\\"Status\\\\\":\\\\\"enabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}]}]\"}", + expectedResult: []string{"select * from `t1` SELECT * FROM `test`.`t1` USE INDEX (`name`) test enabled 2024-03-18 16:38:14.270 2024-03-18 16:38:14.270 utf8mb4 utf8mb4_0900_ai_ci manual 4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0 ", "select * from `test` . `t1` SELECT * FROM `test`.`t1` USE INDEX (`name`) enabled 2024-03-18 16:38:14.270 2024-03-18 16:38:14.270 utf8mb4 utf8mb4_0900_ai_ci manual 4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0 "}, + }, + // 2 bindings in 1 array, one is enabled while another is disabled + { + bindingStr: "{\"bindings\": \"[{\\\\\"OriginalSQL\\\\\":\\\\\"select * from `t1`\\\\\",\\\\\"Db\\\\\":\\\\\"test\\\\\",\\\\\"Bindings\\\\\":[{\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`name`)\\\\\",\\\\\"Status\\\\\":\\\\\"enabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}, {\\\\\"BindSQL\\\\\":\\\\\"SELECT * FROM `test`.`t1` USE INDEX (`primary`)\\\\\",\\\\\"Status\\\\\":\\\\\"disabled\\\\\",\\\\\"CreateTime\\\\\":2279073240653628855,\\\\\"UpdateTime\\\\\":2279073240653628855,\\\\\"Source\\\\\":\\\\\"manual\\\\\",\\\\\"Charset\\\\\":\\\\\"utf8mb4\\\\\",\\\\\"Collation\\\\\":\\\\\"utf8mb4_0900_ai_ci\\\\\",\\\\\"SQLDigest\\\\\":\\\\\"4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0\\\\\",\\\\\"PlanDigest\\\\\":\\\\\"\\\\\"}]}]\"}", + expectedResult: []string{"select * from `t1` SELECT * FROM `test`.`t1` USE INDEX (`primary`) test disabled 2024-03-18 16:38:14.270 2024-03-18 16:38:14.270 utf8mb4 utf8mb4_0900_ai_ci manual 4ea0618129ffc6a7effbc0eff4bbcb41a7f5d4c53a6fa0b2e9be81c7010915b0 "}, + }, + } + + for _, test := range tests { + setSQL := fmt.Sprintf("set session_states '%s'", test.bindingStr) + tk := testkit.NewTestKit(t, store) + tk.MustExec(setSQL) + tk.MustQuery("show session bindings").Sort().Check(testkit.Rows(test.expectedResult...)) + } +} + func TestShowStateFail(t *testing.T) { store := testkit.CreateMockStore(t) sv := server.CreateMockServer(t, store) diff --git a/pkg/sessionctx/stmtctx/stmtctx.go b/pkg/sessionctx/stmtctx/stmtctx.go index 0c3ad9a2f1e97..fd2d9192c18e4 100644 --- a/pkg/sessionctx/stmtctx/stmtctx.go +++ b/pkg/sessionctx/stmtctx/stmtctx.go @@ -359,7 +359,7 @@ type StatementContext struct { // Timeout to wait for sync-load Timeout time.Duration // NeededItems stores the columns/indices whose stats are needed for planner. - NeededItems []model.TableItemID + NeededItems []model.StatsLoadItem // ResultCh to receive stats loading results ResultCh chan StatsLoadResult // LoadStartTime is to record the load start time to calculate latency @@ -1255,6 +1255,7 @@ type UsedStatsInfoForTable struct { ModifyCount int64 ColumnStatsLoadStatus map[int64]string IndexStatsLoadStatus map[int64]string + ColAndIdxStatus any } // FormatForExplain format the content in the format expected to be printed in the execution plan. diff --git a/pkg/sessionctx/variable/noop.go b/pkg/sessionctx/variable/noop.go index 098439c4262fb..8fc1e1b680b60 100644 --- a/pkg/sessionctx/variable/noop.go +++ b/pkg/sessionctx/variable/noop.go @@ -46,7 +46,12 @@ var noopSysVars = []*SysVar{ }}, {Scope: ScopeGlobal, Name: ConnectTimeout, Value: "10", Type: TypeUnsigned, MinValue: 2, MaxValue: secondsPerYear}, {Scope: ScopeGlobal | ScopeSession, Name: QueryCacheWlockInvalidate, Value: Off, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: "sql_buffer_result", Value: Off, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "sql_buffer_result", + Value: Off, + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: MyISAMUseMmap, Value: Off, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeGlobal, Name: "gtid_mode", Value: Off, Type: TypeBool}, {Scope: ScopeGlobal, Name: FlushTime, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: secondsPerYear}, @@ -88,7 +93,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "have_query_cache", Value: "YES"}, {Scope: ScopeGlobal, Name: "innodb_flush_log_at_timeout", Value: "1"}, {Scope: ScopeGlobal, Name: "innodb_max_undo_log_size", Value: ""}, - {Scope: ScopeGlobal | ScopeSession, Name: "range_alloc_block_size", Value: "4096", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "range_alloc_block_size", + Value: "4096", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "have_rtree_keys", Value: "YES"}, {Scope: ScopeGlobal, Name: "innodb_old_blocks_pct", Value: "37"}, {Scope: ScopeGlobal, Name: "innodb_file_format", Value: "Barracuda", Type: TypeEnum, PossibleValues: []string{"Antelope", "Barracuda"}}, @@ -140,7 +150,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "log_warnings", Value: "1"}, {Scope: ScopeGlobal | ScopeSession, Name: InnodbStrictMode, Value: On, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeGlobal, Name: "innodb_rollback_segments", Value: "128"}, - {Scope: ScopeGlobal | ScopeSession, Name: "join_buffer_size", Value: "262144", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "join_buffer_size", + Value: "262144", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "innodb_mirrored_log_groups", Value: "1"}, {Scope: ScopeGlobal, Name: "max_binlog_size", Value: "1073741824"}, {Scope: ScopeGlobal, Name: "concurrent_insert", Value: "AUTO"}, @@ -152,7 +167,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "innodb_file_format_check", Value: "1"}, {Scope: ScopeNone, Name: "myisam_mmap_size", Value: "18446744073709551615"}, {Scope: ScopeNone, Name: "innodb_buffer_pool_instances", Value: "8"}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_length_for_sort_data", Value: "1024", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_length_for_sort_data", + Value: "1024", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: CharacterSetSystem, Value: "utf8"}, {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetFilesystem, Value: "binary", Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetFilesystem) @@ -176,7 +196,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "innodb_undo_tablespaces", Value: "0"}, {Scope: ScopeGlobal, Name: InnodbStatusOutputLocks, Value: Off, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeNone, Name: "performance_schema_accounts_size", Value: "100"}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_error_count", Value: "64", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_error_count", + Value: "64", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "max_write_lock_count", Value: "18446744073709551615"}, {Scope: ScopeNone, Name: "performance_schema_max_socket_instances", Value: "322"}, {Scope: ScopeNone, Name: "performance_schema_max_table_instances", Value: "12500"}, @@ -190,43 +215,68 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "ft_stopword_file", Value: "(built-in)"}, {Scope: ScopeGlobal, Name: "innodb_max_dirty_pages_pct_lwm", Value: "0"}, {Scope: ScopeGlobal, Name: LogQueriesNotUsingIndexes, Value: Off, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_heap_table_size", Value: "16777216", IsHintUpdatableVerfied: true}, - {Scope: ScopeGlobal | ScopeSession, Name: "tmp_table_size", Value: "16777216", Type: TypeUnsigned, MinValue: 1024, MaxValue: math.MaxUint64, IsHintUpdatableVerfied: true}, - {Scope: ScopeGlobal | ScopeSession, Name: "div_precision_increment", Value: "4", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_heap_table_size", + Value: "16777216", + IsHintUpdatableVerified: true, + }, + { + Scope: ScopeGlobal | ScopeSession, + Name: "tmp_table_size", + Value: "16777216", + Type: TypeUnsigned, + MinValue: 1024, + MaxValue: math.MaxUint64, + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "innodb_lru_scan_depth", Value: "1024"}, {Scope: ScopeGlobal, Name: "innodb_purge_rseg_truncate_frequency", Value: ""}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLAutoIsNull, Value: Off, Type: TypeBool, IsHintUpdatableVerfied: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - // checkSQLAutoIsNull requires TiDBEnableNoopFuncs != OFF for the same scope otherwise an error will be returned. - // See also https://github.com/pingcap/tidb/issues/28230 - errMsg := ErrFunctionsNoopImpl.FastGenByArgs("sql_auto_is_null") - if TiDBOptOn(normalizedValue) { - if scope == ScopeSession && vars.NoopFuncsMode != OnInt { - if vars.NoopFuncsMode == OffInt { - return Off, errors.Trace(errMsg) - } - vars.StmtCtx.AppendWarning(errMsg) - } - if scope == ScopeGlobal { - val, err := vars.GlobalVarsAccessor.GetGlobalSysVar(TiDBEnableNoopFuncs) - if err != nil { - return originalValue, errUnknownSystemVariable.GenWithStackByArgs(TiDBEnableNoopFuncs) - } - if val == Off { - return Off, errors.Trace(errMsg) - } - if val == Warn { + { + Scope: ScopeGlobal | ScopeSession, + Name: SQLAutoIsNull, + Value: Off, + Type: TypeBool, + IsHintUpdatableVerified: true, + Validation: func( + vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag, + ) (string, error) { + // checkSQLAutoIsNull requires TiDBEnableNoopFuncs != OFF for the same scope otherwise an error will be returned. + // See also https://github.com/pingcap/tidb/issues/28230 + errMsg := ErrFunctionsNoopImpl.FastGenByArgs("sql_auto_is_null") + if TiDBOptOn(normalizedValue) { + if scope == ScopeSession && vars.NoopFuncsMode != OnInt { + if vars.NoopFuncsMode == OffInt { + return Off, errors.Trace(errMsg) + } vars.StmtCtx.AppendWarning(errMsg) } + if scope == ScopeGlobal { + val, err := vars.GlobalVarsAccessor.GetGlobalSysVar(TiDBEnableNoopFuncs) + if err != nil { + return originalValue, errUnknownSystemVariable.GenWithStackByArgs(TiDBEnableNoopFuncs) + } + if val == Off { + return Off, errors.Trace(errMsg) + } + if val == Warn { + vars.StmtCtx.AppendWarning(errMsg) + } + } } - } - return normalizedValue, nil - }}, + return normalizedValue, nil + }}, {Scope: ScopeNone, Name: "innodb_api_enable_binlog", Value: "0"}, {Scope: ScopeGlobal | ScopeSession, Name: "innodb_ft_user_stopword_table", Value: ""}, {Scope: ScopeNone, Name: "server_id_bits", Value: "32"}, {Scope: ScopeGlobal, Name: "innodb_log_checksum_algorithm", Value: ""}, {Scope: ScopeNone, Name: "innodb_buffer_pool_load_at_startup", Value: "1"}, - {Scope: ScopeGlobal | ScopeSession, Name: "sort_buffer_size", Value: "262144", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "sort_buffer_size", + Value: "262144", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "innodb_flush_neighbors", Value: "1"}, {Scope: ScopeNone, Name: "innodb_use_sys_malloc", Value: "1"}, {Scope: ScopeNone, Name: "performance_schema_max_socket_classes", Value: "10"}, @@ -239,13 +289,31 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "myisam_data_pointer_size", Value: "6"}, {Scope: ScopeGlobal, Name: "ndb_optimization_delay", Value: ""}, {Scope: ScopeGlobal, Name: "innodb_ft_num_word_optimize", Value: "2000"}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_join_size", Value: "18446744073709551615", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_join_size", + Value: "18446744073709551615", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: CoreFile, Value: Off, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_seeks_for_key", Value: "18446744073709551615", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_seeks_for_key", + Value: "18446744073709551615", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "innodb_log_buffer_size", Value: "8388608"}, {Scope: ScopeGlobal, Name: "delayed_insert_timeout", Value: "300"}, {Scope: ScopeGlobal, Name: "max_relay_log_size", Value: "0"}, - {Scope: ScopeGlobal | ScopeSession, Name: MaxSortLength, Value: "1024", Type: TypeUnsigned, MinValue: 4, MaxValue: 8388608, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: MaxSortLength, + Value: "1024", + Type: TypeUnsigned, + MinValue: 4, + MaxValue: 8388608, + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "metadata_locks_hash_instances", Value: "8"}, {Scope: ScopeGlobal, Name: "ndb_eventbuffer_free_percent", Value: ""}, {Scope: ScopeNone, Name: "large_files_support", Value: "1"}, @@ -294,17 +362,33 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: InnodbStatsAutoRecalc, Value: "1"}, // lc_messages cannot be read_only, see https://github.com/pingcap/tidb/issues/38231. {Scope: ScopeGlobal | ScopeSession, Name: "lc_messages", Value: "en_US"}, - {Scope: ScopeGlobal | ScopeSession, Name: "bulk_insert_buffer_size", Value: "8388608", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "bulk_insert_buffer_size", + Value: "8388608", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal | ScopeSession, Name: BinlogDirectNonTransactionalUpdates, Value: Off, Type: TypeBool}, {Scope: ScopeGlobal, Name: "innodb_change_buffering", Value: "all"}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLBigSelects, Value: On, Type: TypeBool, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: SQLBigSelects, + Value: On, + Type: TypeBool, + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "innodb_max_purge_lag_delay", Value: "0"}, {Scope: ScopeGlobal | ScopeSession, Name: "session_track_schema", Value: ""}, {Scope: ScopeGlobal, Name: "innodb_io_capacity_max", Value: "2000"}, {Scope: ScopeGlobal, Name: "innodb_autoextend_increment", Value: "64"}, {Scope: ScopeGlobal | ScopeSession, Name: "binlog_format", Value: "STATEMENT"}, {Scope: ScopeGlobal | ScopeSession, Name: "optimizer_trace", Value: "enabled=off,one_line=off"}, - {Scope: ScopeGlobal | ScopeSession, Name: "read_rnd_buffer_size", Value: "262144", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "read_rnd_buffer_size", + Value: "262144", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal | ScopeSession, Name: NetWriteTimeout, Value: "60"}, {Scope: ScopeGlobal, Name: InnodbBufferPoolLoadAbort, Value: Off, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeGlobal | ScopeSession, Name: "transaction_prealloc_size", Value: "4096"}, @@ -327,7 +411,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "table_open_cache_instances", Value: "1"}, {Scope: ScopeGlobal, Name: InnodbStatsPersistent, Value: On, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeGlobal | ScopeSession, Name: "session_track_state_change", Value: ""}, - {Scope: ScopeNone, Name: OptimizerSwitch, Value: "index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,engine_condition_pushdown=on,index_condition_pushdown=on,mrr=on,mrr_cost_based=on,block_nested_loop=on,batched_key_access=off,materialization=on,semijoin=on,loosescan=on,firstmatch=on,subquery_materialization_cost_based=on,use_index_extensions=on", IsHintUpdatableVerfied: true}, + { + Scope: ScopeNone, + Name: OptimizerSwitch, + Value: "index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,engine_condition_pushdown=on,index_condition_pushdown=on,mrr=on,mrr_cost_based=on,block_nested_loop=on,batched_key_access=off,materialization=on,semijoin=on,loosescan=on,firstmatch=on,subquery_materialization_cost_based=on,use_index_extensions=on", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "delayed_queue_size", Value: "1000"}, {Scope: ScopeNone, Name: "innodb_read_only", Value: "0"}, {Scope: ScopeNone, Name: "datetime_format", Value: "%Y-%m-%d %H:%i:%s"}, @@ -359,7 +448,13 @@ var noopSysVars = []*SysVar{ }}, {Scope: ScopeNone, Name: "max_tmp_tables", Value: "32"}, {Scope: ScopeGlobal, Name: InnodbRandomReadAhead, Value: Off, Type: TypeBool, AutoConvertNegativeBool: true}, - {Scope: ScopeGlobal | ScopeSession, Name: UniqueChecks, Value: On, Type: TypeBool, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: UniqueChecks, + Value: On, + Type: TypeBool, + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "internal_tmp_disk_storage_engine", Value: ""}, {Scope: ScopeGlobal | ScopeSession, Name: "myisam_repair_threads", Value: "1"}, {Scope: ScopeGlobal, Name: "ndb_eventbuffer_max_alloc", Value: ""}, @@ -369,7 +464,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "gtid_purged", Value: ""}, {Scope: ScopeGlobal, Name: "max_binlog_stmt_cache_size", Value: "18446744073709547520"}, {Scope: ScopeGlobal | ScopeSession, Name: "lock_wait_timeout", Value: "31536000"}, - {Scope: ScopeGlobal | ScopeSession, Name: "read_buffer_size", Value: "131072", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "read_buffer_size", + Value: "131072", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "innodb_read_io_threads", Value: "4"}, {Scope: ScopeGlobal | ScopeSession, Name: MaxSpRecursionDepth, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: 255}, {Scope: ScopeNone, Name: "ignore_builtin_innodb", Value: "0"}, @@ -389,9 +489,24 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "table_open_cache", Value: "2000"}, {Scope: ScopeNone, Name: "performance_schema_events_stages_history_long_size", Value: "10000"}, {Scope: ScopeSession, Name: "insert_id", Value: ""}, - {Scope: ScopeGlobal | ScopeSession, Name: "default_tmp_storage_engine", Value: "InnoDB", IsHintUpdatableVerfied: true}, - {Scope: ScopeGlobal | ScopeSession, Name: "optimizer_search_depth", Value: "62", IsHintUpdatableVerfied: true}, - {Scope: ScopeGlobal | ScopeSession, Name: "max_points_in_geometry", Value: "65536", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "default_tmp_storage_engine", + Value: "InnoDB", + IsHintUpdatableVerified: true, + }, + { + Scope: ScopeGlobal | ScopeSession, + Name: "optimizer_search_depth", + Value: "62", + IsHintUpdatableVerified: true, + }, + { + Scope: ScopeGlobal | ScopeSession, + Name: "max_points_in_geometry", + Value: "65536", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: "innodb_stats_sample_pages", Value: "8"}, {Scope: ScopeGlobal | ScopeSession, Name: "profiling_history_size", Value: "15"}, {Scope: ScopeNone, Name: "have_symlink", Value: "YES"}, @@ -415,8 +530,18 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "innodb_flush_log_at_trx_commit", Value: "1"}, {Scope: ScopeGlobal, Name: "rewriter_enabled", Value: ""}, {Scope: ScopeGlobal, Name: "query_cache_min_res_unit", Value: "4096"}, - {Scope: ScopeGlobal | ScopeSession, Name: "updatable_views_with_limit", Value: "YES", IsHintUpdatableVerfied: true}, - {Scope: ScopeGlobal | ScopeSession, Name: "optimizer_prune_level", Value: "1", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "updatable_views_with_limit", + Value: "YES", + IsHintUpdatableVerified: true, + }, + { + Scope: ScopeGlobal | ScopeSession, + Name: "optimizer_prune_level", + Value: "1", + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal | ScopeSession, Name: "completion_type", Value: "NO_CHAIN"}, {Scope: ScopeGlobal, Name: "binlog_checksum", Value: "CRC32"}, {Scope: ScopeNone, Name: "report_port", Value: "3306"}, @@ -457,7 +582,12 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "performance_schema_max_cond_instances", Value: "3504"}, {Scope: ScopeGlobal, Name: "delayed_insert_limit", Value: "100"}, {Scope: ScopeGlobal, Name: Flush, Value: Off, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: "eq_range_index_dive_limit", Value: "200", IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: "eq_range_index_dive_limit", + Value: "200", + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "performance_schema_events_stages_history_size", Value: "10"}, {Scope: ScopeGlobal | ScopeSession, Name: "ndb_join_pushdown", Value: ""}, {Scope: ScopeNone, Name: "performance_schema_max_thread_instances", Value: "402"}, @@ -473,13 +603,25 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "innodb_undo_directory", Value: "."}, {Scope: ScopeNone, Name: "bind_address", Value: "*"}, {Scope: ScopeGlobal, Name: "innodb_sync_spin_loops", Value: "30"}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLSafeUpdates, Value: Off, Type: TypeBool, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: SQLSafeUpdates, + Value: Off, + Type: TypeBool, + IsHintUpdatableVerified: true, + }, {Scope: ScopeNone, Name: "tmpdir", Value: "/var/tmp/"}, {Scope: ScopeGlobal, Name: "innodb_thread_concurrency", Value: "0"}, {Scope: ScopeGlobal, Name: "innodb_buffer_pool_dump_pct", Value: ""}, {Scope: ScopeGlobal | ScopeSession, Name: "lc_time_names", Value: "en_US", ReadOnly: true}, {Scope: ScopeGlobal | ScopeSession, Name: "max_statement_time", Value: ""}, - {Scope: ScopeGlobal | ScopeSession, Name: EndMarkersInJSON, Value: Off, Type: TypeBool, IsHintUpdatableVerfied: true}, + { + Scope: ScopeGlobal | ScopeSession, + Name: EndMarkersInJSON, + Value: Off, + Type: TypeBool, + IsHintUpdatableVerified: true, + }, {Scope: ScopeGlobal, Name: AvoidTemporalUpgrade, Value: Off, Type: TypeBool}, {Scope: ScopeGlobal, Name: "key_cache_age_threshold", Value: "300"}, {Scope: ScopeGlobal, Name: InnodbStatusOutput, Value: Off, Type: TypeBool, AutoConvertNegativeBool: true}, @@ -490,6 +632,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: ThreadPoolSize, Value: "16", Type: TypeUnsigned, MinValue: 1, MaxValue: 64}, {Scope: ScopeNone, Name: "lower_case_file_system", Value: "1"}, {Scope: ScopeNone, Name: LowerCaseTableNames, Value: "2"}, + {Scope: ScopeGlobal, Name: TiDBSchemaCacheSize, Value: "0"}, // for compatibility purpose, we should leave them alone. // TODO: Follow the Terminology Updates of MySQL after their changes arrived. diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 3726df4574930..f0a64fda8011f 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1067,6 +1067,9 @@ type SessionVars struct { // See https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_execution_time MaxExecutionTime uint64 + // LoadBindingTimeout is the timeout for loading the bind info. + LoadBindingTimeout uint64 + // TiKVClientReadTimeout is the timeout for readonly kv request in milliseconds, 0 means using default value // See https://github.com/pingcap/tidb/blob/7105505a78fc886c33258caa5813baf197b15247/docs/design/2023-06-30-configurable-kv-timeout.md?plain=1#L14-L15 TiKVClientReadTimeout uint64 @@ -1202,10 +1205,8 @@ type SessionVars struct { // EnableParallelApply indicates that thether to use parallel apply. EnableParallelApply bool - // EnableRedactLog indicates that whether redact log. - EnableRedactLog bool - // EnableRedactNew indicates that whether redact log. Possible values are 'OFF', 'ON', 'MARKER'. - EnableRedactNew string + // EnableRedactLog indicates that whether redact log. Possible values are 'OFF', 'ON', 'MARKER'. + EnableRedactLog string // ShardAllocateStep indicates the max size of continuous rowid shard in one transaction. ShardAllocateStep int64 @@ -1573,12 +1574,13 @@ type SessionVars struct { CompressionAlgorithm int CompressionLevel int - // EnableParallelSort indicates whether use parallel sort. Default is false. - EnableParallelSort bool - // TxnEntrySizeLimit indicates indicates the max size of a entry in membuf. The default limit (from config) will be // overwritten if this value is not 0. TxnEntrySizeLimit uint64 + + // DivPrecisionIncrement indicates the number of digits by which to increase the scale of the result + // of division operations performed with the / operator. + DivPrecisionIncrement int } // GetOptimizerFixControlMap returns the specified value of the optimizer fix control. @@ -2621,6 +2623,11 @@ func (s *SessionVars) GetPrevStmtDigest() string { return s.prevStmtDigest } +// GetDivPrecisionIncrement returns the specified value of DivPrecisionIncrement. +func (s *SessionVars) GetDivPrecisionIncrement() int { + return s.DivPrecisionIncrement +} + // LazyCheckKeyNotExists returns if we can lazy check key not exists. func (s *SessionVars) LazyCheckKeyNotExists() bool { return s.PresumeKeyNotExists || (s.TxnCtx != nil && s.TxnCtx.IsPessimistic && s.StmtCtx.ErrGroupLevel(errctx.ErrGroupDupKey) == errctx.LevelError) diff --git a/pkg/sessionctx/variable/setvar_affect.go b/pkg/sessionctx/variable/setvar_affect.go index 866dd611fcf33..6437e75249dd0 100644 --- a/pkg/sessionctx/variable/setvar_affect.go +++ b/pkg/sessionctx/variable/setvar_affect.go @@ -117,7 +117,7 @@ var isHintUpdatableVerified = map[string]struct{}{ func setHintUpdatable(vars []*SysVar) { for _, v := range vars { if _, ok := isHintUpdatableVerified[v.Name]; ok { - v.IsHintUpdatableVerfied = true + v.IsHintUpdatableVerified = true } } } diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 090324c1d9cc3..190a275b8ff0c 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -1243,10 +1243,16 @@ var defaultSysVars = []*SysVar{ s.EnableNonPreparedPlanCacheForDML = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptEnableFuzzyBinding, Value: BoolToOnOff(false), Type: TypeBool, IsHintUpdatableVerfied: true, SetSession: func(s *SessionVars, val string) error { - s.EnableFuzzyBinding = TiDBOptOn(val) - return nil - }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: TiDBOptEnableFuzzyBinding, + Value: BoolToOnOff(false), + Type: TypeBool, + IsHintUpdatableVerified: true, + SetSession: func(s *SessionVars, val string) error { + s.EnableFuzzyBinding = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBNonPreparedPlanCacheSize, Value: strconv.FormatUint(uint64(DefTiDBNonPreparedPlanCacheSize), 10), Type: TypeUnsigned, MinValue: 1, MaxValue: 100000, SetSession: func(s *SessionVars, val string) error { uVal, err := strconv.ParseUint(val, 10, 64) if err == nil { @@ -1462,34 +1468,70 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: DefaultWeekFormat, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: 7}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLModeVar, Value: mysql.DefaultSQLMode, IsHintUpdatableVerfied: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - // Ensure the SQL mode parses - normalizedValue = mysql.FormatSQLModeStr(normalizedValue) - if _, err := mysql.GetSQLMode(normalizedValue); err != nil { - return originalValue, err - } - return normalizedValue, nil - }, SetSession: func(s *SessionVars, val string) error { - val = mysql.FormatSQLModeStr(val) - // Modes is a list of different modes separated by commas. - sqlMode, err := mysql.GetSQLMode(val) - if err != nil { - return errors.Trace(err) - } - s.SQLMode = sqlMode - s.SetStatusFlag(mysql.ServerStatusNoBackslashEscaped, sqlMode.HasNoBackslashEscapesMode()) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: MaxExecutionTime, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, IsHintUpdatableVerfied: true, SetSession: func(s *SessionVars, val string) error { - timeoutMS := tidbOptPositiveInt32(val, 0) - s.MaxExecutionTime = uint64(timeoutMS) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiKVClientReadTimeout, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, IsHintUpdatableVerfied: true, SetSession: func(s *SessionVars, val string) error { - timeoutMS := tidbOptPositiveInt32(val, 0) - s.TiKVClientReadTimeout = uint64(timeoutMS) - return nil - }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: SQLModeVar, + Value: mysql.DefaultSQLMode, + IsHintUpdatableVerified: true, + Validation: func( + vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag, + ) (string, error) { + // Ensure the SQL mode parses + normalizedValue = mysql.FormatSQLModeStr(normalizedValue) + if _, err := mysql.GetSQLMode(normalizedValue); err != nil { + return originalValue, err + } + return normalizedValue, nil + }, SetSession: func(s *SessionVars, val string) error { + val = mysql.FormatSQLModeStr(val) + // Modes is a list of different modes separated by commas. + sqlMode, err := mysql.GetSQLMode(val) + if err != nil { + return errors.Trace(err) + } + s.SQLMode = sqlMode + s.SetStatusFlag(mysql.ServerStatusNoBackslashEscaped, sqlMode.HasNoBackslashEscapesMode()) + return nil + }}, + { + Scope: ScopeGlobal, + Name: TiDBLoadBindingTimeout, + Value: "200", + Type: TypeUnsigned, + MinValue: 0, + MaxValue: math.MaxInt32, + IsHintUpdatableVerified: false, + SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error { + timeoutMS := tidbOptPositiveInt32(s, 0) + vars.LoadBindingTimeout = uint64(timeoutMS) + return nil + }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: MaxExecutionTime, + Value: "0", + Type: TypeUnsigned, + MinValue: 0, + MaxValue: math.MaxInt32, + IsHintUpdatableVerified: true, + SetSession: func(s *SessionVars, val string) error { + timeoutMS := tidbOptPositiveInt32(val, 0) + s.MaxExecutionTime = uint64(timeoutMS) + return nil + }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: TiKVClientReadTimeout, + Value: "0", + Type: TypeUnsigned, + MinValue: 0, + MaxValue: math.MaxInt32, + IsHintUpdatableVerified: true, + SetSession: func(s *SessionVars, val string) error { + timeoutMS := tidbOptPositiveInt32(val, 0) + s.TiKVClientReadTimeout = uint64(timeoutMS) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: CollationServer, Value: mysql.DefaultCollationName, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCollation(vars, normalizedValue, originalValue, scope) }, SetSession: func(s *SessionVars, val string) error { @@ -1509,20 +1551,28 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: SQLLogBin, Value: On, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: TimeZone, Value: "SYSTEM", IsHintUpdatableVerfied: true, Validation: func(varErrFunctionsNoopImpls *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - if strings.EqualFold(normalizedValue, "SYSTEM") { - return "SYSTEM", nil - } - _, err := timeutil.ParseTimeZone(normalizedValue) - return normalizedValue, err - }, SetSession: func(s *SessionVars, val string) error { - tz, err := timeutil.ParseTimeZone(val) - if err != nil { - return err - } - s.TimeZone = tz - return nil - }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: TimeZone, + Value: "SYSTEM", + IsHintUpdatableVerified: true, + Validation: func( + varErrFunctionsNoopImpls *SessionVars, normalizedValue string, originalValue string, + scope ScopeFlag, + ) (string, error) { + if strings.EqualFold(normalizedValue, "SYSTEM") { + return "SYSTEM", nil + } + _, err := timeutil.ParseTimeZone(normalizedValue) + return normalizedValue, err + }, SetSession: func(s *SessionVars, val string) error { + tz, err := timeutil.ParseTimeZone(val) + if err != nil { + return err + } + s.TimeZone = tz + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: ForeignKeyChecks, Value: BoolToOnOff(DefTiDBForeignKeyChecks), Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if TiDBOptOn(normalizedValue) { vars.ForeignKeyChecks = true @@ -1607,21 +1657,31 @@ var defaultSysVars = []*SysVar{ s.LockWaitTimeout = lockWaitSec * 1000 return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: GroupConcatMaxLen, Value: "1024", IsHintUpdatableVerfied: true, Type: TypeUnsigned, MinValue: 4, MaxValue: math.MaxUint64, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - // https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_group_concat_max_len - // Minimum Value 4 - // Maximum Value (64-bit platforms) 18446744073709551615 - // Maximum Value (32-bit platforms) 4294967295 - if mathutil.IntBits == 32 { - if val, err := strconv.ParseUint(normalizedValue, 10, 64); err == nil { - if val > uint64(math.MaxUint32) { - vars.StmtCtx.AppendWarning(ErrTruncatedWrongValue.FastGenByArgs(GroupConcatMaxLen, originalValue)) - return strconv.FormatInt(int64(math.MaxUint32), 10), nil + { + Scope: ScopeGlobal | ScopeSession, + Name: GroupConcatMaxLen, + Value: "1024", + IsHintUpdatableVerified: true, + Type: TypeUnsigned, + MinValue: 4, + MaxValue: math.MaxUint64, + Validation: func( + vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag, + ) (string, error) { + // https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_group_concat_max_len + // Minimum Value 4 + // Maximum Value (64-bit platforms) 18446744073709551615 + // Maximum Value (32-bit platforms) 4294967295 + if mathutil.IntBits == 32 { + if val, err := strconv.ParseUint(normalizedValue, 10, 64); err == nil { + if val > uint64(math.MaxUint32) { + vars.StmtCtx.AppendWarning(ErrTruncatedWrongValue.FastGenByArgs(GroupConcatMaxLen, originalValue)) + return strconv.FormatInt(int64(math.MaxUint32), 10), nil + } } } - } - return normalizedValue, nil - }}, + return normalizedValue, nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetConnection, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetConnection) }, SetSession: func(s *SessionVars, val string) error { @@ -1668,10 +1728,16 @@ var defaultSysVars = []*SysVar{ return nil }, }, - {Scope: ScopeGlobal | ScopeSession, Name: WindowingUseHighPrecision, Value: On, Type: TypeBool, IsHintUpdatableVerfied: true, SetSession: func(s *SessionVars, val string) error { - s.WindowingUseHighPrecision = TiDBOptOn(val) - return nil - }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: WindowingUseHighPrecision, + Value: On, + Type: TypeBool, + IsHintUpdatableVerified: true, + SetSession: func(s *SessionVars, val string) error { + s.WindowingUseHighPrecision = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: BlockEncryptionMode, Value: "aes-128-ecb", Type: TypeEnum, PossibleValues: []string{"aes-128-ecb", "aes-192-ecb", "aes-256-ecb", "aes-128-cbc", "aes-192-cbc", "aes-256-cbc", "aes-128-ofb", "aes-192-ofb", "aes-256-ofb", "aes-128-cfb", "aes-192-cfb", "aes-256-cfb"}}, /* TiDB specific variables */ {Scope: ScopeGlobal | ScopeSession, Name: TiDBAllowMPPExecution, Type: TypeBool, Value: BoolToOnOff(DefTiDBAllowMPPExecution), Depended: true, SetSession: func(s *SessionVars, val string) error { @@ -1732,10 +1798,16 @@ var defaultSysVars = []*SysVar{ s.SetAllowInSubqToJoinAndAgg(TiDBOptOn(val)) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptPreferRangeScan, Value: BoolToOnOff(DefOptPreferRangeScan), Type: TypeBool, IsHintUpdatableVerfied: true, SetSession: func(s *SessionVars, val string) error { - s.SetAllowPreferRangeScan(TiDBOptOn(val)) - return nil - }}, + { + Scope: ScopeGlobal | ScopeSession, + Name: TiDBOptPreferRangeScan, + Value: BoolToOnOff(DefOptPreferRangeScan), + Type: TypeBool, + IsHintUpdatableVerified: true, + SetSession: func(s *SessionVars, val string) error { + s.SetAllowPreferRangeScan(TiDBOptOn(val)) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptLimitPushDownThreshold, Value: strconv.Itoa(DefOptLimitPushDownThreshold), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.LimitPushDownThreshold = TidbOptInt64(val, DefOptLimitPushDownThreshold) return nil @@ -2152,8 +2224,7 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: DefTiDBRedactLog, Type: TypeEnum, PossibleValues: []string{Off, On, Marker}, SetSession: func(s *SessionVars, val string) error { - s.EnableRedactLog = val != Off - s.EnableRedactNew = val + s.EnableRedactLog = val errors.RedactLogEnabled.Store(val) return nil }}, @@ -2711,7 +2782,12 @@ var defaultSysVars = []*SysVar{ s.EnableMPPSharedCTEExecution = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptFixControl, Value: "", Type: TypeStr, IsHintUpdatableVerfied: true, + { + Scope: ScopeGlobal | ScopeSession, + Name: TiDBOptFixControl, + Value: "", + Type: TypeStr, + IsHintUpdatableVerified: true, SetGlobal: func(ctx context.Context, vars *SessionVars, val string) error { // validation logic for setting global // we don't put this in Validation to avoid repeating the checking logic for setting session. @@ -2979,14 +3055,6 @@ var defaultSysVars = []*SysVar{ }, GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { return BoolToOnOff(EnableCheckConstraint.Load()), nil }}, - {Scope: ScopeGlobal, Name: TiDBSchemaCacheSize, Value: strconv.Itoa(DefTiDBSchemaCacheSize), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt32, SetGlobal: func(ctx context.Context, vars *SessionVars, val string) error { - // It does not take effect immediately, but within a ddl lease, infoschema reload would cause the v2 to be used. - SchemaCacheSize.Store(TidbOptInt64(val, DefTiDBSchemaCacheSize)) - return nil - }, GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { - val := SchemaCacheSize.Load() - return strconv.FormatInt(val, 10), nil - }}, {Scope: ScopeSession, Name: TiDBSessionAlias, Value: "", Type: TypeStr, Validation: func(s *SessionVars, normalizedValue string, originalValue string, _ ScopeFlag) (string, error) { chars := []rune(normalizedValue) @@ -3061,9 +3129,9 @@ var defaultSysVars = []*SysVar{ s.IdleTransactionTimeout = tidbOptPositiveInt32(val, DefTiDBIdleTransactionTimeout) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableParallelSort, Value: BoolToOnOff(DefEnableParallelSort), Type: TypeBool, - SetSession: func(vars *SessionVars, s string) error { - vars.EnableParallelSort = TiDBOptOn(s) + {Scope: ScopeGlobal | ScopeSession, Name: DivPrecisionIncrement, Value: strconv.Itoa(DefDivPrecisionIncrement), Type: TypeUnsigned, MinValue: 0, MaxValue: 30, + SetSession: func(s *SessionVars, val string) error { + s.DivPrecisionIncrement = tidbOptPositiveInt32(val, DefDivPrecisionIncrement) return nil }}, {Scope: ScopeSession, Name: TiDBDMLType, Value: DefTiDBDMLType, Type: TypeStr, @@ -3078,7 +3146,9 @@ var defaultSysVars = []*SysVar{ return nil } return errors.Errorf("unsupport DML type: %s", val) - }}, + }, + IsHintUpdatableVerified: true, + }, } // GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store) @@ -3421,6 +3491,8 @@ const ( MaxExecutionTime = "max_execution_time" // TiKVClientReadTimeout is the name of the 'tikv_client_read_timeout' system variable. TiKVClientReadTimeout = "tikv_client_read_timeout" + // TiDBLoadBindingTimeout is the name of the 'tidb_load_binding_timeout' system variable. + TiDBLoadBindingTimeout = "tidb_load_binding_timeout" // ReadOnly is the name of the 'read_only' system variable. ReadOnly = "read_only" // DefaultAuthPlugin is the name of 'default_authentication_plugin' system variable. diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index d655384bfb694..d5e9b34324d5e 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -948,6 +948,10 @@ const ( // TiDBSchemaCacheSize indicates the size of infoschema meta data which are cached in V2 implementation. TiDBSchemaCacheSize = "tidb_schema_cache_size" + + // DivPrecisionIncrement indicates the number of digits by which to increase the scale of the result of + // division operations performed with the / operator. + DivPrecisionIncrement = "div_precision_increment" ) // TiDB vars that have only global scope @@ -1150,8 +1154,6 @@ const ( // Any idle transaction will be killed after being idle for `tidb_idle_transaction_timeout` seconds. // This is similar to https://docs.percona.com/percona-server/5.7/management/innodb_kill_idle_trx.html and https://mariadb.com/kb/en/transaction-timeouts/ TiDBIdleTransactionTimeout = "tidb_idle_transaction_timeout" - // TiDBEnableParallelSort indicates if parallel sort is enabled. - TiDBEnableParallelSort = "enable_parallel_sort" // TiDBLowResolutionTSOUpdateInterval defines how often to refresh low resolution timestamps. TiDBLowResolutionTSOUpdateInterval = "tidb_low_resolution_tso_update_interval" // TiDBDMLType indicates the execution type of DML in TiDB. @@ -1480,10 +1482,10 @@ const ( DefTiDBOptObjective = OptObjectiveModerate DefTiDBSchemaVersionCacheLimit = 16 DefTiDBIdleTransactionTimeout = 0 - DefEnableParallelSort = false DefTiDBTxnEntrySizeLimit = 0 DefTiDBSchemaCacheSize = 0 DefTiDBLowResolutionTSOUpdateInterval = 2000 + DefDivPrecisionIncrement = 4 DefTiDBDMLType = "STANDARD" ) diff --git a/pkg/sessionctx/variable/variable.go b/pkg/sessionctx/variable/variable.go index 5c66ce2e4ec25..a05dcd792fe4c 100644 --- a/pkg/sessionctx/variable/variable.go +++ b/pkg/sessionctx/variable/variable.go @@ -142,8 +142,8 @@ type SysVar struct { SetSession func(*SessionVars, string) error // SetGlobal is called after validation SetGlobal func(context.Context, *SessionVars, string) error - // IsHintUpdatableVerfied indicate whether we've confirmed that SET_VAR() hint is worked for this hint. - IsHintUpdatableVerfied bool + // IsHintUpdatableVerified indicate whether we've confirmed that SET_VAR() hint is worked for this hint. + IsHintUpdatableVerified bool // Deprecated: Hidden previously meant that the variable still responds to SET but doesn't show up in SHOW VARIABLES // However, this feature is no longer used. All variables are visble. Hidden bool diff --git a/pkg/statistics/BUILD.bazel b/pkg/statistics/BUILD.bazel index e6b3b730c3a66..10e37351b43a5 100644 --- a/pkg/statistics/BUILD.bazel +++ b/pkg/statistics/BUILD.bazel @@ -79,7 +79,7 @@ go_test( data = glob(["testdata/**"]), embed = [":statistics"], flaky = True, - shard_count = 36, + shard_count = 37, deps = [ "//pkg/config", "//pkg/parser/ast", diff --git a/pkg/statistics/column.go b/pkg/statistics/column.go index d3d8ab43371ed..224de21cd16a6 100644 --- a/pkg/statistics/column.go +++ b/pkg/statistics/column.go @@ -21,8 +21,6 @@ import ( "github.com/pingcap/tidb/pkg/planner/util/debugtrace" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" - "github.com/pingcap/tidb/pkg/util/logutil" - "go.uber.org/zap" ) // Column represents a column histogram. @@ -138,13 +136,10 @@ func (c *Column) MemoryUsage() CacheItemMemoryUsage { // Currently, we only load index/pk's Histogram from kv automatically. Columns' are loaded by needs. var HistogramNeededItems = neededStatsMap{items: map[model.TableItemID]struct{}{}} -// IsInvalid checks if this column is invalid. +// ColumnStatsIsInvalid checks if this column is invalid. // If this column has histogram but not loaded yet, // then we mark it as need histogram. -func (c *Column) IsInvalid( - sctx context.PlanContext, - collPseudo bool, -) (res bool) { +func ColumnStatsIsInvalid(colStats *Column, sctx context.PlanContext, histColl *HistColl, cid int64) (res bool) { var totalCount float64 var ndv int64 var inValidForCollPseudo, essentialLoaded bool @@ -163,31 +158,33 @@ func (c *Column) IsInvalid( } if sctx != nil { stmtctx := sctx.GetSessionVars().StmtCtx - if (!c.IsStatsInitialized() || c.IsLoadNeeded()) && stmtctx != nil { - if stmtctx.StatsLoad.Timeout > 0 { - logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.", - zap.Int64("table_id", c.PhysicalID), - zap.Int64("column_id", c.Info.ID), - zap.String("column_name", c.Info.Name.O)) - } - // In some tests, the c.Info is not set, so we add this check here. - // When we are using stats from PseudoTable(), the table ID will possibly be -1. - // In this case, we don't trigger stats loading. - if c.Info != nil && c.PhysicalID > 0 { - HistogramNeededItems.Insert(model.TableItemID{TableID: c.PhysicalID, ID: c.Info.ID, IsIndex: false}) - } + if (colStats == nil || !colStats.IsStatsInitialized() || colStats.IsLoadNeeded()) && + stmtctx != nil && + !histColl.CanNotTriggerLoad { + HistogramNeededItems.Insert(model.TableItemID{ + TableID: histColl.PhysicalID, + ID: cid, + IsIndex: false, + IsSyncLoadFailed: sctx.GetSessionVars().StmtCtx.StatsLoad.Timeout > 0, + }) } } - if collPseudo { + if histColl.Pseudo { inValidForCollPseudo = true return true } + if colStats == nil { + totalCount = -1 + ndv = -1 + essentialLoaded = false + return true + } // In some cases, some statistics in column would be evicted // For example: the cmsketch of the column might be evicted while the histogram and the topn are still exists // In this case, we will think this column as valid due to we can still use the rest of the statistics to do optimize. - totalCount = c.TotalRowCount() - essentialLoaded = c.IsEssentialStatsLoaded() - ndv = c.Histogram.NDV + totalCount = colStats.TotalRowCount() + essentialLoaded = colStats.IsEssentialStatsLoaded() + ndv = colStats.Histogram.NDV return totalCount == 0 || (!essentialLoaded && ndv > 0) } @@ -210,7 +207,7 @@ func (c *Column) DropUnnecessaryData() { // IsAllEvicted indicates whether all stats evicted func (c *Column) IsAllEvicted() bool { - return c.statsInitialized && c.evictedStatus >= AllEvicted + return c == nil || (c.statsInitialized && c.evictedStatus >= AllEvicted) } // GetEvictedStatus indicates the evicted status diff --git a/pkg/statistics/handle/autoanalyze/autoanalyze.go b/pkg/statistics/handle/autoanalyze/autoanalyze.go index 6a526dc5e2923..231c6c039850b 100644 --- a/pkg/statistics/handle/autoanalyze/autoanalyze.go +++ b/pkg/statistics/handle/autoanalyze/autoanalyze.go @@ -487,7 +487,7 @@ func tryAutoAnalyzeTable( // Whether the table needs to analyze or not, we need to check the indices of the table. for _, idx := range tblInfo.Indices { - if _, ok := statsTbl.Indices[idx.ID]; !ok && idx.State == model.StatePublic { + if _, ok := statsTbl.Indices[idx.ID]; !ok && !statsTbl.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) && idx.State == model.StatePublic { sqlWithIdx := sql + " index %n" paramsWithIdx := append(params, idx.Name.O) escaped, err := sqlescape.EscapeSQL(sqlWithIdx, paramsWithIdx...) @@ -634,7 +634,7 @@ func tryAutoAnalyzePartitionTableInDynamicMode( continue } // 2. If the index is not analyzed, we need to analyze it. - if _, ok := partitionStats.Indices[idx.ID]; !ok { + if !partitionStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) { needAnalyzePartitionNames = append(needAnalyzePartitionNames, def.Name.O) statistics.CheckAnalyzeVerOnTable(partitionStats, &tableStatsVer) } diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job.go b/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job.go index e15385b2b424c..bed09736f4925 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job.go @@ -111,7 +111,9 @@ func (j *DynamicPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool { // IsValidToAnalyze checks whether the table or partition is valid to analyze. // We need to check each partition to determine whether the table is valid to analyze. -func (j *DynamicPartitionedTableAnalysisJob) IsValidToAnalyze(sctx sessionctx.Context) (bool, string) { +func (j *DynamicPartitionedTableAnalysisJob) IsValidToAnalyze( + sctx sessionctx.Context, +) (bool, string) { if valid, failReason := isValidWeight(j.Weight); !valid { return false, failReason } @@ -202,6 +204,7 @@ func (j *DynamicPartitionedTableAnalysisJob) analyzePartitionIndexes( ) { analyzePartitionBatchSize := int(variable.AutoAnalyzePartitionBatchSize.Load()) +OnlyPickOneIndex: for indexName, partitionNames := range j.PartitionIndexes { needAnalyzePartitionNames := make([]any, 0, len(partitionNames)) for _, partition := range partitionNames { @@ -218,6 +221,10 @@ func (j *DynamicPartitionedTableAnalysisJob) analyzePartitionIndexes( params := append([]any{j.TableSchema, j.GlobalTableName}, needAnalyzePartitionNames[start:end]...) params = append(params, indexName) exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...) + // Halt execution after analyzing one index. + // This is because analyzing a single index also analyzes all other indexes and columns. + // Therefore, to avoid redundancy, we prevent multiple analyses of the same partition. + break OnlyPickOneIndex } } } diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job_test.go b/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job_test.go index 0c30b8d952966..70da77f5ea13f 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job_test.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job_test.go @@ -65,50 +65,54 @@ func TestAnalyzeDynamicPartitionedTableIndexes(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec("create table t (a int, b int, index idx(a)) partition by range (a) (partition p0 values less than (2), partition p1 values less than (4))") + tk.MustExec("create table t (a int, b int, index idx(a), index idx1(b)) partition by range (a) (partition p0 values less than (2), partition p1 values less than (4))") tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3)") job := &priorityqueue.DynamicPartitionedTableAnalysisJob{ TableSchema: "test", GlobalTableName: "t", PartitionIndexes: map[string][]string{ - "idx": {"p0", "p1"}, + "idx": {"p0", "p1"}, + "idx1": {"p0", "p1"}, }, TableStatsVer: 2, } // Before analyze partitions. handle := dom.StatsHandle() - // Check the result of analyze. + // Check the result of analyze index. is := dom.InfoSchema() tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) pid := tbl.Meta().GetPartitionInfo().Definitions[0].ID tblStats := handle.GetPartitionStats(tbl.Meta(), pid) require.True(t, tblStats.Pseudo) - // Check the result of analyze index. require.NotNil(t, tblStats.Indices[1]) require.False(t, tblStats.Indices[1].IsAnalyzed()) job.Analyze(handle, dom.SysProcTracker()) - // Check the result of analyze. + // Check the result of analyze index. is = dom.InfoSchema() tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) pid = tbl.Meta().GetPartitionInfo().Definitions[0].ID tblStats = handle.GetPartitionStats(tbl.Meta(), pid) require.False(t, tblStats.Pseudo) - require.Equal(t, int64(1), tblStats.RealtimeCount) - // Check the result of analyze index. require.NotNil(t, tblStats.Indices[1]) require.True(t, tblStats.Indices[1].IsAnalyzed()) + require.NotNil(t, tblStats.Indices[2]) + require.True(t, tblStats.Indices[2].IsAnalyzed()) // partition p1 pid = tbl.Meta().GetPartitionInfo().Definitions[1].ID tblStats = handle.GetPartitionStats(tbl.Meta(), pid) require.False(t, tblStats.Pseudo) - require.Equal(t, int64(2), tblStats.RealtimeCount) - // Check the result of analyze index. require.NotNil(t, tblStats.Indices[1]) require.True(t, tblStats.Indices[1].IsAnalyzed()) + require.NotNil(t, tblStats.Indices[2]) + require.True(t, tblStats.Indices[2].IsAnalyzed()) + // Check analyze jobs are created. + rows := tk.MustQuery("select * from mysql.analyze_jobs").Rows() + // Because analyze one index will analyze all indexes and all columns together, so there are 5 jobs. + require.Len(t, rows, 5) } func TestIsValidToAnalyzeForDynamicPartitionedTable(t *testing.T) { diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/job.go b/pkg/statistics/handle/autoanalyze/priorityqueue/job.go index 8171f720de2ab..f06e5d93e864f 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/job.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/job.go @@ -19,7 +19,7 @@ import ( "time" "github.com/pingcap/tidb/pkg/sessionctx" - statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil" + "github.com/pingcap/tidb/pkg/statistics/handle/logutil" statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" "github.com/pingcap/tidb/pkg/util/intest" "go.uber.org/zap" @@ -50,7 +50,9 @@ type AnalysisJob interface { // It checks the last failed analysis duration and the average analysis duration. // If the last failed analysis duration is less than 2 times the average analysis duration, // we skip this table to avoid too much failed analysis. - IsValidToAnalyze(sctx sessionctx.Context) (bool, string) + IsValidToAnalyze( + sctx sessionctx.Context, + ) (bool, string) // Analyze executes the analyze statement within a transaction. Analyze( @@ -98,7 +100,7 @@ func isValidToAnalyze( lastFailedAnalysisDuration, err := GetLastFailedAnalysisDuration(sctx, schema, table, partitionNames...) if err != nil { - statslogutil.StatsLogger().Warn( + logutil.SingletonStatsSamplerLogger().Warn( "Fail to get last failed analysis duration", zap.String("schema", schema), zap.String("table", table), @@ -111,7 +113,7 @@ func isValidToAnalyze( averageAnalysisDuration, err := GetAverageAnalysisDuration(sctx, schema, table, partitionNames...) if err != nil { - statslogutil.StatsLogger().Warn( + logutil.SingletonStatsSamplerLogger().Warn( "Fail to get average analysis duration", zap.String("schema", schema), zap.String("table", table), @@ -124,7 +126,7 @@ func isValidToAnalyze( // Last analysis just failed, we should not analyze it again. if lastFailedAnalysisDuration == justFailed { // The last analysis failed, we should not analyze it again. - statslogutil.StatsLogger().Info( + logutil.SingletonStatsSamplerLogger().Info( "Skip analysis because the last analysis just failed", zap.String("schema", schema), zap.String("table", table), @@ -137,7 +139,7 @@ func isValidToAnalyze( // Skip this table to avoid too much failed analysis. onlyFailedAnalysis := lastFailedAnalysisDuration != NoRecord && averageAnalysisDuration == NoRecord if onlyFailedAnalysis && lastFailedAnalysisDuration < defaultFailedAnalysisWaitTime { - statslogutil.StatsLogger().Info( + logutil.SingletonStatsSamplerLogger().Info( fmt.Sprintf("Skip analysis because the last failed analysis duration is less than %v", defaultFailedAnalysisWaitTime), zap.String("schema", schema), zap.String("table", table), @@ -151,7 +153,7 @@ func isValidToAnalyze( meetSkipCondition := lastFailedAnalysisDuration != NoRecord && lastFailedAnalysisDuration < 2*averageAnalysisDuration if meetSkipCondition { - statslogutil.StatsLogger().Info( + logutil.SingletonStatsSamplerLogger().Info( "Skip analysis because the last failed analysis duration is less than 2 times the average analysis duration", zap.String("schema", schema), zap.String("table", table), diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job.go b/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job.go index dfe4ef4508789..748d1873bc7b9 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job.go @@ -91,7 +91,9 @@ func (j *NonPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool { // IsValidToAnalyze checks whether the table is valid to analyze. // We will check the last failed job and average analyze duration to determine whether the table is valid to analyze. -func (j *NonPartitionedTableAnalysisJob) IsValidToAnalyze(sctx sessionctx.Context) (bool, string) { +func (j *NonPartitionedTableAnalysisJob) IsValidToAnalyze( + sctx sessionctx.Context, +) (bool, string) { if valid, failReason := isValidWeight(j.Weight); !valid { return false, failReason } @@ -171,10 +173,15 @@ func (j *NonPartitionedTableAnalysisJob) analyzeIndexes( statsHandle statstypes.StatsHandle, sysProcTracker sessionctx.SysProcTracker, ) { - for _, index := range j.Indexes { - sql, params := j.GenSQLForAnalyzeIndex(index) - exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...) + if len(j.Indexes) == 0 { + return } + // Only analyze the first index. + // This is because analyzing a single index also analyzes all other indexes and columns. + // Therefore, to avoid redundancy, we prevent multiple analyses of the same table. + firstIndex := j.Indexes[0] + sql, params := j.GenSQLForAnalyzeIndex(firstIndex) + exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...) } // GenSQLForAnalyzeIndex generates the SQL for analyzing the specified index. diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job_test.go b/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job_test.go index 26aa72c46a2b4..5914c5bb513ff 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job_test.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job_test.go @@ -92,12 +92,12 @@ func TestAnalyzeNonPartitionedIndexes(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec("create table t (a int, b int, index idx(a))") + tk.MustExec("create table t (a int, b int, index idx(a), index idx1(b))") tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3)") job := &priorityqueue.NonPartitionedTableAnalysisJob{ TableSchema: "test", TableName: "t", - Indexes: []string{"idx"}, + Indexes: []string{"idx", "idx1"}, TableStatsVer: 2, } handle := dom.StatsHandle() @@ -116,30 +116,12 @@ func TestAnalyzeNonPartitionedIndexes(t *testing.T) { tblStats = handle.GetTableStats(tbl.Meta()) require.NotNil(t, tblStats.Indices[1]) require.True(t, tblStats.Indices[1].IsAnalyzed()) - // Add a new index. - tk.MustExec("alter table t add index idx2(b)") - job = &priorityqueue.NonPartitionedTableAnalysisJob{ - TableSchema: "test", - TableName: "t", - Indexes: []string{"idx", "idx2"}, - TableStatsVer: 2, - } - require.NoError(t, handle.Update(dom.InfoSchema())) - // Before analyze indexes. - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tblStats = handle.GetTableStats(tbl.Meta()) - require.Len(t, tblStats.Indices, 1) - - job.Analyze(handle, dom.SysProcTracker()) - // Check the result of analyze. - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tblStats = handle.GetTableStats(tbl.Meta()) require.NotNil(t, tblStats.Indices[2]) require.True(t, tblStats.Indices[2].IsAnalyzed()) + // Check analyze jobs are created. + rows := tk.MustQuery("select * from mysql.analyze_jobs").Rows() + // Because analyze one index will analyze all indexes and all columns together, so there is only 1 job. + require.Len(t, rows, 1) } func TestNonPartitionedTableIsValidToAnalyze(t *testing.T) { diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job.go b/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job.go index f44818a9599a1..29bee38911a5d 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job.go @@ -104,7 +104,9 @@ func (j *StaticPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool { // IsValidToAnalyze checks whether the partition is valid to analyze. // Only the specified static partition is checked. -func (j *StaticPartitionedTableAnalysisJob) IsValidToAnalyze(sctx sessionctx.Context) (bool, string) { +func (j *StaticPartitionedTableAnalysisJob) IsValidToAnalyze( + sctx sessionctx.Context, +) (bool, string) { if valid, failReason := isValidWeight(j.Weight); !valid { return false, failReason } @@ -183,10 +185,15 @@ func (j *StaticPartitionedTableAnalysisJob) analyzeStaticPartitionIndexes( statsHandle statstypes.StatsHandle, sysProcTracker sessionctx.SysProcTracker, ) { - for _, index := range j.Indexes { - sql, params := j.GenSQLForAnalyzeStaticPartitionIndex(index) - exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...) + if len(j.Indexes) == 0 { + return } + // Only analyze the first index. + // This is because analyzing a single index also analyzes all other indexes and columns. + // Therefore, to avoid redundancy, we prevent multiple analyses of the same partition. + firstIndex := j.Indexes[0] + sql, params := j.GenSQLForAnalyzeStaticPartitionIndex(firstIndex) + exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...) } // GenSQLForAnalyzeStaticPartition generates the SQL for analyzing the specified static partition. diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job_test.go b/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job_test.go index bceec628f5c42..2b76335693eb5 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job_test.go +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job_test.go @@ -99,13 +99,13 @@ func TestAnalyzeStaticPartitionedTableIndexes(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec("create table t (a int, b int, index idx(a)) partition by range (a) (partition p0 values less than (2), partition p1 values less than (4))") + tk.MustExec("create table t (a int, b int, index idx(a), index idx1(b)) partition by range (a) (partition p0 values less than (2), partition p1 values less than (4))") tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3)") job := &priorityqueue.StaticPartitionedTableAnalysisJob{ TableSchema: "test", GlobalTableName: "t", StaticPartitionName: "p0", - Indexes: []string{"idx"}, + Indexes: []string{"idx", "idx1"}, TableStatsVer: 2, } handle := dom.StatsHandle() @@ -126,33 +126,12 @@ func TestAnalyzeStaticPartitionedTableIndexes(t *testing.T) { tblStats = handle.GetPartitionStats(tbl.Meta(), pid) require.NotNil(t, tblStats.Indices[1]) require.True(t, tblStats.Indices[1].IsAnalyzed()) - // Add a new index. - tk.MustExec("alter table t add index idx2(b)") - job = &priorityqueue.StaticPartitionedTableAnalysisJob{ - TableSchema: "test", - GlobalTableName: "t", - StaticPartitionName: "p0", - Indexes: []string{"idx", "idx2"}, - TableStatsVer: 2, - } - require.NoError(t, handle.Update(dom.InfoSchema())) - // Before analyze indexes. - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - pid = tbl.Meta().GetPartitionInfo().Definitions[0].ID - tblStats = handle.GetPartitionStats(tbl.Meta(), pid) - require.Len(t, tblStats.Indices, 1) - - job.Analyze(handle, dom.SysProcTracker()) - // Check the result of analyze. - is = dom.InfoSchema() - tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - pid = tbl.Meta().GetPartitionInfo().Definitions[0].ID - tblStats = handle.GetPartitionStats(tbl.Meta(), pid) require.NotNil(t, tblStats.Indices[2]) require.True(t, tblStats.Indices[2].IsAnalyzed()) + // Check analyze jobs are created. + rows := tk.MustQuery("select * from mysql.analyze_jobs").Rows() + // Because analyze one index will analyze all indexes and all columns together, so there are 4 jobs. + require.Len(t, rows, 4) } func TestStaticPartitionedTableIsValidToAnalyze(t *testing.T) { diff --git a/pkg/statistics/handle/autoanalyze/refresher/refresher.go b/pkg/statistics/handle/autoanalyze/refresher/refresher.go index c2b0c920705c3..375ccf1491940 100644 --- a/pkg/statistics/handle/autoanalyze/refresher/refresher.go +++ b/pkg/statistics/handle/autoanalyze/refresher/refresher.go @@ -91,7 +91,7 @@ func (r *Refresher) PickOneTableAndAnalyzeByPriority() bool { if valid, failReason := job.IsValidToAnalyze( sctx, ); !valid { - statslogutil.StatsLogger().Info( + statslogutil.SingletonStatsSamplerLogger().Info( "Table is not ready to analyze", zap.String("failReason", failReason), zap.Stringer("job", job), @@ -116,7 +116,7 @@ func (r *Refresher) PickOneTableAndAnalyzeByPriority() bool { // Only analyze one table each time. return true } - statslogutil.StatsLogger().Debug( + statslogutil.SingletonStatsSamplerLogger().Info( "No table to analyze", ) return false @@ -446,7 +446,7 @@ func CheckIndexesNeedAnalyze( indexes := make([]string, 0, len(tblInfo.Indices)) // Check if missing index stats. for _, idx := range tblInfo.Indices { - if _, ok := tblStats.Indices[idx.ID]; !ok && idx.State == model.StatePublic { + if _, ok := tblStats.Indices[idx.ID]; !ok && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) && idx.State == model.StatePublic { indexes = append(indexes, idx.Name.O) } } @@ -571,7 +571,7 @@ func CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable( // Find all the partitions that need to analyze this index. names := make([]string, 0, len(partitionStats)) for pIDAndName, tblStats := range partitionStats { - if _, ok := tblStats.Indices[idx.ID]; !ok { + if _, ok := tblStats.Indices[idx.ID]; !ok && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) { names = append(names, pIDAndName.Name) } } diff --git a/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go b/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go index 225850a351f90..d5cdbaf8968b9 100644 --- a/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go +++ b/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go @@ -381,6 +381,12 @@ func TestCalculateChangePercentage(t *testing.T) { StatsVer: 2, }, } + bothUnanalyzedMap := statistics.NewColAndIndexExistenceMap(0, 0) + bothAnalyzedMap := statistics.NewColAndIndexExistenceMap(2, 2) + bothAnalyzedMap.InsertCol(1, nil, true) + bothAnalyzedMap.InsertCol(2, nil, true) + bothAnalyzedMap.InsertIndex(1, nil, true) + bothAnalyzedMap.InsertIndex(2, nil, true) tests := []struct { name string tblStats *statistics.Table @@ -396,6 +402,7 @@ func TestCalculateChangePercentage(t *testing.T) { Columns: unanalyzedColumns, Indices: unanalyzedIndices, }, + ColAndIdxExistenceMap: bothUnanalyzedMap, }, autoAnalyzeRatio: 0.5, want: 1, @@ -410,7 +417,8 @@ func TestCalculateChangePercentage(t *testing.T) { Indices: analyzedIndices, ModifyCount: (exec.AutoAnalyzeMinCnt + 1) * 2, }, - LastAnalyzeVersion: 1, + ColAndIdxExistenceMap: bothAnalyzedMap, + LastAnalyzeVersion: 1, }, autoAnalyzeRatio: 0.5, want: 2, @@ -466,6 +474,9 @@ func TestGetTableLastAnalyzeDurationForUnanalyzedTable(t *testing.T) { } func TestCheckIndexesNeedAnalyze(t *testing.T) { + analyzedMap := statistics.NewColAndIndexExistenceMap(1, 0) + analyzedMap.InsertCol(1, nil, true) + analyzedMap.InsertIndex(1, nil, false) tests := []struct { name string tblInfo *model.TableInfo @@ -483,7 +494,7 @@ func TestCheckIndexesNeedAnalyze(t *testing.T) { }, }, }, - tblStats: &statistics.Table{}, + tblStats: &statistics.Table{ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0)}, want: nil, }, { @@ -507,7 +518,8 @@ func TestCheckIndexesNeedAnalyze(t *testing.T) { }, }, }, - LastAnalyzeVersion: 1, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: 1, }, want: []string{"index1"}, }, @@ -528,6 +540,11 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { // 2023-12-31 10:00:00 lastUpdateTime := time.Date(2023, 12, 31, 10, 0, 0, 0, time.UTC) lastUpdateTs := oracle.GoTimeToTS(lastUpdateTime) + unanalyzedMap := statistics.NewColAndIndexExistenceMap(0, 0) + analyzedMap := statistics.NewColAndIndexExistenceMap(2, 1) + analyzedMap.InsertCol(1, nil, true) + analyzedMap.InsertCol(2, nil, true) + analyzedMap.InsertIndex(1, nil, true) tests := []struct { name string tblInfo *model.TableInfo @@ -568,6 +585,7 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { Pseudo: false, RealtimeCount: exec.AutoAnalyzeMinCnt + 1, }, + ColAndIdxExistenceMap: unanalyzedMap, }, { ID: 2, @@ -577,6 +595,7 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { Pseudo: false, RealtimeCount: exec.AutoAnalyzeMinCnt + 1, }, + ColAndIdxExistenceMap: unanalyzedMap, }, }, defs: []model.PartitionDefinition{ @@ -639,8 +658,9 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { }, }, }, - Version: currentTs, - LastAnalyzeVersion: lastUpdateTs, + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, }, { ID: 2, @@ -665,8 +685,9 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { }, }, }, - Version: currentTs, - LastAnalyzeVersion: lastUpdateTs, + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, }, }, defs: []model.PartitionDefinition{ @@ -729,8 +750,9 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { }, }, }, - Version: currentTs, - LastAnalyzeVersion: lastUpdateTs, + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, }, { ID: 2, @@ -755,8 +777,9 @@ func TestCalculateIndicatorsForPartitions(t *testing.T) { }, }, }, - Version: currentTs, - LastAnalyzeVersion: lastUpdateTs, + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, }, }, defs: []model.PartitionDefinition{ @@ -835,6 +858,7 @@ func TestCheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(t *testing.T) { ModifyCount: 0, Indices: map[int64]*statistics.Index{}, }, + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0), }, { ID: 2, @@ -850,8 +874,10 @@ func TestCheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(t *testing.T) { }, }, }, + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 1), }, } + partitionIndexes := refresher.CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(&tblInfo, partitionStats) expected := map[string][]string{"index1": {"p0", "p1"}, "index2": {"p0"}} require.Equal(t, len(expected), len(partitionIndexes)) diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go index 31304c4a867f6..394699f54059e 100644 --- a/pkg/statistics/handle/bootstrap.go +++ b/pkg/statistics/handle/bootstrap.go @@ -51,13 +51,15 @@ func (h *Handle) initStatsMeta4Chunk(is infoschema.InfoSchema, cache statstypes. HavePhysicalID: true, RealtimeCount: row.GetInt64(3), ModifyCount: row.GetInt64(2), - Columns: make(map[int64]*statistics.Column, len(tableInfo.Columns)), - Indices: make(map[int64]*statistics.Index, len(tableInfo.Indices)), + Columns: make(map[int64]*statistics.Column, 4), + Indices: make(map[int64]*statistics.Index, 4), } tbl := &statistics.Table{ - HistColl: newHistColl, - Version: row.GetUint64(0), - Name: util.GetFullTableName(is, tableInfo), + HistColl: newHistColl, + Version: row.GetUint64(0), + Name: util.GetFullTableName(is, tableInfo), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(len(tableInfo.Columns), len(tableInfo.Indices)), + IsPkIsHandle: tableInfo.PKIsHandle, } cache.Put(physicalID, tbl) // put this table again since it is updated } @@ -100,11 +102,8 @@ func (h *Handle) initStatsHistograms4ChunkLite(is infoschema.InfoSchema, cache s isIndex := row.GetInt64(1) id := row.GetInt64(2) ndv := row.GetInt64(3) - version := row.GetUint64(4) nullCount := row.GetInt64(5) statsVer := row.GetInt64(7) - flag := row.GetInt64(9) - lastAnalyzePos := row.GetDatum(10, types.NewFieldType(mysql.TypeBlob)) tbl, _ := h.TableInfoByID(is, table.PhysicalID) // All the objects in the table share the same stats version. if statsVer != statistics.Version0 { @@ -121,21 +120,11 @@ func (h *Handle) initStatsHistograms4ChunkLite(is infoschema.InfoSchema, cache s if idxInfo == nil { continue } - hist := statistics.NewHistogram(id, ndv, nullCount, version, types.NewFieldType(mysql.TypeBlob), 0, 0) - index := &statistics.Index{ - Histogram: *hist, - Info: idxInfo, - StatsVer: statsVer, - Flag: flag, - PhysicalID: tblID, - } - lastAnalyzePos.Copy(&index.LastAnalyzePos) - if index.IsAnalyzed() { - index.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus() + table.ColAndIdxExistenceMap.InsertIndex(idxInfo.ID, idxInfo, statsVer != statistics.Version0) + if statsVer != statistics.Version0 { // The LastAnalyzeVersion is added by ALTER table so its value might be 0. - table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, version) + table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, row.GetUint64(4)) } - table.Indices[hist.ID] = index } else { var colInfo *model.ColumnInfo for _, col := range tbl.Meta().Columns { @@ -147,23 +136,11 @@ func (h *Handle) initStatsHistograms4ChunkLite(is infoschema.InfoSchema, cache s if colInfo == nil { continue } - hist := statistics.NewHistogram(id, ndv, nullCount, version, &colInfo.FieldType, 0, row.GetInt64(6)) - hist.Correlation = row.GetFloat64(8) - col := &statistics.Column{ - Histogram: *hist, - PhysicalID: tblID, - Info: colInfo, - IsHandle: tbl.Meta().PKIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()), - Flag: flag, - StatsVer: statsVer, - } - lastAnalyzePos.Copy(&col.LastAnalyzePos) - if col.StatsAvailable() { - col.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus() + table.ColAndIdxExistenceMap.InsertCol(colInfo.ID, colInfo, statsVer != statistics.Version0 || ndv > 0 || nullCount > 0) + if statsVer != statistics.Version0 { // The LastAnalyzeVersion is added by ALTER table so its value might be 0. - table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, version) + table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, row.GetUint64(4)) } - table.Columns[hist.ID] = col } cache.Put(tblID, table) // put this table again since it is updated } @@ -217,6 +194,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, cache stats } lastAnalyzePos.Copy(&index.LastAnalyzePos) table.Indices[hist.ID] = index + table.ColAndIdxExistenceMap.InsertIndex(idxInfo.ID, idxInfo, statsVer != statistics.Version0) } else { var colInfo *model.ColumnInfo for _, col := range tbl.Meta().Columns { @@ -240,6 +218,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, cache stats } lastAnalyzePos.Copy(&col.LastAnalyzePos) table.Columns[hist.ID] = col + table.ColAndIdxExistenceMap.InsertCol(colInfo.ID, colInfo, statsVer != statistics.Version0 || ndv > 0 || nullCount > 0) if statsVer != statistics.Version0 { // The LastAnalyzeVersion is added by ALTER table so its value might be 0. table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, version) diff --git a/pkg/statistics/handle/ddl/ddl_test.go b/pkg/statistics/handle/ddl/ddl_test.go index e1613a2a2e750..fc2964ce4ac8b 100644 --- a/pkg/statistics/handle/ddl/ddl_test.go +++ b/pkg/statistics/handle/ddl/ddl_test.go @@ -241,6 +241,7 @@ func TestDDLHistogram(t *testing.T) { require.NoError(t, err) tableInfo := tbl.Meta() statsTbl := do.StatsHandle().GetTableStats(tableInfo) + require.True(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(2, false)) require.False(t, statsTbl.Pseudo) require.True(t, statsTbl.Columns[tableInfo.Columns[2].ID].IsStatsInitialized()) require.Equal(t, int64(2), statsTbl.Columns[tableInfo.Columns[2].ID].NullCount) @@ -256,6 +257,7 @@ func TestDDLHistogram(t *testing.T) { tableInfo = tbl.Meta() statsTbl = do.StatsHandle().GetTableStats(tableInfo) require.False(t, statsTbl.Pseudo) + require.True(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(3, false)) require.True(t, statsTbl.Columns[tableInfo.Columns[3].ID].IsStatsInitialized()) sctx := mock.NewContext() count, err := cardinality.ColumnEqualRowCount(sctx, statsTbl, types.NewIntDatum(0), tableInfo.Columns[3].ID) @@ -276,6 +278,7 @@ func TestDDLHistogram(t *testing.T) { statsTbl = do.StatsHandle().GetTableStats(tableInfo) // If we don't use original default value, we will get a pseudo table. require.False(t, statsTbl.Pseudo) + require.True(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(4, false)) testKit.MustExec("alter table t add column c5 varchar(15) DEFAULT '123'") err = h.HandleDDLEvent(<-h.DDLEventCh()) @@ -287,6 +290,7 @@ func TestDDLHistogram(t *testing.T) { tableInfo = tbl.Meta() statsTbl = do.StatsHandle().GetTableStats(tableInfo) require.False(t, statsTbl.Pseudo) + require.True(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(5, false)) require.True(t, statsTbl.Columns[tableInfo.Columns[5].ID].IsStatsInitialized()) require.Equal(t, 3.0, cardinality.AvgColSize(statsTbl.Columns[tableInfo.Columns[5].ID], statsTbl.RealtimeCount, false)) @@ -302,7 +306,17 @@ func TestDDLHistogram(t *testing.T) { require.False(t, statsTbl.Pseudo) testKit.MustExec("create index i on t(c2, c1)") + tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo = tbl.Meta() + statsTbl = do.StatsHandle().GetTableStats(tableInfo) + require.False(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(1, true)) testKit.MustExec("analyze table t") + tbl, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo = tbl.Meta() + statsTbl = do.StatsHandle().GetTableStats(tableInfo) + require.True(t, statsTbl.ColAndIdxExistenceMap.HasAnalyzed(1, true)) rs := testKit.MustQuery("select count(*) from mysql.stats_histograms where table_id = ? and hist_id = 1 and is_index =1", tableInfo.ID) rs.Check(testkit.Rows("1")) rs = testKit.MustQuery("select count(*) from mysql.stats_buckets where table_id = ? and hist_id = 1 and is_index = 1", tableInfo.ID) diff --git a/pkg/statistics/handle/handle.go b/pkg/statistics/handle/handle.go index 1e6b3fd8f6208..462cb002399bd 100644 --- a/pkg/statistics/handle/handle.go +++ b/pkg/statistics/handle/handle.go @@ -168,14 +168,14 @@ func (h *Handle) GetPartitionStatsForAutoAnalyze(tblInfo *model.TableInfo, pid i func (h *Handle) getPartitionStats(tblInfo *model.TableInfo, pid int64, returnPseudo bool) *statistics.Table { var tbl *statistics.Table if h == nil { - tbl = statistics.PseudoTable(tblInfo, false) + tbl = statistics.PseudoTable(tblInfo, false, false) tbl.PhysicalID = pid return tbl } tbl, ok := h.Get(pid) if !ok { if returnPseudo { - tbl = statistics.PseudoTable(tblInfo, false) + tbl = statistics.PseudoTable(tblInfo, false, true) tbl.PhysicalID = pid if tblInfo.GetPartitionInfo() == nil || h.Len() < 64 { h.UpdateStatsCache([]*statistics.Table{tbl}, nil) diff --git a/pkg/statistics/handle/handletest/BUILD.bazel b/pkg/statistics/handle/handletest/BUILD.bazel index 1ab3c8d429ccb..505a2c5737c70 100644 --- a/pkg/statistics/handle/handletest/BUILD.bazel +++ b/pkg/statistics/handle/handletest/BUILD.bazel @@ -18,7 +18,6 @@ go_test( "//pkg/sessionctx/variable", "//pkg/statistics", "//pkg/statistics/handle", - "//pkg/statistics/handle/internal", "//pkg/statistics/handle/util", "//pkg/testkit", "//pkg/testkit/testsetup", diff --git a/pkg/statistics/handle/handletest/handle_test.go b/pkg/statistics/handle/handletest/handle_test.go index 926c0b386874a..313fbc99ca0e6 100644 --- a/pkg/statistics/handle/handletest/handle_test.go +++ b/pkg/statistics/handle/handletest/handle_test.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle" - "github.com/pingcap/tidb/pkg/statistics/handle/internal" "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" @@ -183,11 +182,12 @@ func TestVersion(t *testing.T) { require.NoError(t, h.Update(is)) statsTbl2 = h.GetTableStats(tableInfo2) require.False(t, statsTbl2.Pseudo) - // We can read it without analyze again! Thanks for PrevLastVersion. - require.NotNil(t, statsTbl2.Columns[int64(3)]) - // assert WithGetTableStatsByQuery get the same result - statsTbl2 = h.GetTableStats(tableInfo2) - require.False(t, statsTbl2.Pseudo) + require.Nil(t, statsTbl2.Columns[int64(3)]) + tbl2, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t2")) + require.NoError(t, err) + tableInfo2 = tbl2.Meta() + statsTbl2, err = h.TableStatsFromStorage(tableInfo2, tableInfo2.ID, true, 0) + require.NoError(t, err) require.NotNil(t, statsTbl2.Columns[int64(3)]) } @@ -1390,7 +1390,15 @@ func TestInitStatsLite(t *testing.T) { require.NoError(t, h.InitStatsLite(is)) statsTbl1 := h.GetTableStats(tblInfo) checkAllEvicted(t, statsTbl1) - internal.AssertTableEqual(t, statsTbl0, statsTbl1) + { + // internal.AssertTableEqual(t, statsTbl0, statsTbl1) + // statsTbl0 is loaded when the cache has pseudo table. + // TODO: We haven't optimize the pseudo table's memory usage yet. So here the two will be different. + require.True(t, len(statsTbl0.Columns) > 0) + require.True(t, len(statsTbl0.Indices) > 0) + require.True(t, len(statsTbl1.Columns) == 0) + require.True(t, len(statsTbl1.Indices) == 0) + } // async stats load tk.MustExec("set @@tidb_stats_load_sync_wait = 0") diff --git a/pkg/statistics/handle/handletest/statstest/stats_test.go b/pkg/statistics/handle/handletest/statstest/stats_test.go index 7c710becadc37..593c5cdd3b23b 100644 --- a/pkg/statistics/handle/handletest/statstest/stats_test.go +++ b/pkg/statistics/handle/handletest/statstest/stats_test.go @@ -233,8 +233,25 @@ func TestInitStats(t *testing.T) { require.Equal(t, uint8(0x38), cols[3].LastAnalyzePos.GetBytes()[0]) h.Clear() require.NoError(t, h.Update(is)) - table1 := h.GetTableStats(tbl.Meta()) - internal.AssertTableEqual(t, table0, table1) + // Index and pk are loaded. + needed := fmt.Sprintf(`Table:%v RealtimeCount:6 +column:1 ndv:6 totColSize:0 +num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0 +num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0 +num: 1 lower_bound: 3 upper_bound: 3 repeats: 1 ndv: 0 +num: 1 lower_bound: 4 upper_bound: 4 repeats: 1 ndv: 0 +num: 1 lower_bound: 5 upper_bound: 5 repeats: 1 ndv: 0 +num: 1 lower_bound: 6 upper_bound: 6 repeats: 1 ndv: 0 +column:2 ndv:6 totColSize:6 +column:3 ndv:6 totColSize:6 +index:1 ndv:6 +num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0 +num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0 +num: 1 lower_bound: 3 upper_bound: 3 repeats: 1 ndv: 0 +num: 1 lower_bound: 4 upper_bound: 4 repeats: 1 ndv: 0 +num: 1 lower_bound: 5 upper_bound: 5 repeats: 1 ndv: 0 +num: 1 lower_bound: 7 upper_bound: 7 repeats: 1 ndv: 0`, tbl.Meta().ID) + require.Equal(t, needed, table0.String()) h.SetLease(0) } diff --git a/pkg/statistics/handle/internal/testutil.go b/pkg/statistics/handle/internal/testutil.go index 9757fc7fbb7b7..54898bec336f2 100644 --- a/pkg/statistics/handle/internal/testutil.go +++ b/pkg/statistics/handle/internal/testutil.go @@ -47,6 +47,7 @@ func AssertTableEqual(t *testing.T, a *statistics.Table, b *statistics.Table) { require.True(t, a.Indices[i].TopN.Equal(b.Indices[i].TopN)) } require.True(t, IsSameExtendedStats(a.ExtendedStats, b.ExtendedStats)) + require.True(t, statistics.ColAndIdxExistenceMapIsEqual(a.ColAndIdxExistenceMap, b.ColAndIdxExistenceMap)) } // IsSameExtendedStats is to judge whether the extended states is the same. diff --git a/pkg/statistics/handle/logutil/BUILD.bazel b/pkg/statistics/handle/logutil/BUILD.bazel index 49bc63b86585e..78b06985a8f13 100644 --- a/pkg/statistics/handle/logutil/BUILD.bazel +++ b/pkg/statistics/handle/logutil/BUILD.bazel @@ -8,5 +8,6 @@ go_library( deps = [ "//pkg/util/logutil", "@org_uber_go_zap//:zap", + "@org_uber_go_zap//zapcore", ], ) diff --git a/pkg/statistics/handle/logutil/logutil.go b/pkg/statistics/handle/logutil/logutil.go index ea791531a7760..a7c4413404028 100644 --- a/pkg/statistics/handle/logutil/logutil.go +++ b/pkg/statistics/handle/logutil/logutil.go @@ -15,11 +15,42 @@ package logutil import ( + "sync" + "time" + "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) // StatsLogger with category "stats" is used to log statistic related messages. +// Do not use it to log the message that is not related to statistics. func StatsLogger() *zap.Logger { return logutil.BgLogger().With(zap.String("category", "stats")) } + +var ( + initSamplerLoggerOnce sync.Once + samplerLogger *zap.Logger +) + +// SingletonStatsSamplerLogger with category "stats" is used to log statistic related messages. +// It is used to sample the log to avoid too many logs. +// NOTE: Do not create a new logger for each log, it will cause the sampler not work. +// Because we need to record the log count with the same level and message in this specific logger. +// Do not use it to log the message that is not related to statistics. +func SingletonStatsSamplerLogger() *zap.Logger { + init := func() { + if samplerLogger == nil { + // Create a new zapcore sampler with options + // This will log the first 2 log entries with the same level and message in a minute and ignore the rest of the logs. + sampler := zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewSamplerWithOptions(core, time.Minute, 2, 0) + }) + samplerLogger = StatsLogger().WithOptions(sampler) + } + } + + initSamplerLoggerOnce.Do(init) + return samplerLogger +} diff --git a/pkg/statistics/handle/storage/json.go b/pkg/statistics/handle/storage/json.go index b934fb8380ddc..f98882f2350b1 100644 --- a/pkg/statistics/handle/storage/json.go +++ b/pkg/statistics/handle/storage/json.go @@ -140,7 +140,8 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *u Indices: make(map[int64]*statistics.Index, len(jsonTbl.Indices)), } tbl := &statistics.Table{ - HistColl: newHistColl, + HistColl: newHistColl, + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(len(tableInfo.Columns), len(tableInfo.Indices)), } for id, jsonIdx := range jsonTbl.Indices { for _, idxInfo := range tableInfo.Indices { @@ -172,6 +173,7 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *u tbl.StatsVer = int(statsVer) } tbl.Indices[idx.ID] = idx + tbl.ColAndIdxExistenceMap.InsertIndex(idxInfo.ID, idxInfo, true) } } @@ -224,6 +226,7 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *u tbl.StatsVer = int(statsVer) } tbl.Columns[col.ID] = col + tbl.ColAndIdxExistenceMap.InsertCol(colInfo.ID, colInfo, true) } } tbl.ExtendedStats = extendedStatsFromJSON(jsonTbl.ExtStats) diff --git a/pkg/statistics/handle/storage/read.go b/pkg/statistics/handle/storage/read.go index 5e384202b79f6..504c42461c546 100644 --- a/pkg/statistics/handle/storage/read.go +++ b/pkg/statistics/handle/storage/read.go @@ -16,7 +16,6 @@ package storage import ( "encoding/json" - "fmt" "strconv" "time" @@ -54,6 +53,34 @@ func StatsMetaCountAndModifyCount(sctx sessionctx.Context, tableID int64) (count return count, modifyCount, false, nil } +// HistMetaFromStorage reads the meta info of the histogram from the storage. +func HistMetaFromStorage(sctx sessionctx.Context, item *model.TableItemID, possibleColInfo *model.ColumnInfo) (*statistics.Histogram, *types.Datum, int64, int64, error) { + isIndex := 0 + var tp *types.FieldType + if item.IsIndex { + isIndex = 1 + tp = types.NewFieldType(mysql.TypeBlob) + } else { + tp = &possibleColInfo.FieldType + } + rows, _, err := util.ExecRows(sctx, + "select distinct_count, version, null_count, tot_col_size, stats_ver, correlation, flag, last_analyze_pos from mysql.stats_histograms where table_id = %? and hist_id = %? and is_index = %?", + item.TableID, + item.ID, + isIndex, + ) + if err != nil { + return nil, nil, 0, 0, err + } + if len(rows) == 0 { + return nil, nil, 0, 0, nil + } + hist := statistics.NewHistogram(item.ID, rows[0].GetInt64(0), rows[0].GetInt64(2), rows[0].GetUint64(1), tp, chunk.InitialCapacity, rows[0].GetInt64(3)) + hist.Correlation = rows[0].GetFloat64(5) + lastPos := rows[0].GetDatum(7, types.NewFieldType(mysql.TypeBlob)) + return hist, &lastPos, rows[0].GetInt64(4), rows[0].GetInt64(6), nil +} + // HistogramFromStorage reads histogram from storage. func HistogramFromStorage(sctx sessionctx.Context, tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64) (_ *statistics.Histogram, err error) { rows, fields, err := util.ExecRows(sctx, "select count, repeats, lower_bound, upper_bound, ndv from mysql.stats_buckets where table_id = %? and is_index = %? and hist_id = %? order by bucket_id", tableID, isIndex, colID) @@ -236,10 +263,12 @@ func indexStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *statis if histID != idxInfo.ID { continue } + table.ColAndIdxExistenceMap.InsertIndex(idxInfo.ID, idxInfo, statsVer != statistics.Version0) // All the objects in the table shares the same stats version. // Update here. if statsVer != statistics.Version0 { table.StatsVer = int(statsVer) + table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, histVer) } // We will not load buckets, topn and cmsketch if: // 1. lease > 0, and: @@ -251,6 +280,10 @@ func indexStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *statis !loadAll && config.GetGlobalConfig().Performance.LiteInitStats if notNeedLoad { + // If we don't have this index in memory, skip it. + if idx == nil { + return nil + } idx = &statistics.Index{ Histogram: *statistics.NewHistogram(histID, distinct, nullCount, histVer, types.NewFieldType(mysql.TypeBlob), 0, 0), StatsVer: statsVer, @@ -303,9 +336,6 @@ func indexStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *statis if tracker != nil { tracker.Consume(idx.MemoryUsage().TotalMemoryUsage()) } - if idx.IsAnalyzed() { - table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, idx.LastUpdateVersion) - } table.Indices[histID] = idx } else { logutil.BgLogger().Debug("we cannot find index id in table info. It may be deleted.", zap.Int64("indexID", histID), zap.String("table", tableInfo.Name.O)) @@ -329,10 +359,12 @@ func columnStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *stati if histID != colInfo.ID { continue } + table.ColAndIdxExistenceMap.InsertCol(histID, colInfo, statsVer != statistics.Version0 || distinct > 0 || nullCount > 0) // All the objects in the table shares the same stats version. // Update here. if statsVer != statistics.Version0 { table.StatsVer = int(statsVer) + table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, histVer) } isHandle := tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()) // We will not load buckets, topn and cmsketch if: @@ -356,6 +388,10 @@ func columnStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *stati (col == nil || ((!col.IsStatsInitialized() || col.IsAllEvicted()) && col.LastUpdateVersion < histVer)) && !loadAll if notNeedLoad { + // If we don't have the column in memory currently, just skip it. + if col == nil { + return nil + } col = &statistics.Column{ PhysicalID: table.PhysicalID, Histogram: *statistics.NewHistogram(histID, distinct, nullCount, histVer, &colInfo.FieldType, 0, totColSize), @@ -417,9 +453,6 @@ func columnStatsFromStorage(sctx sessionctx.Context, row chunk.Row, table *stati if tracker != nil { tracker.Consume(col.MemoryUsage().TotalMemoryUsage()) } - if col.IsAnalyzed() { - table.LastAnalyzeVersion = max(table.LastAnalyzeVersion, col.LastUpdateVersion) - } table.Columns[col.ID] = col } else { // If we didn't find a Column or Index in tableInfo, we won't load the histogram for it. @@ -441,11 +474,12 @@ func TableStatsFromStorage(sctx sessionctx.Context, snapshot uint64, tableInfo * histColl := statistics.HistColl{ PhysicalID: tableID, HavePhysicalID: true, - Columns: make(map[int64]*statistics.Column, len(tableInfo.Columns)), - Indices: make(map[int64]*statistics.Index, len(tableInfo.Indices)), + Columns: make(map[int64]*statistics.Column, 4), + Indices: make(map[int64]*statistics.Index, 4), } table = &statistics.Table{ - HistColl: histColl, + HistColl: histColl, + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(len(tableInfo.Columns), len(tableInfo.Indices)), } } else { // We copy it before writing to avoid race. @@ -524,17 +558,49 @@ func LoadNeededHistograms(sctx sessionctx.Context, statsCache statstypes.StatsCa return nil } +// CleanFakeItemsForShowHistInFlights cleans the invalid inserted items. +func CleanFakeItemsForShowHistInFlights(statsCache statstypes.StatsCache) int { + items := statistics.HistogramNeededItems.AllItems() + reallyNeeded := 0 + for _, item := range items { + tbl, ok := statsCache.Get(item.TableID) + if !ok { + statistics.HistogramNeededItems.Delete(item) + continue + } + loadNeeded := false + if item.IsIndex { + _, loadNeeded = tbl.IndexIsLoadNeeded(item.ID) + } else { + _, loadNeeded = tbl.ColumnIsLoadNeeded(item.ID) + } + if !loadNeeded { + statistics.HistogramNeededItems.Delete(item) + continue + } + reallyNeeded++ + } + return reallyNeeded +} + func loadNeededColumnHistograms(sctx sessionctx.Context, statsCache statstypes.StatsCache, col model.TableItemID, loadFMSketch bool) (err error) { tbl, ok := statsCache.Get(col.TableID) if !ok { return nil } - c, ok := tbl.Columns[col.ID] - if !ok || !c.IsLoadNeeded() { + var colInfo *model.ColumnInfo + c, loadNeeded := tbl.ColumnIsLoadNeeded(col.ID) + if !loadNeeded { statistics.HistogramNeededItems.Delete(col) return nil } - hg, err := HistogramFromStorage(sctx, col.TableID, c.ID, &c.Info.FieldType, c.Histogram.NDV, 0, c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation) + colInfo = tbl.ColAndIdxExistenceMap.GetCol(col.ID) + hgMeta, _, statsVer, _, err := HistMetaFromStorage(sctx, &col, colInfo) + if hgMeta == nil || err != nil { + statistics.HistogramNeededItems.Delete(col) + return err + } + hg, err := HistogramFromStorage(sctx, col.TableID, col.ID, &colInfo.FieldType, hgMeta.NDV, 0, hgMeta.LastUpdateVersion, hgMeta.NullCount, hgMeta.TotColSize, hgMeta.Correlation) if err != nil { return errors.Trace(err) } @@ -549,23 +615,14 @@ func loadNeededColumnHistograms(sctx sessionctx.Context, statsCache statstypes.S return errors.Trace(err) } } - rows, _, err := util.ExecRows(sctx, "select stats_ver from mysql.stats_histograms where is_index = 0 and table_id = %? and hist_id = %?", col.TableID, col.ID) - if err != nil { - return errors.Trace(err) - } - if len(rows) == 0 { - logutil.BgLogger().Error("fail to get stats version for this histogram", zap.Int64("table_id", col.TableID), zap.Int64("hist_id", col.ID)) - return errors.Trace(fmt.Errorf("fail to get stats version for this histogram, table_id:%v, hist_id:%v", col.TableID, col.ID)) - } - statsVer := rows[0].GetInt64(0) colHist := &statistics.Column{ PhysicalID: col.TableID, Histogram: *hg, - Info: c.Info, + Info: colInfo, CMSketch: cms, TopN: topN, FMSketch: fms, - IsHandle: c.IsHandle, + IsHandle: tbl.IsPkIsHandle && mysql.HasPriKeyFlag(colInfo.GetFlag()), StatsVer: statsVer, } // Reload the latest stats cache, otherwise the `updateStatsCache` may fail with high probability, because functions @@ -582,9 +639,15 @@ func loadNeededColumnHistograms(sctx sessionctx.Context, statsCache statstypes.S tbl.StatsVer = int(statsVer) } } - tbl.Columns[c.ID] = colHist + tbl.Columns[col.ID] = colHist statsCache.UpdateStatsCache([]*statistics.Table{tbl}, nil) statistics.HistogramNeededItems.Delete(col) + if col.IsSyncLoadFailed { + logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.", + zap.Int64("table_id", c.PhysicalID), + zap.Int64("column_id", c.Info.ID), + zap.String("column_name", c.Info.Name.O)) + } return nil } @@ -593,12 +656,18 @@ func loadNeededIndexHistograms(sctx sessionctx.Context, statsCache statstypes.St if !ok { return nil } - index, ok := tbl.Indices[idx.ID] - if !ok { + _, loadNeeded := tbl.IndexIsLoadNeeded(idx.ID) + if !loadNeeded { statistics.HistogramNeededItems.Delete(idx) return nil } - hg, err := HistogramFromStorage(sctx, idx.TableID, index.ID, types.NewFieldType(mysql.TypeBlob), index.Histogram.NDV, 1, index.LastUpdateVersion, index.NullCount, index.TotColSize, index.Correlation) + hgMeta, lastAnalyzePos, statsVer, flag, err := HistMetaFromStorage(sctx, &idx, nil) + if hgMeta == nil || err != nil { + statistics.HistogramNeededItems.Delete(idx) + return err + } + idxInfo := tbl.ColAndIdxExistenceMap.GetIndex(idx.ID) + hg, err := HistogramFromStorage(sctx, idx.TableID, idx.ID, types.NewFieldType(mysql.TypeBlob), hgMeta.NDV, 1, hgMeta.LastUpdateVersion, hgMeta.NullCount, hgMeta.TotColSize, hgMeta.Correlation) if err != nil { return errors.Trace(err) } @@ -613,19 +682,11 @@ func loadNeededIndexHistograms(sctx sessionctx.Context, statsCache statstypes.St return errors.Trace(err) } } - rows, _, err := util.ExecRows(sctx, "select stats_ver from mysql.stats_histograms where is_index = 1 and table_id = %? and hist_id = %?", idx.TableID, idx.ID) - if err != nil { - return errors.Trace(err) - } - if len(rows) == 0 { - logutil.BgLogger().Error("fail to get stats version for this histogram", zap.Int64("table_id", idx.TableID), zap.Int64("hist_id", idx.ID)) - return errors.Trace(fmt.Errorf("fail to get stats version for this histogram, table_id:%v, hist_id:%v", idx.TableID, idx.ID)) - } idxHist := &statistics.Index{Histogram: *hg, CMSketch: cms, TopN: topN, FMSketch: fms, - Info: index.Info, StatsVer: rows[0].GetInt64(0), - Flag: index.Flag, PhysicalID: idx.TableID, + Info: idxInfo, StatsVer: statsVer, + Flag: flag, PhysicalID: idx.TableID, StatsLoadedStatus: statistics.NewStatsFullLoadStatus()} - index.LastAnalyzePos.Copy(&idxHist.LastAnalyzePos) + lastAnalyzePos.Copy(&idxHist.LastAnalyzePos) tbl, ok = statsCache.Get(idx.TableID) if !ok { @@ -638,6 +699,12 @@ func loadNeededIndexHistograms(sctx sessionctx.Context, statsCache statstypes.St tbl.Indices[idx.ID] = idxHist tbl.LastAnalyzeVersion = max(tbl.LastAnalyzeVersion, idxHist.LastUpdateVersion) statsCache.UpdateStatsCache([]*statistics.Table{tbl}, nil) + if idx.IsSyncLoadFailed { + logutil.BgLogger().Warn("Hist for column should already be loaded as sync but not found.", + zap.Int64("table_id", tbl.PhysicalID), + zap.Int64("column_id", idxHist.Info.ID), + zap.String("column_name", idxHist.Info.Name.O)) + } statistics.HistogramNeededItems.Delete(idx) return nil } diff --git a/pkg/statistics/handle/storage/read_test.go b/pkg/statistics/handle/storage/read_test.go index 0413fffdba81f..e24d50559e445 100644 --- a/pkg/statistics/handle/storage/read_test.go +++ b/pkg/statistics/handle/storage/read_test.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/planner/cardinality" + "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" "github.com/stretchr/testify/require" @@ -52,21 +53,17 @@ func TestLoadStats(t *testing.T) { // Index/column stats are not be loaded after analyze. stat := h.GetTableStats(tableInfo) require.True(t, stat.Columns[colAID].IsAllEvicted()) - hg := stat.Columns[colAID].Histogram - require.Equal(t, hg.Len(), 0) - cms := stat.Columns[colAID].CMSketch - require.Nil(t, cms) + c, ok := stat.Columns[colAID] + require.True(t, !ok || c.Histogram.Len() == 0) + require.True(t, !ok || c.CMSketch == nil) require.True(t, stat.Indices[idxBID].IsAllEvicted()) - hg = stat.Indices[idxBID].Histogram - require.Equal(t, hg.Len(), 0) - cms = stat.Indices[idxBID].CMSketch - topN := stat.Indices[idxBID].TopN - require.Equal(t, cms.TotalCount()+topN.TotalCount(), uint64(0)) + idx, ok := stat.Indices[idxBID] + require.True(t, !ok || idx.Histogram.Len() == 0) + require.True(t, !ok || (idx.CMSketch.TotalCount()+idx.TopN.TotalCount() == 0)) require.True(t, stat.Columns[colCID].IsAllEvicted()) - hg = stat.Columns[colCID].Histogram - require.Equal(t, 0, hg.Len()) - cms = stat.Columns[colCID].CMSketch - require.Nil(t, cms) + c, ok = stat.Columns[colCID] + require.True(t, !ok || c.Histogram.Len() == 0) + require.True(t, !ok || c.CMSketch == nil) // Column stats are loaded after they are needed. _, err = cardinality.ColumnEqualRowCount(testKit.Session().GetPlanCtx(), stat, types.NewIntDatum(1), colAID) @@ -76,10 +73,10 @@ func TestLoadStats(t *testing.T) { require.NoError(t, h.LoadNeededHistograms()) stat = h.GetTableStats(tableInfo) require.True(t, stat.Columns[colAID].IsFullLoad()) - hg = stat.Columns[colAID].Histogram + hg := stat.Columns[colAID].Histogram require.Greater(t, hg.Len(), 0) // We don't maintain cmsketch for pk. - cms = stat.Columns[colAID].CMSketch + cms := stat.Columns[colAID].CMSketch require.Nil(t, cms) require.True(t, stat.Columns[colCID].IsFullLoad()) hg = stat.Columns[colCID].Histogram @@ -88,20 +85,17 @@ func TestLoadStats(t *testing.T) { require.NotNil(t, cms) // Index stats are loaded after they are needed. - idx := stat.Indices[idxBID] - hg = idx.Histogram - cms = idx.CMSketch - topN = idx.TopN - require.Equal(t, float64(cms.TotalCount()+topN.TotalCount())+hg.TotalRowCount(), float64(0)) - require.False(t, idx.IsEssentialStatsLoaded()) + idx, ok = stat.Indices[idxBID] + require.True(t, !ok || (float64(idx.CMSketch.TotalCount())+float64(idx.TopN.TotalCount())+idx.Histogram.TotalRowCount() == 0)) + require.False(t, ok && idx.IsEssentialStatsLoaded()) // IsInvalid adds the index to HistogramNeededItems. - idx.IsInvalid(testKit.Session().GetPlanCtx(), false) + statistics.IndexStatsIsInvalid(testKit.Session().GetPlanCtx(), idx, &stat.HistColl, idxBID) require.NoError(t, h.LoadNeededHistograms()) stat = h.GetTableStats(tableInfo) idx = stat.Indices[tableInfo.Indices[0].ID] hg = idx.Histogram cms = idx.CMSketch - topN = idx.TopN + topN := idx.TopN require.Greater(t, float64(cms.TotalCount()+topN.TotalCount())+hg.TotalRowCount(), float64(0)) require.True(t, idx.IsFullLoad()) } diff --git a/pkg/statistics/handle/storage/stats_read_writer.go b/pkg/statistics/handle/storage/stats_read_writer.go index c2fd6fdb0e2af..4a479a0a820ae 100644 --- a/pkg/statistics/handle/storage/stats_read_writer.go +++ b/pkg/statistics/handle/storage/stats_read_writer.go @@ -262,8 +262,8 @@ func (s *statsReadWriter) SaveStatsToStorage( return } -// saveMetaToStorage saves stats meta to the storage. -func (s *statsReadWriter) saveMetaToStorage(tableID, count, modifyCount int64, source string) (err error) { +// SaveMetaToStorage saves stats meta to the storage. +func (s *statsReadWriter) SaveMetaToStorage(tableID, count, modifyCount int64, source string) (err error) { var statsVer uint64 err = util.CallWithSCtx(s.statsHandler.SPool(), func(sctx sessionctx.Context) error { statsVer, err = SaveMetaToStorage(sctx, tableID, count, modifyCount) @@ -708,5 +708,5 @@ func (s *statsReadWriter) loadStatsFromJSON(tableInfo *model.TableInfo, physical if err != nil { return errors.Trace(err) } - return s.saveMetaToStorage(tbl.PhysicalID, tbl.RealtimeCount, tbl.ModifyCount, util.StatsMetaHistorySourceLoadStats) + return s.SaveMetaToStorage(tbl.PhysicalID, tbl.RealtimeCount, tbl.ModifyCount, util.StatsMetaHistorySourceLoadStats) } diff --git a/pkg/statistics/handle/syncload/BUILD.bazel b/pkg/statistics/handle/syncload/BUILD.bazel index 901964873fda2..365fadbfeb6ab 100644 --- a/pkg/statistics/handle/syncload/BUILD.bazel +++ b/pkg/statistics/handle/syncload/BUILD.bazel @@ -15,7 +15,6 @@ go_library( "//pkg/statistics", "//pkg/statistics/handle/storage", "//pkg/statistics/handle/types", - "//pkg/statistics/handle/util", "//pkg/types", "//pkg/util", "//pkg/util/logutil", diff --git a/pkg/statistics/handle/syncload/stats_syncload.go b/pkg/statistics/handle/syncload/stats_syncload.go index ad3b4f5706586..aaa11f9c9ac71 100644 --- a/pkg/statistics/handle/syncload/stats_syncload.go +++ b/pkg/statistics/handle/syncload/stats_syncload.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle/storage" statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" - utilstats "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/logutil" @@ -54,8 +53,10 @@ func NewStatsSyncLoad(statsHandle statstypes.StatsHandle) statstypes.StatsSyncLo } type statsWrapper struct { - col *statistics.Column - idx *statistics.Index + colInfo *model.ColumnInfo + idxInfo *model.IndexInfo + col *statistics.Column + idx *statistics.Index } // SetSubCtxs sets the sessionctx which is used to run queries background. @@ -65,7 +66,7 @@ func (s *statsSyncLoad) SetSubCtxs(idx int, sctx sessionctx.Context) { } // SendLoadRequests send neededColumns requests -func (s *statsSyncLoad) SendLoadRequests(sc *stmtctx.StatementContext, neededHistItems []model.TableItemID, timeout time.Duration) error { +func (s *statsSyncLoad) SendLoadRequests(sc *stmtctx.StatementContext, neededHistItems []model.StatsLoadItem, timeout time.Duration) error { remainedItems := s.removeHistLoadedColumns(neededHistItems) failpoint.Inject("assertSyncLoadItems", func(val failpoint.Value) { @@ -86,9 +87,9 @@ func (s *statsSyncLoad) SendLoadRequests(sc *stmtctx.StatementContext, neededHis tasks := make([]*statstypes.NeededItemTask, 0) for _, item := range remainedItems { task := &statstypes.NeededItemTask{ - TableItemID: item, - ToTimeout: time.Now().Local().Add(timeout), - ResultCh: sc.StatsLoad.ResultCh, + Item: item, + ToTimeout: time.Now().Local().Add(timeout), + ResultCh: sc.StatsLoad.ResultCh, } tasks = append(tasks, task) } @@ -121,7 +122,7 @@ func (*statsSyncLoad) SyncWaitStatsLoad(sc *stmtctx.StatementContext) error { }() resultCheckMap := map[model.TableItemID]struct{}{} for _, col := range sc.StatsLoad.NeededItems { - resultCheckMap[col] = struct{}{} + resultCheckMap[col.TableItemID] = struct{}{} } metrics.SyncLoadCounter.Inc() timer := time.NewTimer(sc.StatsLoad.Timeout) @@ -148,8 +149,8 @@ func (*statsSyncLoad) SyncWaitStatsLoad(sc *stmtctx.StatementContext) error { } // removeHistLoadedColumns removed having-hist columns based on neededColumns and statsCache. -func (s *statsSyncLoad) removeHistLoadedColumns(neededItems []model.TableItemID) []model.TableItemID { - remainedItems := make([]model.TableItemID, 0, len(neededItems)) +func (s *statsSyncLoad) removeHistLoadedColumns(neededItems []model.StatsLoadItem) []model.StatsLoadItem { + remainedItems := make([]model.StatsLoadItem, 0, len(neededItems)) for _, item := range neededItems { tbl, ok := s.statsHandle.Get(item.TableID) if !ok { @@ -159,8 +160,8 @@ func (s *statsSyncLoad) removeHistLoadedColumns(neededItems []model.TableItemID) remainedItems = append(remainedItems, item) continue } - colHist, ok := tbl.Columns[item.ID] - if ok && colHist.IsStatsInitialized() && !colHist.IsFullLoad() { + _, loadNeeded := tbl.ColumnIsLoadNeeded(item.ID) + if loadNeeded { remainedItems = append(remainedItems, item) } } @@ -231,7 +232,7 @@ func (s *statsSyncLoad) HandleOneTask(sctx sessionctx.Context, lastTask *statsty } func (s *statsSyncLoad) handleOneItemTask(sctx sessionctx.Context, task *statstypes.NeededItemTask) (*statstypes.NeededItemTask, error) { - result := stmtctx.StatsLoadResult{Item: task.TableItemID} + result := stmtctx.StatsLoadResult{Item: task.Item.TableItemID} item := result.Item tbl, ok := s.statsHandle.Get(item.TableID) if !ok { @@ -241,19 +242,27 @@ func (s *statsSyncLoad) handleOneItemTask(sctx sessionctx.Context, task *statsty var err error wrapper := &statsWrapper{} if item.IsIndex { - index, ok := tbl.Indices[item.ID] - if !ok || index.IsFullLoad() { + index, loadNeeded := tbl.IndexIsLoadNeeded(item.ID) + if !loadNeeded { s.writeToResultChan(task.ResultCh, result) return nil, nil } - wrapper.idx = index + if index != nil { + wrapper.idxInfo = index.Info + } else { + wrapper.idxInfo = tbl.ColAndIdxExistenceMap.GetIndex(item.ID) + } } else { - col, ok := tbl.Columns[item.ID] - if !ok || col.IsFullLoad() { + col, loadNeeded := tbl.ColumnIsLoadNeeded(item.ID) + if !loadNeeded { s.writeToResultChan(task.ResultCh, result) return nil, nil } - wrapper.col = col + if col != nil { + wrapper.colInfo = col.Info + } else { + wrapper.colInfo = tbl.ColAndIdxExistenceMap.GetCol(item.ID) + } } // to avoid duplicated handling in concurrent scenario working := s.setWorking(result.Item, task.ResultCh) @@ -263,22 +272,22 @@ func (s *statsSyncLoad) handleOneItemTask(sctx sessionctx.Context, task *statsty } t := time.Now() needUpdate := false - wrapper, err = s.readStatsForOneItem(sctx, item, wrapper) + wrapper, err = s.readStatsForOneItem(sctx, item, wrapper, tbl.IsPkIsHandle, task.Item.FullLoad) if err != nil { result.Error = err return task, err } if item.IsIndex { - if wrapper.idx != nil { + if wrapper.idxInfo != nil { needUpdate = true } } else { - if wrapper.col != nil { + if wrapper.colInfo != nil { needUpdate = true } } metrics.ReadStatsHistogram.Observe(float64(time.Since(t).Milliseconds())) - if needUpdate && s.updateCachedItem(item, wrapper.col, wrapper.idx) { + if needUpdate && s.updateCachedItem(item, wrapper.col, wrapper.idx, task.Item.FullLoad) { s.writeToResultChan(task.ResultCh, result) } s.finishWorking(result) @@ -286,85 +295,92 @@ func (s *statsSyncLoad) handleOneItemTask(sctx sessionctx.Context, task *statsty } // readStatsForOneItem reads hist for one column/index, TODO load data via kv-get asynchronously -func (*statsSyncLoad) readStatsForOneItem(sctx sessionctx.Context, item model.TableItemID, w *statsWrapper) (*statsWrapper, error) { +func (*statsSyncLoad) readStatsForOneItem(sctx sessionctx.Context, item model.TableItemID, w *statsWrapper, isPkIsHandle bool, fullLoad bool) (*statsWrapper, error) { failpoint.Inject("mockReadStatsForOnePanic", nil) failpoint.Inject("mockReadStatsForOneFail", func(val failpoint.Value) { if val.(bool) { failpoint.Return(nil, errors.New("gofail ReadStatsForOne error")) } }) - c := w.col - index := w.idx loadFMSketch := config.GetGlobalConfig().Performance.EnableLoadFMSketch var hg *statistics.Histogram var err error isIndexFlag := int64(0) - if item.IsIndex { - isIndexFlag = 1 + hg, lastAnalyzePos, statsVer, flag, err := storage.HistMetaFromStorage(sctx, &item, w.colInfo) + if err != nil { + return nil, err + } + if hg == nil { + logutil.BgLogger().Error("fail to get hist meta for this histogram, possibly a deleted one", zap.Int64("table_id", item.TableID), + zap.Int64("hist_id", item.ID), zap.Bool("is_index", item.IsIndex)) + return nil, errors.Trace(fmt.Errorf("fail to get hist meta for this histogram, table_id:%v, hist_id:%v, is_index:%v", item.TableID, item.ID, item.IsIndex)) } if item.IsIndex { - hg, err = storage.HistogramFromStorage(sctx, item.TableID, item.ID, types.NewFieldType(mysql.TypeBlob), index.Histogram.NDV, int(isIndexFlag), index.LastUpdateVersion, index.NullCount, index.TotColSize, index.Correlation) - if err != nil { - return nil, errors.Trace(err) - } - } else { - hg, err = storage.HistogramFromStorage(sctx, item.TableID, item.ID, &c.Info.FieldType, c.Histogram.NDV, int(isIndexFlag), c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation) - if err != nil { - return nil, errors.Trace(err) - } + isIndexFlag = 1 } var cms *statistics.CMSketch var topN *statistics.TopN - cms, topN, err = storage.CMSketchAndTopNFromStorage(sctx, item.TableID, isIndexFlag, item.ID) - if err != nil { - return nil, errors.Trace(err) - } var fms *statistics.FMSketch - if loadFMSketch { - fms, err = storage.FMSketchFromStorage(sctx, item.TableID, isIndexFlag, item.ID) + if fullLoad { + if item.IsIndex { + hg, err = storage.HistogramFromStorage(sctx, item.TableID, item.ID, types.NewFieldType(mysql.TypeBlob), hg.NDV, int(isIndexFlag), hg.LastUpdateVersion, hg.NullCount, hg.TotColSize, hg.Correlation) + if err != nil { + return nil, errors.Trace(err) + } + } else { + hg, err = storage.HistogramFromStorage(sctx, item.TableID, item.ID, &w.colInfo.FieldType, hg.NDV, int(isIndexFlag), hg.LastUpdateVersion, hg.NullCount, hg.TotColSize, hg.Correlation) + if err != nil { + return nil, errors.Trace(err) + } + } + cms, topN, err = storage.CMSketchAndTopNFromStorage(sctx, item.TableID, isIndexFlag, item.ID) if err != nil { return nil, errors.Trace(err) } + if loadFMSketch { + fms, err = storage.FMSketchFromStorage(sctx, item.TableID, isIndexFlag, item.ID) + if err != nil { + return nil, errors.Trace(err) + } + } } - rows, _, err := utilstats.ExecRows(sctx, "select stats_ver from mysql.stats_histograms where table_id = %? and hist_id = %? and is_index = %?", item.TableID, item.ID, int(isIndexFlag)) - if err != nil { - return nil, errors.Trace(err) - } - if len(rows) == 0 { - logutil.BgLogger().Error("fail to get stats version for this histogram", zap.Int64("table_id", item.TableID), - zap.Int64("hist_id", item.ID), zap.Bool("is_index", item.IsIndex)) - return nil, errors.Trace(fmt.Errorf("fail to get stats version for this histogram, table_id:%v, hist_id:%v, is_index:%v", item.TableID, item.ID, item.IsIndex)) - } - statsVer := rows[0].GetInt64(0) if item.IsIndex { idxHist := &statistics.Index{ Histogram: *hg, CMSketch: cms, TopN: topN, FMSketch: fms, - Info: index.Info, + Info: w.idxInfo, StatsVer: statsVer, - Flag: index.Flag, - PhysicalID: index.PhysicalID, + Flag: flag, + PhysicalID: item.TableID, } if statsVer != statistics.Version0 { - idxHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() + if fullLoad { + idxHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() + } else { + idxHist.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus() + } } - index.LastAnalyzePos.Copy(&idxHist.LastAnalyzePos) + lastAnalyzePos.Copy(&idxHist.LastAnalyzePos) w.idx = idxHist } else { colHist := &statistics.Column{ PhysicalID: item.TableID, Histogram: *hg, - Info: c.Info, + Info: w.colInfo, CMSketch: cms, TopN: topN, FMSketch: fms, - IsHandle: c.IsHandle, + IsHandle: isPkIsHandle && mysql.HasPriKeyFlag(w.colInfo.GetFlag()), StatsVer: statsVer, } if colHist.StatsAvailable() { - colHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() + if fullLoad { + colHist.StatsLoadedStatus = statistics.NewStatsFullLoadStatus() + } else { + colHist.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus() + } } w.col = colHist } @@ -445,7 +461,7 @@ func (*statsSyncLoad) writeToResultChan(resultCh chan stmtctx.StatsLoadResult, r } // updateCachedItem updates the column/index hist to global statsCache. -func (s *statsSyncLoad) updateCachedItem(item model.TableItemID, colHist *statistics.Column, idxHist *statistics.Index) (updated bool) { +func (s *statsSyncLoad) updateCachedItem(item model.TableItemID, colHist *statistics.Column, idxHist *statistics.Index, fullLoaded bool) (updated bool) { s.StatsLoad.Lock() defer s.StatsLoad.Unlock() // Reload the latest stats cache, otherwise the `updateStatsCache` may fail with high probability, because functions @@ -456,24 +472,34 @@ func (s *statsSyncLoad) updateCachedItem(item model.TableItemID, colHist *statis } if !item.IsIndex && colHist != nil { c, ok := tbl.Columns[item.ID] - if !ok || c.IsFullLoad() { + // - If the stats is fully loaded, + // - If the stats is meta-loaded and we also just need the meta. + if ok && (c.IsFullLoad() || !fullLoaded) { return true } tbl = tbl.Copy() - tbl.Columns[c.ID] = colHist + tbl.Columns[item.ID] = colHist + // If the column is analyzed we refresh the map for the possible change. + if colHist.StatsAvailable() { + tbl.ColAndIdxExistenceMap.InsertCol(item.ID, colHist.Info, true) + } // All the objects shares the same stats version. Update it here. if colHist.StatsVer != statistics.Version0 { tbl.StatsVer = statistics.Version0 } } else if item.IsIndex && idxHist != nil { index, ok := tbl.Indices[item.ID] - if !ok || index.IsFullLoad() { + // - If the stats is fully loaded, + // - If the stats is meta-loaded and we also just need the meta. + if ok && (index.IsFullLoad() || !fullLoaded) { return true } tbl = tbl.Copy() tbl.Indices[item.ID] = idxHist - // All the objects shares the same stats version. Update it here. - if idxHist.StatsVer != statistics.Version0 { + // If the index is analyzed we refresh the map for the possible change. + if idxHist.IsAnalyzed() { + tbl.ColAndIdxExistenceMap.InsertIndex(item.ID, idxHist.Info, true) + // All the objects shares the same stats version. Update it here. tbl.StatsVer = statistics.Version0 } } diff --git a/pkg/statistics/handle/syncload/stats_syncload_test.go b/pkg/statistics/handle/syncload/stats_syncload_test.go index 356dda7e6dd2d..2c2768bc4f849 100644 --- a/pkg/statistics/handle/syncload/stats_syncload_test.go +++ b/pkg/statistics/handle/syncload/stats_syncload_test.go @@ -73,24 +73,22 @@ func TestConcurrentLoadHist(t *testing.T) { tableInfo := tbl.Meta() h := dom.StatsHandle() stat := h.GetTableStats(tableInfo) - hg := stat.Columns[tableInfo.Columns[0].ID].Histogram - topn := stat.Columns[tableInfo.Columns[0].ID].TopN - require.Equal(t, 0, hg.Len()+topn.Num()) - hg = stat.Columns[tableInfo.Columns[2].ID].Histogram - topn = stat.Columns[tableInfo.Columns[2].ID].TopN - require.Equal(t, 0, hg.Len()+topn.Num()) + col, ok := stat.Columns[tableInfo.Columns[0].ID] + require.True(t, !ok || col.Histogram.Len()+col.TopN.Num() == 0) + col, ok = stat.Columns[tableInfo.Columns[2].ID] + require.True(t, !ok || col.Histogram.Len()+col.TopN.Num() == 0) stmtCtx := stmtctx.NewStmtCtx() - neededColumns := make([]model.TableItemID, 0, len(tableInfo.Columns)) + neededColumns := make([]model.StatsLoadItem, 0, len(tableInfo.Columns)) for _, col := range tableInfo.Columns { - neededColumns = append(neededColumns, model.TableItemID{TableID: tableInfo.ID, ID: col.ID, IsIndex: false}) + neededColumns = append(neededColumns, model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tableInfo.ID, ID: col.ID, IsIndex: false}, FullLoad: true}) } timeout := time.Nanosecond * mathutil.MaxInt h.SendLoadRequests(stmtCtx, neededColumns, timeout) rs := h.SyncWaitStatsLoad(stmtCtx) require.Nil(t, rs) stat = h.GetTableStats(tableInfo) - hg = stat.Columns[tableInfo.Columns[2].ID].Histogram - topn = stat.Columns[tableInfo.Columns[2].ID].TopN + hg := stat.Columns[tableInfo.Columns[2].ID].Histogram + topn := stat.Columns[tableInfo.Columns[2].ID].TopN require.Greater(t, hg.Len()+topn.Num(), 0) } @@ -118,6 +116,9 @@ func TestConcurrentLoadHistTimeout(t *testing.T) { tableInfo := tbl.Meta() h := dom.StatsHandle() stat := h.GetTableStats(tableInfo) + // TODO: They may need to be empty. Depending on how we operate newly analyzed tables. + // require.Nil(t, stat.Columns[tableInfo.Columns[0].ID]) + // require.Nil(t, stat.Columns[tableInfo.Columns[2].ID]) hg := stat.Columns[tableInfo.Columns[0].ID].Histogram topn := stat.Columns[tableInfo.Columns[0].ID].TopN require.Equal(t, 0, hg.Len()+topn.Num()) @@ -125,16 +126,16 @@ func TestConcurrentLoadHistTimeout(t *testing.T) { topn = stat.Columns[tableInfo.Columns[2].ID].TopN require.Equal(t, 0, hg.Len()+topn.Num()) stmtCtx := stmtctx.NewStmtCtx() - neededColumns := make([]model.TableItemID, 0, len(tableInfo.Columns)) + neededColumns := make([]model.StatsLoadItem, 0, len(tableInfo.Columns)) for _, col := range tableInfo.Columns { - neededColumns = append(neededColumns, model.TableItemID{TableID: tableInfo.ID, ID: col.ID, IsIndex: false}) + neededColumns = append(neededColumns, model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tableInfo.ID, ID: col.ID, IsIndex: false}, FullLoad: true}) } h.SendLoadRequests(stmtCtx, neededColumns, 0) // set timeout to 0 so task will go to timeout channel rs := h.SyncWaitStatsLoad(stmtCtx) require.Error(t, rs) stat = h.GetTableStats(tableInfo) - hg = stat.Columns[tableInfo.Columns[2].ID].Histogram - topn = stat.Columns[tableInfo.Columns[2].ID].TopN + // require.Nil(t, stat.Columns[tableInfo.Columns[2].ID]) + require.NotNil(t, stat.Columns[tableInfo.Columns[2].ID]) require.Equal(t, 0, hg.Len()+topn.Num()) } @@ -166,8 +167,8 @@ func TestConcurrentLoadHistWithPanicAndFail(t *testing.T) { tableInfo := tbl.Meta() h := dom.StatsHandle() - neededColumns := make([]model.TableItemID, 1) - neededColumns[0] = model.TableItemID{TableID: tableInfo.ID, ID: tableInfo.Columns[2].ID, IsIndex: false} + neededColumns := make([]model.StatsLoadItem, 1) + neededColumns[0] = model.StatsLoadItem{TableItemID: model.TableItemID{TableID: tableInfo.ID, ID: tableInfo.Columns[2].ID, IsIndex: false}, FullLoad: true} timeout := time.Nanosecond * mathutil.MaxInt failpoints := []struct { @@ -191,9 +192,8 @@ func TestConcurrentLoadHistWithPanicAndFail(t *testing.T) { // no stats at beginning stat := h.GetTableStats(tableInfo) - hg := stat.Columns[tableInfo.Columns[2].ID].Histogram - topn := stat.Columns[tableInfo.Columns[2].ID].TopN - require.Equal(t, 0, hg.Len()+topn.Num()) + c, ok := stat.Columns[tableInfo.Columns[2].ID] + require.True(t, !ok || (c.Histogram.Len()+c.TopN.Num() == 0)) stmtCtx1 := stmtctx.NewStmtCtx() h.SendLoadRequests(stmtCtx1, neededColumns, timeout) @@ -221,14 +221,14 @@ func TestConcurrentLoadHistWithPanicAndFail(t *testing.T) { rs1, ok1 := <-stmtCtx1.StatsLoad.ResultCh require.True(t, ok1) - require.Equal(t, neededColumns[0], rs1.Item) + require.Equal(t, neededColumns[0].TableItemID, rs1.Item) rs2, ok2 := <-stmtCtx2.StatsLoad.ResultCh require.True(t, ok2) - require.Equal(t, neededColumns[0], rs2.Item) + require.Equal(t, neededColumns[0].TableItemID, rs2.Item) stat = h.GetTableStats(tableInfo) - hg = stat.Columns[tableInfo.Columns[2].ID].Histogram - topn = stat.Columns[tableInfo.Columns[2].ID].TopN + hg := stat.Columns[tableInfo.Columns[2].ID].Histogram + topn := stat.Columns[tableInfo.Columns[2].ID].TopN require.Greater(t, hg.Len()+topn.Num(), 0) } } diff --git a/pkg/statistics/handle/types/interfaces.go b/pkg/statistics/handle/types/interfaces.go index 691f52517d5ad..031760fca7ce9 100644 --- a/pkg/statistics/handle/types/interfaces.go +++ b/pkg/statistics/handle/types/interfaces.go @@ -271,6 +271,9 @@ type StatsReadWriter interface { // SaveTableStatsToStorage saves the stats of a table to storage. SaveTableStatsToStorage(results *statistics.AnalyzeResults, analyzeSnapshot bool, source string) (err error) + // SaveMetaToStorage saves the stats meta of a table to storage. + SaveMetaToStorage(tableID, count, modifyCount int64, source string) (err error) + // InsertColStats2KV inserts columns stats to kv. InsertColStats2KV(physicalID int64, colInfos []*model.ColumnInfo) (err error) @@ -363,9 +366,9 @@ type StatsReadWriter interface { // NeededItemTask represents one needed column/indices with expire time. type NeededItemTask struct { - ToTimeout time.Time - ResultCh chan stmtctx.StatsLoadResult - TableItemID model.TableItemID + ToTimeout time.Time + ResultCh chan stmtctx.StatsLoadResult + Item model.StatsLoadItem } // StatsLoad is used to load stats concurrently @@ -380,7 +383,7 @@ type StatsLoad struct { // StatsSyncLoad implement the sync-load feature. type StatsSyncLoad interface { // SendLoadRequests sends load requests to the channel. - SendLoadRequests(sc *stmtctx.StatementContext, neededHistItems []model.TableItemID, timeout time.Duration) error + SendLoadRequests(sc *stmtctx.StatementContext, neededHistItems []model.StatsLoadItem, timeout time.Duration) error // SyncWaitStatsLoad will wait for the load requests to finish. SyncWaitStatsLoad(sc *stmtctx.StatementContext) error diff --git a/pkg/statistics/handle/updatetest/update_test.go b/pkg/statistics/handle/updatetest/update_test.go index 90d1fdde7408b..ed49d794a01b9 100644 --- a/pkg/statistics/handle/updatetest/update_test.go +++ b/pkg/statistics/handle/updatetest/update_test.go @@ -633,9 +633,7 @@ func TestLoadHistCorrelation(t *testing.T) { h.Clear() require.NoError(t, h.Update(dom.InfoSchema())) result := testKit.MustQuery("show stats_histograms where Table_name = 't'") - // After https://github.com/pingcap/tidb/pull/37444, `show stats_histograms` displays the columns whose hist/topn/cmsketch - // are not loaded and their stats status is allEvicted. - require.Len(t, result.Rows(), 1) + require.Len(t, result.Rows(), 0) testKit.MustExec("explain select * from t where c = 1") require.NoError(t, h.LoadNeededHistograms()) result = testKit.MustQuery("show stats_histograms where Table_name = 't'") diff --git a/pkg/statistics/histogram.go b/pkg/statistics/histogram.go index 662889459d1e4..642110ee2621f 100644 --- a/pkg/statistics/histogram.go +++ b/pkg/statistics/histogram.go @@ -937,7 +937,7 @@ func (hg *Histogram) OutOfRange(val types.Datum) bool { func (hg *Histogram) OutOfRangeRowCount( sctx context.PlanContext, lDatum, rDatum *types.Datum, - modifyCount, histNDV int64, + modifyCount, histNDV int64, increaseFactor float64, ) (result float64) { debugTrace := sctx.GetSessionVars().StmtCtx.EnableOptimizerDebugTrace if debugTrace { @@ -1052,38 +1052,35 @@ func (hg *Histogram) OutOfRangeRowCount( rightPercent = (math.Pow(boundR-actualL, 2) - math.Pow(boundR-actualR, 2)) / math.Pow(histWidth, 2) } - totalPercent := leftPercent*0.5 + rightPercent*0.5 - if totalPercent > 1 { - totalPercent = 1 - } + totalPercent := min(leftPercent*0.5+rightPercent*0.5, 1.0) rowCount = totalPercent * hg.NotNullCount() - // Upper bound logic + // Upper & lower bound logic. + upperBound := rowCount + if histNDV > 0 { + upperBound = hg.NotNullCount() / float64(histNDV) + } allowUseModifyCount := sctx.GetSessionVars().GetOptObjective() != variable.OptObjectiveDeterminate - // Use the modifyCount as the upper bound. Note that modifyCount contains insert, delete and update. So this is - // a rather loose upper bound. - // There are some scenarios where we need to handle out-of-range estimation after both insert and delete happen. - // But we don't know how many increases are in the modifyCount. So we have to use this loose bound to ensure it - // can produce a reasonable results in this scenario. - if rowCount > float64(modifyCount) && allowUseModifyCount { - return float64(modifyCount) - } - - // In OptObjectiveDeterminate mode, we can't rely on the modify count anymore. - // An upper bound is necessary to make the estimation make sense for predicates with bound on only one end, like a > 1. - // But it's impossible to have a reliable upper bound in all cases. - // We use 1/NDV here (only the Histogram part is considered) and it seems reasonable and good enough for now. + if !allowUseModifyCount { - var upperBound float64 - if histNDV > 0 { - upperBound = hg.NotNullCount() / float64(histNDV) - } - if rowCount > upperBound { - return upperBound - } + // In OptObjectiveDeterminate mode, we can't rely on the modify count anymore. + // An upper bound is necessary to make the estimation make sense for predicates with bound on only one end, like a > 1. + // We use 1/NDV here (only the Histogram part is considered) and it seems reasonable and good enough for now. + return min(rowCount, upperBound) + } + + // If the modifyCount is large (compared to original table rows), then any out of range estimate is unreliable. + // Assume at least 1/NDV is returned + if float64(modifyCount) > hg.NotNullCount() && rowCount < upperBound { + rowCount = upperBound + } else if rowCount < upperBound { + // Adjust by increaseFactor if our estimate is low + rowCount *= increaseFactor } - return rowCount + + // Use modifyCount as a final bound + return min(rowCount, float64(modifyCount)) } // Copy deep copies the histogram. diff --git a/pkg/statistics/index.go b/pkg/statistics/index.go index a0a12c9a5df5a..ef124701c7564 100644 --- a/pkg/statistics/index.go +++ b/pkg/statistics/index.go @@ -76,7 +76,7 @@ func (idx *Index) ItemID() int64 { // IsAllEvicted indicates whether all stats evicted func (idx *Index) IsAllEvicted() bool { - return idx.statsInitialized && idx.evictedStatus >= AllEvicted + return idx == nil || (idx.statsInitialized && idx.evictedStatus >= AllEvicted) } // GetEvictedStatus returns the evicted status @@ -127,23 +127,36 @@ func (idx *Index) TotalRowCount() float64 { return idx.Histogram.TotalRowCount() } -// IsInvalid checks if this index is invalid. -func (idx *Index) IsInvalid(sctx context.PlanContext, collPseudo bool) (res bool) { - idx.CheckStats() +// IndexStatsIsInvalid checks whether the index has valid stats or not. +func IndexStatsIsInvalid(sctx context.PlanContext, idxStats *Index, coll *HistColl, cid int64) (res bool) { var totalCount float64 if sctx.GetSessionVars().StmtCtx.EnableOptimizerDebugTrace { debugtrace.EnterContextCommon(sctx) defer func() { debugtrace.RecordAnyValuesWithNames(sctx, "IsInvalid", res, - "CollPseudo", collPseudo, + "CollPseudo", coll.Pseudo, "TotalCount", totalCount, ) debugtrace.LeaveContextCommon(sctx) }() } - totalCount = idx.TotalRowCount() - return (collPseudo) || totalCount == 0 + // If the given index statistics is nil or we found that the index's statistics hasn't been fully loaded, we add this index to NeededItems. + // Also, we need to check that this HistColl has its physical ID and it is permitted to trigger the stats loading. + if (idxStats == nil || !idxStats.IsFullLoad()) && !coll.CanNotTriggerLoad { + HistogramNeededItems.Insert(model.TableItemID{ + TableID: coll.PhysicalID, + ID: cid, + IsIndex: true, + IsSyncLoadFailed: sctx.GetSessionVars().StmtCtx.StatsLoad.Timeout > 0, + }) + // TODO: we can return true here. But need to fix some tests first. + } + if idxStats == nil { + return true + } + totalCount = idxStats.TotalRowCount() + return coll.Pseudo || totalCount == 0 } // EvictAllStats evicts all stats @@ -202,15 +215,6 @@ func (idx *Index) QueryBytes(sctx context.PlanContext, d []byte) (result uint64) return uint64(v) } -// CheckStats will check if the index stats need to be updated. -func (idx *Index) CheckStats() { - // When we are using stats from PseudoTable(), all column/index ID will be -1. - if idx.IsFullLoad() || idx.PhysicalID <= 0 { - return - } - HistogramNeededItems.Insert(model.TableItemID{TableID: idx.PhysicalID, ID: idx.Info.ID, IsIndex: true}) -} - // GetIncreaseFactor get the increase factor to adjust the final estimated count when the table is modified. func (idx *Index) GetIncreaseFactor(realtimeRowCount int64) float64 { columnCount := idx.TotalRowCount() diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index 28a910073640f..aeaba2909ed4c 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -482,6 +482,46 @@ func TestIssue44369(t *testing.T) { tk.MustExec("select * from t where a = 10 and bb > 20;") } +// Test the case that after ALTER TABLE happens, the pointer to the column info/index info should be refreshed. +func TestColAndIdxExistenceMapChangedAfterAlterTable(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + h := dom.StatsHandle() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a int, b int, index iab(a,b));") + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + tk.MustExec("insert into t value(1,1);") + require.NoError(t, h.DumpStatsDeltaToKV(true)) + tk.MustExec("analyze table t;") + is := dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo := tbl.Meta() + statsTbl := h.GetTableStats(tblInfo) + colA := tblInfo.Columns[0] + colInfo := statsTbl.ColAndIdxExistenceMap.GetCol(colA.ID) + require.Equal(t, colA, colInfo) + + tk.MustExec("alter table t modify column a double") + require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh())) + is = dom.InfoSchema() + require.NoError(t, h.Update(is)) + tbl, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo = tbl.Meta() + newColA := tblInfo.Columns[0] + require.NotEqual(t, colA.ID, newColA.ID) + statsTbl = h.GetTableStats(tblInfo) + colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) + require.Equal(t, newColA, colInfo) + tk.MustExec("analyze table t;") + require.NoError(t, h.Update(is)) + statsTbl = h.GetTableStats(tblInfo) + colInfo = statsTbl.ColAndIdxExistenceMap.GetCol(newColA.ID) + require.Equal(t, newColA, colInfo) +} + func TestTableLastAnalyzeVersion(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) h := dom.StatsHandle() diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 844729826e67d..5caed8205aceb 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -59,7 +59,9 @@ var ( // Table represents statistics for a table. type Table struct { ExtendedStats *ExtendedStatsColl - Name string + + ColAndIdxExistenceMap *ColAndIdxExistenceMap + Name string HistColl Version uint64 // It's the timestamp of the last analyze time. @@ -70,6 +72,117 @@ type Table struct { // and the schema of the table does not change, we don't need to load the stats for this // table again. TblInfoUpdateTS uint64 + + IsPkIsHandle bool +} + +// ColAndIdxExistenceMap is the meta map for statistics.Table. +// It can tell whether a column/index really has its statistics. So we won't send useless kv request when we do online stats loading. +type ColAndIdxExistenceMap struct { + colInfoMap map[int64]*model.ColumnInfo + colAnalyzed map[int64]bool + idxInfoMap map[int64]*model.IndexInfo + idxAnalyzed map[int64]bool +} + +// SomeAnalyzed checks whether some part of the table is analyzed. +// The newly added column/index might not have its stats. +func (m *ColAndIdxExistenceMap) SomeAnalyzed() bool { + if m == nil { + return false + } + for _, v := range m.colAnalyzed { + if v { + return true + } + } + for _, v := range m.idxAnalyzed { + if v { + return true + } + } + return false +} + +// Has checks whether a column/index stats exists. +// This method only checks whether the given item exists or not. +// Don't check whether it has statistics or not. +func (m *ColAndIdxExistenceMap) Has(id int64, isIndex bool) bool { + if isIndex { + _, ok := m.idxInfoMap[id] + return ok + } + _, ok := m.colInfoMap[id] + return ok +} + +// HasAnalyzed checks whether a column/index stats exists and it has stats. +// TODO: the map should only keep the analyzed cols. +// There's three possible status of column/index's statistics: +// 1. We don't have this column/index. +// 2. We have it, but it hasn't been analyzed yet. +// 3. We have it and its statistics. +// +// To figure out three status, we use HasAnalyzed's TRUE value to represents the status 3. The Has's FALSE to represents the status 1. +func (m *ColAndIdxExistenceMap) HasAnalyzed(id int64, isIndex bool) bool { + if isIndex { + analyzed, ok := m.idxAnalyzed[id] + return ok && analyzed + } + analyzed, ok := m.colAnalyzed[id] + return ok && analyzed +} + +// InsertCol inserts a column with its meta into the map. +func (m *ColAndIdxExistenceMap) InsertCol(id int64, info *model.ColumnInfo, analyzed bool) { + m.colInfoMap[id] = info + m.colAnalyzed[id] = analyzed +} + +// GetCol gets the meta data of the given column. +func (m *ColAndIdxExistenceMap) GetCol(id int64) *model.ColumnInfo { + return m.colInfoMap[id] +} + +// InsertIndex inserts an index with its meta into the map. +func (m *ColAndIdxExistenceMap) InsertIndex(id int64, info *model.IndexInfo, analyzed bool) { + m.idxInfoMap[id] = info + m.idxAnalyzed[id] = analyzed +} + +// GetIndex gets the meta data of the given index. +func (m *ColAndIdxExistenceMap) GetIndex(id int64) *model.IndexInfo { + return m.idxInfoMap[id] +} + +// IsEmpty checks whether the map is empty. +func (m *ColAndIdxExistenceMap) IsEmpty() bool { + return len(m.colInfoMap)+len(m.idxInfoMap) == 0 +} + +// Clone deeply copies the map. +func (m *ColAndIdxExistenceMap) Clone() *ColAndIdxExistenceMap { + mm := NewColAndIndexExistenceMap(len(m.colInfoMap), len(m.idxInfoMap)) + mm.colInfoMap = maps.Clone(m.colInfoMap) + mm.colAnalyzed = maps.Clone(m.colAnalyzed) + mm.idxAnalyzed = maps.Clone(m.idxAnalyzed) + mm.idxInfoMap = maps.Clone(m.idxInfoMap) + return mm +} + +// NewColAndIndexExistenceMap return a new object with the given capcity. +func NewColAndIndexExistenceMap(colCap, idxCap int) *ColAndIdxExistenceMap { + return &ColAndIdxExistenceMap{ + colInfoMap: make(map[int64]*model.ColumnInfo, colCap), + colAnalyzed: make(map[int64]bool, colCap), + idxInfoMap: make(map[int64]*model.IndexInfo, idxCap), + idxAnalyzed: make(map[int64]bool, idxCap), + } +} + +// ColAndIdxExistenceMapIsEqual is used in testing, checking whether the two are equal. +func ColAndIdxExistenceMapIsEqual(m1, m2 *ColAndIdxExistenceMap) bool { + return maps.Equal(m1.colAnalyzed, m2.colAnalyzed) && maps.Equal(m1.idxAnalyzed, m2.idxAnalyzed) } // ExtendedStatsItem is the cached item of a mysql.stats_extended record. @@ -121,8 +234,9 @@ type HistColl struct { StatsVer int // HavePhysicalID is true means this HistColl is from single table and have its ID's information. // The physical id is used when try to load column stats from storage. - HavePhysicalID bool - Pseudo bool + HavePhysicalID bool + Pseudo bool + CanNotTriggerLoad bool } // TableMemoryUsage records tbl memory usage @@ -304,6 +418,7 @@ func (t *Table) Copy() *Table { Version: t.Version, Name: t.Name, TblInfoUpdateTS: t.TblInfoUpdateTS, + IsPkIsHandle: t.IsPkIsHandle, LastAnalyzeVersion: t.LastAnalyzeVersion, } if t.ExtendedStats != nil { @@ -316,6 +431,9 @@ func (t *Table) Copy() *Table { } nt.ExtendedStats = newExtStatsColl } + if t.ColAndIdxExistenceMap != nil { + nt.ColAndIdxExistenceMap = t.ColAndIdxExistenceMap.Clone() + } return nt } @@ -334,12 +452,13 @@ func (t *Table) ShallowCopy() *Table { StatsVer: t.StatsVer, } nt := &Table{ - HistColl: newHistColl, - Version: t.Version, - Name: t.Name, - TblInfoUpdateTS: t.TblInfoUpdateTS, - ExtendedStats: t.ExtendedStats, - LastAnalyzeVersion: t.LastAnalyzeVersion, + HistColl: newHistColl, + Version: t.Version, + Name: t.Name, + TblInfoUpdateTS: t.TblInfoUpdateTS, + ExtendedStats: t.ExtendedStats, + ColAndIdxExistenceMap: t.ColAndIdxExistenceMap, + LastAnalyzeVersion: t.LastAnalyzeVersion, } return nt } @@ -492,6 +611,40 @@ func (t *Table) GetStatsHealthy() (int64, bool) { return healthy, true } +// ColumnIsLoadNeeded checks whether the column needs trigger the async/sync load. +// The Column should be visible in the table and really has analyzed statistics in the stroage. +// Also, if the stats has been loaded into the memory, we also don't need to load it. +// We return the Column together with the checking result, to avoid accessing the map multiple times. +func (t *Table) ColumnIsLoadNeeded(id int64) (*Column, bool) { + col, ok := t.Columns[id] + // If the column is not in the memory, and we have its stats in the storage. We need to trigger the load. + if !ok && t.ColAndIdxExistenceMap.HasAnalyzed(id, false) { + return nil, true + } + // If the column is in the memory, we check its embedded func. + if ok && col.StatsAvailable() && !col.IsFullLoad() { + return col, true + } + return col, false +} + +// IndexIsLoadNeeded checks whether the index needs trigger the async/sync load. +// The Index should be visible in the table and really has analyzed statistics in the stroage. +// Also, if the stats has been loaded into the memory, we also don't need to load it. +// We return the Index together with the checking result, to avoid accessing the map multiple times. +func (t *Table) IndexIsLoadNeeded(id int64) (*Index, bool) { + idx, ok := t.Indices[id] + // If the column is not in the memory, and we have its stats in the storage. We need to trigger the load. + if !ok && t.ColAndIdxExistenceMap.HasAnalyzed(id, true) { + return nil, true + } + // If the column is in the memory, we check its embedded func. + if ok && idx.IsAnalyzed() && !idx.IsFullLoad() { + return idx, true + } + return idx, false +} + type neededStatsMap struct { items map[model.TableItemID]struct{} m sync.RWMutex @@ -631,7 +784,7 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(tblInfo *model.TableInfo, c newIdxHistMap[idxHist.ID] = idxHist idx2Columns[idxHist.ID] = ids if idxInfo.MVIndex { - cols, ok := PrepareCols4MVIndex(tblInfo, idxInfo, columns) + cols, ok := PrepareCols4MVIndex(tblInfo, idxInfo, columns, true) if ok { mvIdx2Columns[id] = cols } @@ -659,44 +812,45 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(tblInfo *model.TableInfo, c // Usually, we don't want to trigger stats loading for pseudo table. // But there are exceptional cases. In such cases, we should pass allowTriggerLoading as true. // Such case could possibly happen in getStatsTable(). -func PseudoTable(tblInfo *model.TableInfo, allowTriggerLoading bool) *Table { - const fakePhysicalID int64 = -1 +func PseudoTable(tblInfo *model.TableInfo, allowTriggerLoading bool, allowFillHistMeta bool) *Table { pseudoHistColl := HistColl{ - RealtimeCount: PseudoRowCount, - PhysicalID: tblInfo.ID, - HavePhysicalID: true, - Columns: make(map[int64]*Column, len(tblInfo.Columns)), - Indices: make(map[int64]*Index, len(tblInfo.Indices)), - Pseudo: true, + RealtimeCount: PseudoRowCount, + PhysicalID: tblInfo.ID, + HavePhysicalID: true, + Columns: make(map[int64]*Column, 2), + Indices: make(map[int64]*Index, 2), + Pseudo: true, + CanNotTriggerLoad: !allowTriggerLoading, } t := &Table{ - HistColl: pseudoHistColl, + HistColl: pseudoHistColl, + ColAndIdxExistenceMap: NewColAndIndexExistenceMap(len(tblInfo.Columns), len(tblInfo.Indices)), } for _, col := range tblInfo.Columns { // The column is public to use. Also we should check the column is not hidden since hidden means that it's used by expression index. // We would not collect stats for the hidden column and we won't use the hidden column to estimate. // Thus we don't create pseudo stats for it. if col.State == model.StatePublic && !col.Hidden { - t.Columns[col.ID] = &Column{ - PhysicalID: fakePhysicalID, - Info: col, - IsHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.GetFlag()), - Histogram: *NewHistogram(col.ID, 0, 0, 0, &col.FieldType, 0, 0), - } - if allowTriggerLoading { - t.Columns[col.ID].PhysicalID = tblInfo.ID + t.ColAndIdxExistenceMap.InsertCol(col.ID, col, false) + if allowFillHistMeta { + t.Columns[col.ID] = &Column{ + PhysicalID: tblInfo.ID, + Info: col, + IsHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.GetFlag()), + Histogram: *NewHistogram(col.ID, 0, 0, 0, &col.FieldType, 0, 0), + } } } } for _, idx := range tblInfo.Indices { if idx.State == model.StatePublic { - t.Indices[idx.ID] = &Index{ - PhysicalID: fakePhysicalID, - Info: idx, - Histogram: *NewHistogram(idx.ID, 0, 0, 0, types.NewFieldType(mysql.TypeBlob), 0, 0), - } - if allowTriggerLoading { - t.Indices[idx.ID].PhysicalID = tblInfo.ID + t.ColAndIdxExistenceMap.InsertIndex(idx.ID, idx, false) + if allowFillHistMeta { + t.Indices[idx.ID] = &Index{ + PhysicalID: tblInfo.ID, + Info: idx, + Histogram: *NewHistogram(idx.ID, 0, 0, 0, types.NewFieldType(mysql.TypeBlob), 0, 0), + } } } } @@ -721,4 +875,5 @@ var PrepareCols4MVIndex func( tableInfo *model.TableInfo, mvIndex *model.IndexInfo, tblCols []*expression.Column, + checkOnly1ArrayTypeCol bool, ) (idxCols []*expression.Column, ok bool) diff --git a/pkg/store/copr/coprocessor.go b/pkg/store/copr/coprocessor.go index 3abfba960f186..d4e6bb045d974 100644 --- a/pkg/store/copr/coprocessor.go +++ b/pkg/store/copr/coprocessor.go @@ -700,7 +700,8 @@ type copIterator struct { storeBatchedNum atomic.Uint64 storeBatchedFallbackNum atomic.Uint64 - runawayChecker *resourcegroup.RunawayChecker + runawayChecker *resourcegroup.RunawayChecker + unconsumedStats *unconsumedCopRuntimeStats } // copIteratorWorker receives tasks from copIteratorTaskSender, handles tasks and sends the copResponse to respChan. @@ -723,6 +724,7 @@ type copIteratorWorker struct { storeBatchedNum *atomic.Uint64 storeBatchedFallbackNum *atomic.Uint64 + unconsumedStats *unconsumedCopRuntimeStats } // copIteratorTaskSender sends tasks to taskCh then wait for the workers to exit. @@ -833,6 +835,7 @@ func (worker *copIteratorWorker) run(ctx context.Context) { func (it *copIterator) open(ctx context.Context, enabledRateLimitAction, enableCollectExecutionInfo bool) { taskCh := make(chan *copTask, 1) smallTaskCh := make(chan *copTask, 1) + it.unconsumedStats = &unconsumedCopRuntimeStats{} it.wg.Add(it.concurrency + it.smallTaskConcurrency) // Start it.concurrency number of workers to handle cop requests. for i := 0; i < it.concurrency+it.smallTaskConcurrency; i++ { @@ -857,6 +860,7 @@ func (it *copIterator) open(ctx context.Context, enabledRateLimitAction, enableC pagingTaskIdx: &it.pagingTaskIdx, storeBatchedNum: &it.storeBatchedNum, storeBatchedFallbackNum: &it.storeBatchedFallbackNum, + unconsumedStats: it.unconsumedStats, } go worker.run(ctx) } @@ -1096,6 +1100,23 @@ func (it *copIterator) Next(ctx context.Context) (kv.ResultSubset, error) { return resp, nil } +// HasUnconsumedCopRuntimeStats indicate whether has unconsumed CopRuntimeStats. +type HasUnconsumedCopRuntimeStats interface { + // CollectUnconsumedCopRuntimeStats returns unconsumed CopRuntimeStats. + CollectUnconsumedCopRuntimeStats() []*CopRuntimeStats +} + +func (it *copIterator) CollectUnconsumedCopRuntimeStats() []*CopRuntimeStats { + if it == nil || it.unconsumedStats == nil { + return nil + } + it.unconsumedStats.Lock() + stats := make([]*CopRuntimeStats, 0, len(it.unconsumedStats.stats)) + stats = append(stats, it.unconsumedStats.stats...) + it.unconsumedStats.Unlock() + return stats +} + // Associate each region with an independent backoffer. In this way, when multiple regions are // unavailable, TiDB can execute very quickly without blocking func chooseBackoffer(ctx context.Context, backoffermap map[uint64]*Backoffer, task *copTask, worker *copIteratorWorker) *Backoffer { @@ -1261,6 +1282,7 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch err = worker.handleTiDBSendReqErr(err, task, ch) return nil, err } + worker.collectUnconsumedCopRuntimeStats(bo, rpcCtx) return nil, errors.Trace(err) } @@ -1758,17 +1780,24 @@ func (worker *copIteratorWorker) handleCollectExecutionInfo(bo *Backoffer, rpcCt if resp.detail == nil { resp.detail = new(CopRuntimeStats) } - resp.detail.Stats = worker.kvclient.Stats + worker.collectCopRuntimeStats(resp.detail, bo, rpcCtx, resp) +} + +func (worker *copIteratorWorker) collectCopRuntimeStats(copStats *CopRuntimeStats, bo *Backoffer, rpcCtx *tikv.RPCContext, resp *copResponse) { + copStats.Stats = worker.kvclient.Stats backoffTimes := bo.GetBackoffTimes() - resp.detail.BackoffTime = time.Duration(bo.GetTotalSleep()) * time.Millisecond - resp.detail.BackoffSleep = make(map[string]time.Duration, len(backoffTimes)) - resp.detail.BackoffTimes = make(map[string]int, len(backoffTimes)) + copStats.BackoffTime = time.Duration(bo.GetTotalSleep()) * time.Millisecond + copStats.BackoffSleep = make(map[string]time.Duration, len(backoffTimes)) + copStats.BackoffTimes = make(map[string]int, len(backoffTimes)) for backoff := range backoffTimes { - resp.detail.BackoffTimes[backoff] = backoffTimes[backoff] - resp.detail.BackoffSleep[backoff] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond + copStats.BackoffTimes[backoff] = backoffTimes[backoff] + copStats.BackoffSleep[backoff] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond } if rpcCtx != nil { - resp.detail.CalleeAddress = rpcCtx.Addr + copStats.CalleeAddress = rpcCtx.Addr + } + if resp == nil { + return } sd := &util.ScanDetail{} td := util.TimeDetail{} @@ -1791,8 +1820,20 @@ func (worker *copIteratorWorker) handleCollectExecutionInfo(bo *Backoffer, rpcCt } } } - resp.detail.ScanDetail = sd - resp.detail.TimeDetail = td + copStats.ScanDetail = sd + copStats.TimeDetail = td +} + +func (worker *copIteratorWorker) collectUnconsumedCopRuntimeStats(bo *Backoffer, rpcCtx *tikv.RPCContext) { + if worker.kvclient.Stats == nil { + return + } + copStats := &CopRuntimeStats{} + worker.collectCopRuntimeStats(copStats, bo, rpcCtx, nil) + worker.unconsumedStats.Lock() + worker.unconsumedStats.stats = append(worker.unconsumedStats.stats, copStats) + worker.unconsumedStats.Unlock() + worker.kvclient.Stats = nil } // CopRuntimeStats contains execution detail information. @@ -1803,6 +1844,11 @@ type CopRuntimeStats struct { CoprCacheHit bool } +type unconsumedCopRuntimeStats struct { + sync.Mutex + stats []*CopRuntimeStats +} + func (worker *copIteratorWorker) handleTiDBSendReqErr(err error, task *copTask, ch chan<- *copResponse) error { errCode := errno.ErrUnknown errMsg := err.Error() diff --git a/pkg/store/driver/BUILD.bazel b/pkg/store/driver/BUILD.bazel index 52d540fd3274b..8ea468e176415 100644 --- a/pkg/store/driver/BUILD.bazel +++ b/pkg/store/driver/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/store/driver", visibility = ["//visibility:public"], deps = [ - "//pkg/executor/importer", "//pkg/kv", "//pkg/metrics", "//pkg/sessionctx/variable", diff --git a/pkg/store/driver/tikv_driver.go b/pkg/store/driver/tikv_driver.go index 366a25e7ebe5f..fb43748d402d3 100644 --- a/pkg/store/driver/tikv_driver.go +++ b/pkg/store/driver/tikv_driver.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/errors" deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/tidb/pkg/executor/importer" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/sessionctx/variable" @@ -60,8 +59,6 @@ func init() { // Setup the Hooks to dynamic control global resource controller. variable.EnableGlobalResourceControlFunc = tikv.EnableResourceControl variable.DisableGlobalResourceControlFunc = tikv.DisableResourceControl - // cannot use this package directly, it causes import cycle - importer.GetKVStore = getKVStore } // Option is a function that changes some config of Driver diff --git a/pkg/store/driver/txn/batch_getter.go b/pkg/store/driver/txn/batch_getter.go index bd988c64b6563..465ee450db866 100644 --- a/pkg/store/driver/txn/batch_getter.go +++ b/pkg/store/driver/txn/batch_getter.go @@ -31,7 +31,6 @@ type tikvBatchGetter struct { } func (b tikvBatchGetter) BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) { - // toTiDBKeys kvKeys := *(*[]kv.Key)(unsafe.Pointer(&keys)) vals, err := b.tidbBatchGetter.BatchGet(ctx, kvKeys) return vals, err @@ -66,6 +65,29 @@ func (b tikvBatchBufferGetter) Get(ctx context.Context, k []byte) ([]byte, error return val, err } +func (b tikvBatchBufferGetter) BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) { + bufferValues, err := b.tidbBuffer.BatchGet(ctx, keys) + if err != nil { + return nil, err + } + if b.tidbMiddleCache == nil { + return bufferValues, nil + } + for _, key := range keys { + if _, ok := bufferValues[string(key)]; !ok { + val, err := b.tidbMiddleCache.Get(ctx, key) + if err != nil { + if kv.IsErrNotFound(err) { + continue + } + return nil, err + } + bufferValues[string(key)] = val + } + } + return bufferValues, nil +} + func (b tikvBatchBufferGetter) Len() int { return b.tidbBuffer.Len() } @@ -74,6 +96,8 @@ func (b tikvBatchBufferGetter) Len() int { type BatchBufferGetter interface { Len() int Getter + // BatchGet gets a batch of values, keys are in bytes slice format. + BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) } // BatchGetter is the interface for BatchGet. diff --git a/pkg/store/driver/txn/batch_getter_test.go b/pkg/store/driver/txn/batch_getter_test.go index 1b07532be3062..a0341260040b7 100644 --- a/pkg/store/driver/txn/batch_getter_test.go +++ b/pkg/store/driver/txn/batch_getter_test.go @@ -16,6 +16,7 @@ package txn import ( "context" "testing" + "unsafe" "github.com/pingcap/tidb/pkg/kv" "github.com/stretchr/testify/require" @@ -41,7 +42,7 @@ func TestBufferBatchGetter(t *testing.T) { require.NoError(t, buffer.Set(ka, []byte("a2"))) require.NoError(t, buffer.Delete(kb)) - batchGetter := NewBufferBatchGetter(buffer, middle, snap) + batchGetter := NewBufferBatchGetter(&mockBufferBatchGetterStore{buffer}, middle, snap) result, err := batchGetter.BatchGet(context.Background(), []kv.Key{ka, kb, kc, kd}) require.NoError(t, err) require.Len(t, result, 3) @@ -105,3 +106,12 @@ func (s *mockBatchGetterStore) Set(k kv.Key, v []byte) error { func (s *mockBatchGetterStore) Delete(k kv.Key) error { return s.Set(k, []byte{}) } + +type mockBufferBatchGetterStore struct { + *mockBatchGetterStore +} + +func (s *mockBufferBatchGetterStore) BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) { + kvKeys := *(*[]kv.Key)(unsafe.Pointer(&keys)) + return s.mockBatchGetterStore.BatchGet(ctx, kvKeys) +} diff --git a/pkg/store/driver/txn/txn_driver.go b/pkg/store/driver/txn/txn_driver.go index bfe2d9bafc793..9214225cf02fc 100644 --- a/pkg/store/driver/txn/txn_driver.go +++ b/pkg/store/driver/txn/txn_driver.go @@ -316,12 +316,13 @@ func (txn *tikvTxn) GetVars() any { func (txn *tikvTxn) extractKeyErr(err error) error { if e, ok := errors.Cause(err).(*tikverr.ErrKeyExist); ok { - return txn.extractKeyExistsErr(e.GetKey()) + return txn.extractKeyExistsErr(e) } return extractKeyErr(err) } -func (txn *tikvTxn) extractKeyExistsErr(key kv.Key) error { +func (txn *tikvTxn) extractKeyExistsErr(errExist *tikverr.ErrKeyExist) error { + var key kv.Key = errExist.GetKey() tableID, indexID, isRecord, err := tablecodec.DecodeKeyHead(key) if err != nil { return genKeyExistsError("UNKNOWN", key.String(), err) @@ -332,10 +333,19 @@ func (txn *tikvTxn) extractKeyExistsErr(key kv.Key) error { if tblInfo == nil { return genKeyExistsError("UNKNOWN", key.String(), errors.New("cannot find table info")) } + var value []byte if txn.IsPipelined() { - return genKeyExistsError("UNKNOWN", key.String(), errors.New("currently pipelined dml doesn't extract value from key exists error")) + value = errExist.Value + if len(value) == 0 { + return genKeyExistsError( + "UNKNOWN", + key.String(), + errors.New("The value is empty (a delete)"), + ) + } + } else { + value, err = txn.KVTxn.GetUnionStore().GetMemBuffer().GetMemDB().SelectValueHistory(key, func(value []byte) bool { return len(value) != 0 }) } - value, err := txn.KVTxn.GetUnionStore().GetMemBuffer().GetMemDB().SelectValueHistory(key, func(value []byte) bool { return len(value) != 0 }) if err != nil { return genKeyExistsError("UNKNOWN", key.String(), err) } @@ -417,6 +427,15 @@ func (txn *tikvTxn) IsInFairLockingMode() bool { return txn.KVTxn.IsInAggressiveLockingMode() } +// MayFlush wraps the flush function and extract the error. +func (txn *tikvTxn) MayFlush() error { + if !txn.IsPipelined() { + return nil + } + _, err := txn.KVTxn.GetMemBuffer().Flush(false) + return txn.extractKeyErr(err) +} + // TiDBKVFilter is the filter specific to TiDB to filter out KV pairs that needn't be committed. type TiDBKVFilter struct{} diff --git a/pkg/store/driver/txn/unionstore_driver.go b/pkg/store/driver/txn/unionstore_driver.go index e12e673ba8555..c6043be69a60a 100644 --- a/pkg/store/driver/txn/unionstore_driver.go +++ b/pkg/store/driver/txn/unionstore_driver.go @@ -68,24 +68,14 @@ func (m *memBuffer) GetFlags(key kv.Key) (kv.KeyFlags, error) { } func (m *memBuffer) Staging() kv.StagingHandle { - if m.isPipelinedDML { - // 0 stands for staging not supported. - return 0 - } return kv.StagingHandle(m.MemBuffer.Staging()) } func (m *memBuffer) Cleanup(h kv.StagingHandle) { - if m.isPipelinedDML { - return - } m.MemBuffer.Cleanup(int(h)) } func (m *memBuffer) Release(h kv.StagingHandle) { - if m.isPipelinedDML { - return - } m.MemBuffer.Release(int(h)) } @@ -148,13 +138,15 @@ func (m *memBuffer) SnapshotGetter() kv.Getter { return newKVGetter(m.MemBuffer.SnapshotGetter()) } -// MayFlush implements kv.MemBuffer.MayFlush interface. -func (m *memBuffer) MayFlush() error { - if !m.isPipelinedDML { - return nil - } - _, err := m.MemBuffer.Flush(false) - return err +// GetLocal implements kv.MemBuffer interface +func (m *memBuffer) GetLocal(ctx context.Context, key []byte) ([]byte, error) { + data, err := m.MemBuffer.GetLocal(ctx, key) + return data, derr.ToTiDBErr(err) +} + +func (m *memBuffer) BatchGet(ctx context.Context, keys [][]byte) (map[string][]byte, error) { + data, err := m.MemBuffer.BatchGet(ctx, keys) + return data, derr.ToTiDBErr(err) } type tikvGetter struct { diff --git a/pkg/store/mockstore/mockcopr/BUILD.bazel b/pkg/store/mockstore/mockcopr/BUILD.bazel index 04f01dbe0cec1..8650be3fed213 100644 --- a/pkg/store/mockstore/mockcopr/BUILD.bazel +++ b/pkg/store/mockstore/mockcopr/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//pkg/parser/terror", "//pkg/sessionctx", "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", "//pkg/statistics", "//pkg/tablecodec", "//pkg/types", diff --git a/pkg/store/mockstore/mockcopr/cop_handler_dag.go b/pkg/store/mockstore/mockcopr/cop_handler_dag.go index a5a53931b3195..0d19a8a7cb60f 100644 --- a/pkg/store/mockstore/mockcopr/cop_handler_dag.go +++ b/pkg/store/mockstore/mockcopr/cop_handler_dag.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" @@ -111,6 +112,11 @@ func (h coprHandler) buildDAGExecutor(req *coprocessor.Request) (*dagContext, ex return nil, nil, nil, errors.Trace(err) } sctx := flagsAndTzToSessionContext(dagReq.Flags, tz) + if dagReq.DivPrecisionIncrement != nil { + sctx.GetSessionVars().DivPrecisionIncrement = int(*dagReq.DivPrecisionIncrement) + } else { + sctx.GetSessionVars().DivPrecisionIncrement = variable.DefDivPrecisionIncrement + } ctx := &dagContext{ dagReq: dagReq, diff --git a/pkg/store/mockstore/unistore/cophandler/BUILD.bazel b/pkg/store/mockstore/unistore/cophandler/BUILD.bazel index 2a6e691a3cbaa..3916d0ac31deb 100644 --- a/pkg/store/mockstore/unistore/cophandler/BUILD.bazel +++ b/pkg/store/mockstore/unistore/cophandler/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//pkg/parser/terror", "//pkg/sessionctx", "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", "//pkg/statistics", "//pkg/store/mockstore/unistore/client", "//pkg/store/mockstore/unistore/lockstore", diff --git a/pkg/store/mockstore/unistore/cophandler/cop_handler.go b/pkg/store/mockstore/unistore/cophandler/cop_handler.go index 9cf1aca52b86a..607ac6666ae16 100644 --- a/pkg/store/mockstore/unistore/cophandler/cop_handler.go +++ b/pkg/store/mockstore/unistore/cophandler/cop_handler.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/store/mockstore/unistore/client" "github.com/pingcap/tidb/pkg/store/mockstore/unistore/lockstore" "github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv/dbreader" @@ -313,6 +314,11 @@ func buildDAG(reader *dbreader.DBReader, lockStore *lockstore.MemStore, req *cop } } sctx := flagsAndTzToSessionContext(dagReq.Flags, tz) + if dagReq.DivPrecisionIncrement != nil { + sctx.GetSessionVars().DivPrecisionIncrement = int(*dagReq.DivPrecisionIncrement) + } else { + sctx.GetSessionVars().DivPrecisionIncrement = variable.DefDivPrecisionIncrement + } ctx := &dagContext{ evalContext: &evalContext{sctx: sctx}, dbReader: reader, diff --git a/pkg/store/mockstore/unistore/rpc.go b/pkg/store/mockstore/unistore/rpc.go index c2f7ac78a00e7..b255138dff63b 100644 --- a/pkg/store/mockstore/unistore/rpc.go +++ b/pkg/store/mockstore/unistore/rpc.go @@ -93,6 +93,10 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R failpoint.Return(tikvrpc.GenRegionErrorResp(req, &errorpb.Error{Message: "Deadline is exceeded"})) } }) + failpoint.Inject("unistoreRPCSlowByInjestSleep", func(val failpoint.Value) { + time.Sleep(time.Duration(val.(int) * int(time.Millisecond))) + failpoint.Return(tikvrpc.GenRegionErrorResp(req, &errorpb.Error{Message: "Deadline is exceeded"})) + }) select { case <-ctx.Done(): diff --git a/pkg/store/mockstore/unistore/tikv/mvcc.go b/pkg/store/mockstore/unistore/tikv/mvcc.go index a7690bb557274..f4bab77bb1215 100644 --- a/pkg/store/mockstore/unistore/tikv/mvcc.go +++ b/pkg/store/mockstore/unistore/tikv/mvcc.go @@ -1493,12 +1493,27 @@ func (store *MVCCStore) ReadBufferFromLock(startTS uint64, keys ...[]byte) []*kv if lock.StartTS != startTS { continue } - val := getValueFromLock(&lock) - if len(val) > 0 { - pairs = append(pairs, &kvrpcpb.KvPair{ - Key: key, - Value: safeCopy(val), - }) + switch lock.Op { + case uint8(kvrpcpb.Op_Put): + val := getValueFromLock(&lock) + if val == nil { + panic("Op_Put has a nil value") + } + pairs = append( + pairs, &kvrpcpb.KvPair{ + Key: key, + Value: safeCopy(val), + }, + ) + case uint8(kvrpcpb.Op_Del): + pairs = append( + pairs, &kvrpcpb.KvPair{ + Key: key, + Value: []byte{}, + }, + ) + default: + panic("unexpected op. Optimistic txn should only contain put and delete locks") } } return pairs diff --git a/pkg/table/tables/BUILD.bazel b/pkg/table/tables/BUILD.bazel index 3bc808a3f12ba..fff0c106c3ff1 100644 --- a/pkg/table/tables/BUILD.bazel +++ b/pkg/table/tables/BUILD.bazel @@ -74,8 +74,10 @@ go_test( ], embed = [":tables"], flaky = True, - shard_count = 30, + shard_count = 31, deps = [ + "//br/pkg/lightning/backend/encode", + "//br/pkg/lightning/backend/kv", "//pkg/ddl", "//pkg/ddl/util/callback", "//pkg/domain", diff --git a/pkg/table/tables/index.go b/pkg/table/tables/index.go index c90eb900b21af..bde5c0e9624d4 100644 --- a/pkg/table/tables/index.go +++ b/pkg/table/tables/index.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" @@ -27,6 +28,7 @@ import ( "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/rowcodec" "github.com/pingcap/tidb/pkg/util/tracing" @@ -194,6 +196,12 @@ func (c *index) Create(sctx table.MutateContext, txn kv.Transaction, indexedValu } } + if txn.IsPipelined() { + // For pipelined DML, disable the untouched optimization to avoid extra RPCs for MemBuffer.Get(). + // TODO: optimize this. + opt.Untouched = false + } + if opt.Untouched { txn, err1 := sctx.Txn(true) if err1 != nil { @@ -276,10 +284,10 @@ func (c *index) Create(sctx table.MutateContext, txn kv.Transaction, indexedValu if c.tblInfo.TempTableType != model.TempTableNone { // Always check key for temporary table because it does not write to TiKV value, err = txn.Get(ctx, key) - } else if sctx.GetSessionVars().LazyCheckKeyNotExists() && !keyIsTempIdxKey { + } else if (txn.IsPipelined() || sctx.GetSessionVars().LazyCheckKeyNotExists()) && !keyIsTempIdxKey { // For temp index keys, we can't get the temp value from memory buffer, even if the lazy check is enabled. // Otherwise, it may cause the temp index value to be overwritten, leading to data inconsistency. - value, err = txn.GetMemBuffer().Get(ctx, key) + value, err = txn.GetMemBuffer().GetLocal(ctx, key) } else { value, err = txn.Get(ctx, key) } @@ -296,7 +304,7 @@ func (c *index) Create(sctx table.MutateContext, txn kv.Transaction, indexedValu // The index key value is not found or deleted. if err != nil || len(value) == 0 || (!tempIdxVal.IsEmpty() && tempIdxVal.Current().Delete) { val := idxVal - lazyCheck := sctx.GetSessionVars().LazyCheckKeyNotExists() && err != nil + lazyCheck := (txn.IsPipelined() || sctx.GetSessionVars().LazyCheckKeyNotExists()) && err != nil if keyIsTempIdxKey { tempVal := tablecodec.TempIndexValueElem{Value: idxVal, KeyVer: keyVer, Distinct: true} val = tempVal.Encode(value) @@ -708,3 +716,30 @@ func TryAppendCommonHandleRowcodecColInfos(colInfo []rowcodec.ColInfo, tblInfo * } return colInfo } + +// GenIndexValueFromIndex generate index value from index. +func GenIndexValueFromIndex(key []byte, value []byte, tblInfo *model.TableInfo, idxInfo *model.IndexInfo) ([]string, error) { + idxColLen := len(idxInfo.Columns) + colInfos := BuildRowcodecColInfoForIndexColumns(idxInfo, tblInfo) + values, err := tablecodec.DecodeIndexKV(key, value, idxColLen, tablecodec.HandleNotNeeded, colInfos) + if err != nil { + return nil, errors.Trace(err) + } + valueStr := make([]string, 0, idxColLen) + for i, val := range values[:idxColLen] { + d, err := tablecodec.DecodeColumnValue(val, colInfos[i].Ft, time.Local) + if err != nil { + return nil, errors.Trace(err) + } + str, err := d.ToString() + if err != nil { + str = string(val) + } + if types.IsBinaryStr(colInfos[i].Ft) || types.IsTypeBit(colInfos[i].Ft) { + str = util.FmtNonASCIIPrintableCharToHex(str) + } + valueStr = append(valueStr, str) + } + + return valueStr, nil +} diff --git a/pkg/table/tables/index_test.go b/pkg/table/tables/index_test.go index 889a341a87cc4..d9a212f13e86c 100644 --- a/pkg/table/tables/index_test.go +++ b/pkg/table/tables/index_test.go @@ -18,11 +18,14 @@ import ( "context" "testing" + "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" + lkv "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" @@ -168,3 +171,44 @@ func buildTableInfo(t *testing.T, sql string) *model.TableInfo { require.NoError(t, err) return tblInfo } + +func TestGenIndexValueFromIndex(t *testing.T) { + tblInfo := buildTableInfo(t, "create table a (a int primary key, b int not null, c text, unique key key_b(b));") + tblInfo.State = model.StatePublic + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + require.NoError(t, err) + + sessionOpts := encode.SessionOptions{ + SQLMode: mysql.ModeStrictAllTables, + Timestamp: 1234567890, + } + + encoder, err := lkv.NewBaseKVEncoder(&encode.EncodingConfig{ + Table: tbl, + SessionOptions: sessionOpts, + }) + require.NoError(t, err) + encoder.SessionCtx.GetSessionVars().RowEncoder.Enable = true + + data1 := []types.Datum{ + types.NewIntDatum(1), + types.NewIntDatum(23), + types.NewStringDatum("4.csv"), + } + tctx := encoder.SessionCtx.GetTableCtx() + _, err = encoder.Table.AddRecord(tctx, data1) + require.NoError(t, err) + kvPairs := encoder.SessionCtx.TakeKvPairs() + + indexKey := kvPairs.Pairs[1].Key + indexValue := kvPairs.Pairs[1].Val + + _, idxID, _, err := tablecodec.DecodeIndexKey(indexKey) + require.NoError(t, err) + + idxInfo := model.FindIndexInfoByID(tbl.Meta().Indices, idxID) + + valueStr, err := tables.GenIndexValueFromIndex(indexKey, indexValue, tbl.Meta(), idxInfo) + require.NoError(t, err) + require.Equal(t, []string{"23"}, valueStr) +} diff --git a/pkg/table/tables/mutation_checker.go b/pkg/table/tables/mutation_checker.go index 3ef2c42730b1e..ad47a1dfc565d 100644 --- a/pkg/table/tables/mutation_checker.go +++ b/pkg/table/tables/mutation_checker.go @@ -83,6 +83,9 @@ func CheckDataConsistency( if t.Meta().GetPartitionInfo() != nil { return nil } + if txn.IsPipelined() { + return nil + } if sh == 0 { // some implementations of MemBuffer doesn't support staging, e.g. that in br/pkg/lightning/backend/kv return nil diff --git a/pkg/table/tables/tables.go b/pkg/table/tables/tables.go index 7670f2547fdde..c388641c0b3e0 100644 --- a/pkg/table/tables/tables.go +++ b/pkg/table/tables/tables.go @@ -629,7 +629,7 @@ func (t *TableCommon) UpdateRecord(ctx context.Context, sctx table.MutateContext colSize[col.ID] = int64(newLen - oldLen) } sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 0, 1, colSize) - return memBuffer.MayFlush() + return nil } func (t *TableCommon) rebuildIndices(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, touched []bool, oldData []types.Datum, newData []types.Datum, opts ...table.CreateIdxOptFunc) error { @@ -1016,9 +1016,9 @@ func (t *TableCommon) AddRecord(sctx table.MutateContext, r []types.Datum, opts if t.meta.TempTableType != model.TempTableNone { // Always check key for temporary table because it does not write to TiKV _, err = txn.Get(ctx, key) - } else if sctx.GetSessionVars().LazyCheckKeyNotExists() { + } else if sctx.GetSessionVars().LazyCheckKeyNotExists() || txn.IsPipelined() { var v []byte - v, err = txn.GetMemBuffer().Get(ctx, key) + v, err = txn.GetMemBuffer().GetLocal(ctx, key) if err != nil { setPresume = true } @@ -1123,9 +1123,6 @@ func (t *TableCommon) AddRecord(sctx table.MutateContext, r []types.Datum, opts colSize[col.ID] = int64(size) - 1 } sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 1, 1, colSize) - if err = memBuffer.MayFlush(); err != nil { - return nil, err - } return recordID, nil } @@ -1403,10 +1400,7 @@ func (t *TableCommon) RemoveRecord(ctx table.MutateContext, h kv.Handle, r []typ colSize[col.ID] = -int64(size - 1) } ctx.GetSessionVars().TxnCtx.UpdateDeltaForTable(t.physicalTableID, -1, 1, colSize) - if err != nil { - return err - } - return memBuffer.MayFlush() + return err } func (t *TableCommon) addInsertBinlog(ctx table.MutateContext, h kv.Handle, row []types.Datum, colIDs []int64) error { @@ -2352,7 +2346,7 @@ type TemporaryTable struct { func TempTableFromMeta(tblInfo *model.TableInfo) tableutil.TempTable { return &TemporaryTable{ modified: false, - stats: statistics.PseudoTable(tblInfo, false), + stats: statistics.PseudoTable(tblInfo, false, false), autoIDAllocator: autoid.NewAllocatorFromTempTblInfo(tblInfo), meta: tblInfo, } diff --git a/pkg/timer/BUILD.bazel b/pkg/timer/BUILD.bazel index 967cd29c3a9d3..4a4ab63159e87 100644 --- a/pkg/timer/BUILD.bazel +++ b/pkg/timer/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 5, + shard_count = 6, deps = [ "//pkg/sessionctx", "//pkg/testkit", diff --git a/pkg/timer/api/client_test.go b/pkg/timer/api/client_test.go index ac8316cdf7905..888defc1170ac 100644 --- a/pkg/timer/api/client_test.go +++ b/pkg/timer/api/client_test.go @@ -246,7 +246,7 @@ func TestDefaultClient(t *testing.T) { require.Empty(t, timer.EventData) require.True(t, timer.EventStart.IsZero()) require.Equal(t, []byte("s1"), timer.SummaryData) - require.Equal(t, eventStart, timer.Watermark) + require.Equal(t, eventStart.Unix(), timer.Watermark.Unix()) require.Equal(t, EventExtra{}, timer.EventExtra) // close event with option @@ -267,7 +267,7 @@ func TestDefaultClient(t *testing.T) { require.Empty(t, timer.EventData) require.True(t, timer.EventStart.IsZero()) require.Equal(t, []byte("s2"), timer.SummaryData) - require.Equal(t, watermark, timer.Watermark) + require.Equal(t, watermark.Unix(), timer.Watermark.Unix()) // manual trigger err = store.Update(ctx, timer.ID, &TimerUpdate{ diff --git a/pkg/timer/api/mem_store.go b/pkg/timer/api/mem_store.go index 803fa1d3fa16e..d46e24067a6ee 100644 --- a/pkg/timer/api/mem_store.go +++ b/pkg/timer/api/mem_store.go @@ -84,6 +84,8 @@ func (s *memoryStoreCore) Create(_ context.Context, record *TimerRecord) (string record.EventStatus = SchedEventIdle } + normalizeTimeFields(record) + if _, ok := s.id2Timers[record.ID]; ok { return "", errors.Trace(ErrTimerExists) } @@ -137,6 +139,7 @@ func (s *memoryStoreCore) Update(_ context.Context, timerID string, update *Time return err } + normalizeTimeFields(newRecord) if err = newRecord.Validate(); err != nil { return err } @@ -303,3 +306,21 @@ func getMemStoreTimeZoneLoc(tz string) *time.Location { return timeutil.SystemLocation() } + +func normalizeTimeFields(record *TimerRecord) { + if record.Location == nil { + return + } + + if !record.Watermark.IsZero() { + record.Watermark = record.Watermark.In(record.Location) + } + + if !record.EventStart.IsZero() { + record.EventStart = record.EventStart.In(record.Location) + } + + if !record.CreateTime.IsZero() { + record.CreateTime = record.CreateTime.In(record.Location) + } +} diff --git a/pkg/timer/runtime/worker_test.go b/pkg/timer/runtime/worker_test.go index 0c2aef2079bd7..21983bdaf7cc2 100644 --- a/pkg/timer/runtime/worker_test.go +++ b/pkg/timer/runtime/worker_test.go @@ -106,7 +106,7 @@ func prepareTimer(t *testing.T, cli api.TimerClient) *api.TimerRecord { require.Equal(t, "1m", timer.SchedPolicyExpr) require.Equal(t, "h1", timer.HookClass) require.True(t, timer.Enable) - require.Equal(t, watermark, timer.Watermark) + require.Equal(t, watermark.Unix(), timer.Watermark.Unix()) require.Equal(t, []byte("summary1"), timer.SummaryData) require.True(t, !timer.CreateTime.Before(now)) require.True(t, !timer.CreateTime.After(time.Now())) diff --git a/pkg/timer/store_intergartion_test.go b/pkg/timer/store_intergartion_test.go index b68a6a2b847d3..954c906ac17cc 100644 --- a/pkg/timer/store_intergartion_test.go +++ b/pkg/timer/store_intergartion_test.go @@ -16,6 +16,7 @@ package timer_test import ( "context" + "fmt" "sync/atomic" "testing" "time" @@ -256,6 +257,7 @@ func runTimerStoreUpdate(ctx context.Context, t *testing.T, store *api.TimerStor EventManualRequestID: "req2", EventWatermark: time.Unix(456, 0), } + tpl.CreateTime = tpl.CreateTime.In(time.UTC) require.Equal(t, *tpl, *record) // tags full update again @@ -328,6 +330,7 @@ func runTimerStoreUpdate(ctx context.Context, t *testing.T, store *api.TimerStor tpl.EventExtra = api.EventExtra{} tpl.Watermark = zeroTime tpl.SummaryData = nil + tpl.CreateTime = tpl.CreateTime.In(tpl.Location) require.Equal(t, *tpl, *record) // err check version @@ -872,3 +875,136 @@ func TestTableStoreManualTrigger(t *testing.T) { require.True(t, timer.ManualProcessed) require.Equal(t, api.EventExtra{}, timer.EventExtra) } + +func TestTimerStoreWithTimeZone(t *testing.T) { + // mem store + testTimerStoreWithTimeZone(t, api.NewMemoryTimerStore(), timeutil.SystemLocation().String()) + + // table store + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + dbName := "test" + tblName := "timerstore" + tk.MustExec("use test") + tk.MustExec(tablestore.CreateTimerTableSQL(dbName, tblName)) + tk.MustExec("set @@time_zone = 'America/Los_Angeles'") + + pool := pools.NewResourcePool(func() (pools.Resource, error) { + return tk.Session(), nil + }, 1, 1, time.Second) + defer pool.Close() + + timerStore := tablestore.NewTableTimerStore(1, pool, dbName, tblName, nil) + defer timerStore.Close() + + testTimerStoreWithTimeZone(t, timerStore, timeutil.SystemLocation().String()) + tk.MustExec("set @@global.time_zone='Asia/Tokyo'") + tk.MustExec(fmt.Sprintf("truncate table %s.%s", dbName, tblName)) + testTimerStoreWithTimeZone(t, timerStore, "Asia/Tokyo") + + // check time zone should be set back to the previous one. + require.Equal(t, "America/Los_Angeles", tk.Session().GetSessionVars().Location().String()) +} + +func testTimerStoreWithTimeZone(t *testing.T, timerStore *api.TimerStore, defaultTZ string) { + // 2024-11-03 09:30:00 UTC is 2024-11-03 01:30:00 -08:00 in `America/Los_Angeles` + // We should notice that it should not be regarded as 2024-11-03 01:30:00 -07:00 + // because of DST these two times have the same format in time zone `America/Los_Angeles`. + time1, err := time.ParseInLocation(time.DateTime, "2024-11-03 09:30:00", time.UTC) + require.NoError(t, err) + + time2, err := time.ParseInLocation(time.DateTime, "2024-11-03 08:30:00", time.UTC) + require.NoError(t, err) + + id1, err := timerStore.Create(context.TODO(), &api.TimerRecord{ + TimerSpec: api.TimerSpec{ + Namespace: "default", + Key: "test1", + SchedPolicyType: api.SchedEventInterval, + SchedPolicyExpr: "1h", + Watermark: time1, + }, + EventStatus: api.SchedEventTrigger, + EventStart: time2, + }) + require.NoError(t, err) + + id2, err := timerStore.Create(context.TODO(), &api.TimerRecord{ + TimerSpec: api.TimerSpec{ + Namespace: "default", + Key: "test2", + SchedPolicyType: api.SchedEventInterval, + SchedPolicyExpr: "1h", + Watermark: time2, + }, + EventStatus: api.SchedEventTrigger, + EventStart: time1, + }) + require.NoError(t, err) + + // create case + timer1, err := timerStore.GetByID(context.TODO(), id1) + require.NoError(t, err) + require.Equal(t, time1.In(time.UTC).String(), timer1.Watermark.In(time.UTC).String()) + require.Equal(t, time2.In(time.UTC).String(), timer1.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer1, defaultTZ) + + timer2, err := timerStore.GetByID(context.TODO(), id2) + require.NoError(t, err) + require.Equal(t, time2.In(time.UTC).String(), timer2.Watermark.In(time.UTC).String()) + require.Equal(t, time1.In(time.UTC).String(), timer2.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer2, defaultTZ) + + // update time + require.NoError(t, timerStore.Update(context.TODO(), id1, &api.TimerUpdate{ + Watermark: api.NewOptionalVal(time2), + EventStart: api.NewOptionalVal(time1), + })) + + require.NoError(t, timerStore.Update(context.TODO(), id2, &api.TimerUpdate{ + Watermark: api.NewOptionalVal(time1), + EventStart: api.NewOptionalVal(time2), + })) + + timer1, err = timerStore.GetByID(context.TODO(), id1) + require.NoError(t, err) + require.Equal(t, time2.In(time.UTC).String(), timer1.Watermark.In(time.UTC).String()) + require.Equal(t, time1.In(time.UTC).String(), timer1.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer1, defaultTZ) + + timer2, err = timerStore.GetByID(context.TODO(), id2) + require.NoError(t, err) + require.Equal(t, time1.In(time.UTC).String(), timer2.Watermark.In(time.UTC).String()) + require.Equal(t, time2.In(time.UTC).String(), timer2.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer2, defaultTZ) + + // update timezone + require.NoError(t, timerStore.Update(context.TODO(), id1, &api.TimerUpdate{ + TimeZone: api.NewOptionalVal("Europe/Berlin"), + })) + + timer1, err = timerStore.GetByID(context.TODO(), id1) + require.NoError(t, err) + require.Equal(t, time2.In(time.UTC).String(), timer1.Watermark.In(time.UTC).String()) + require.Equal(t, time1.In(time.UTC).String(), timer1.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer1, "Europe/Berlin") + + require.NoError(t, timerStore.Update(context.TODO(), id1, &api.TimerUpdate{ + TimeZone: api.NewOptionalVal(""), + })) + + timer1, err = timerStore.GetByID(context.TODO(), id1) + require.NoError(t, err) + require.Equal(t, time2.In(time.UTC).String(), timer1.Watermark.In(time.UTC).String()) + require.Equal(t, time1.In(time.UTC).String(), timer1.EventStart.In(time.UTC).String()) + checkTimerRecordLocation(t, timer1, defaultTZ) +} + +func checkTimerRecordLocation(t *testing.T, r *api.TimerRecord, tz string) { + require.Equal(t, tz, r.Location.String()) + require.Same(t, r.Location, r.Watermark.Location()) + require.Same(t, r.Location, r.CreateTime.Location()) + if !r.EventStart.IsZero() { + require.Same(t, r.Location, r.EventStart.Location()) + } +} diff --git a/pkg/timer/tablestore/BUILD.bazel b/pkg/timer/tablestore/BUILD.bazel index 9af2e0252410c..1a49d4bf46803 100644 --- a/pkg/timer/tablestore/BUILD.bazel +++ b/pkg/timer/tablestore/BUILD.bazel @@ -39,9 +39,13 @@ go_test( shard_count = 8, deps = [ "//pkg/kv", + "//pkg/parser/ast", + "//pkg/parser/model", + "//pkg/parser/mysql", "//pkg/sessionctx", "//pkg/sessionctx/variable", "//pkg/timer/api", + "//pkg/types", "//pkg/util/sqlexec", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", diff --git a/pkg/timer/tablestore/sql_test.go b/pkg/timer/tablestore/sql_test.go index 3b63314e3906e..083d4b2f98f00 100644 --- a/pkg/timer/tablestore/sql_test.go +++ b/pkg/timer/tablestore/sql_test.go @@ -25,9 +25,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/timer/api" + "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -614,9 +618,56 @@ func TestTakeSession(t *testing.T) { require.EqualError(t, err, "mockErr") pool.AssertExpectations(t) - // Get returns a session + // init session returns error se := &mockSession{} pool.On("Get").Return(se, nil).Once() + se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). + Return(nil, errors.New("mockErr")). + Once() + pool.On("Put", se).Once() + r, back, err = core.takeSession() + require.Nil(t, r) + require.Nil(t, back) + require.EqualError(t, err, "mockErr") + pool.AssertExpectations(t) + se.AssertExpectations(t) + + // init session returns error2 + pool.On("Get").Return(se, nil).Once() + se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). + Return(nil, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SELECT @@time_zone", []any(nil)). + Return(nil, errors.New("mockErr2")). + Once() + pool.On("Put", se).Once() + r, back, err = core.takeSession() + require.Nil(t, r) + require.Nil(t, back) + require.EqualError(t, err, "mockErr2") + pool.AssertExpectations(t) + se.AssertExpectations(t) + + // Get returns a session + pool.On("Get").Return(se, nil).Once() + rs := &sqlexec.SimpleRecordSet{ + ResultFields: []*ast.ResultField{{ + Column: &model.ColumnInfo{ + FieldType: *types.NewFieldType(mysql.TypeString), + }, + }}, + MaxChunkSize: 1, + Rows: [][]any{{"tz1"}}, + } + se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). + Return(nil, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SELECT @@time_zone", []any(nil)). + Return(rs, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SET @@time_zone='UTC'", []any(nil)). + Return(nil, nil). + Once() r, back, err = core.takeSession() require.Equal(t, r, se) require.NotNil(t, back) @@ -633,8 +684,29 @@ func TestTakeSession(t *testing.T) { pool.AssertExpectations(t) se.AssertExpectations(t) + // Put session failed2 + se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). + Return(nil, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SET @@time_zone=%?", []any{"tz1"}). + Return(nil, errors.New("mockErr2")). + Once() + se.On("Close").Once() + back() + pool.AssertExpectations(t) + se.AssertExpectations(t) + // Put session success pool.On("Get").Return(se, nil).Once() + se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). + Return(nil, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SELECT @@time_zone", []any(nil)). + Return(rs, nil). + Once() + se.On("ExecuteInternal", matchCtx, "SET @@time_zone='UTC'", []any(nil)). + Return(nil, nil). + Once() r, back, err = core.takeSession() require.Equal(t, r, se) require.NotNil(t, back) @@ -642,6 +714,9 @@ func TestTakeSession(t *testing.T) { se.On("ExecuteInternal", matchCtx, "ROLLBACK", []any(nil)). Return(nil, nil). Once() + se.On("ExecuteInternal", matchCtx, "SET @@time_zone=%?", []any{"tz1"}). + Return(nil, nil). + Once() pool.On("Put", se).Once() back() pool.AssertExpectations(t) diff --git a/pkg/timer/tablestore/store.go b/pkg/timer/tablestore/store.go index 988c23feb0961..3e6f275ed6002 100644 --- a/pkg/timer/tablestore/store.go +++ b/pkg/timer/tablestore/store.go @@ -141,6 +141,11 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T return nil, err } + tidbTimeZone, err := sctx.GetSessionVars().GetGlobalSystemVar(ctx, variable.TimeZone) + if err != nil { + return nil, err + } + timers := make([]*api.TimerRecord, 0, len(rows)) for _, row := range rows { var timerData []byte @@ -148,12 +153,25 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T timerData = row.GetBytes(3) } + tz := row.GetString(4) + tzParse := tz + // handling value "TIDB" is for compatibility of version 7.3.0 + if tz == "" || strings.EqualFold(tz, "TIDB") { + tzParse = tidbTimeZone + } + + loc, err := timeutil.ParseTimeZone(tzParse) + if err != nil { + loc = timeutil.SystemLocation() + } + var watermark time.Time if !row.IsNull(8) { watermark, err = row.GetTime(8).GoTime(seTZ) if err != nil { return nil, err } + watermark = watermark.In(loc) } var ext timerExt @@ -175,6 +193,7 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T if err != nil { return nil, err } + eventStart = eventStart.In(loc) } var summaryData []byte @@ -188,6 +207,7 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T if err != nil { return nil, err } + createTime = createTime.In(loc) } timer := &api.TimerRecord{ @@ -197,7 +217,7 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T Key: row.GetString(2), Tags: ext.Tags, Data: timerData, - TimeZone: row.GetString(4), + TimeZone: tz, SchedPolicyType: api.SchedPolicyType(row.GetString(5)), SchedPolicyExpr: row.GetString(6), HookClass: row.GetString(7), @@ -211,25 +231,10 @@ func (s *tableTimerStoreCore) List(ctx context.Context, cond api.Cond) ([]*api.T EventStart: eventStart, EventExtra: ext.Event.ToEventExtra(), SummaryData: summaryData, + Location: loc, CreateTime: createTime, Version: row.GetUint64(18), } - - tz := timer.TimeZone - // handling value "TIDB" is for compatibility of version 7.3.0 - if tz == "" || strings.EqualFold(tz, "TIDB") { - if tz, err = sctx.GetSessionVars().GetGlobalSystemVar(ctx, variable.TimeZone); err != nil { - return nil, err - } - } - - loc, err := timeutil.ParseTimeZone(tz) - if err == nil { - timer.Location = loc - } else { - timer.Location = timeutil.SystemLocation() - } - timers = append(timers, timer) } return timers, nil @@ -327,20 +332,47 @@ func (s *tableTimerStoreCore) Close() { s.notifier.Close() } -func (s *tableTimerStoreCore) takeSession() (sessionctx.Context, func(), error) { +func (s *tableTimerStoreCore) takeSession() (_ sessionctx.Context, _ func(), err error) { r, err := s.pool.Get() if err != nil { return nil, nil, err } + defer func() { + if err != nil { + s.pool.Put(r) + } + }() + sctx, ok := r.(sessionctx.Context) if !ok { - s.pool.Put(r) return nil, nil, errors.New("session is not the type sessionctx.Context") } + ctx := context.Background() + + // rollback first to terminate unexpected transactions + if _, err = executeSQL(ctx, sctx, "ROLLBACK"); err != nil { + return nil, nil, err + } + + // we should force to set time zone to UTC to make sure time operations are consistent. + rows, err := executeSQL(ctx, sctx, "SELECT @@time_zone") + if err != nil { + return nil, nil, err + } + + if len(rows) == 0 || rows[0].Len() == 0 { + return nil, nil, errors.New("failed to get original time zone of session") + } + + if _, err = executeSQL(ctx, sctx, "SET @@time_zone='UTC'"); err != nil { + return nil, nil, err + } + + originalTimeZone := rows[0].GetString(0) back := func() { - if _, err = executeSQL(context.Background(), sctx, "ROLLBACK"); err != nil { + if _, err = executeSQL(ctx, sctx, "ROLLBACK"); err != nil { // Though this branch is rarely to be called because "ROLLBACK" will always be successfully, we still need // to handle it here to make sure the code is strong. terror.Log(err) @@ -348,6 +380,13 @@ func (s *tableTimerStoreCore) takeSession() (sessionctx.Context, func(), error) r.Close() return } + + if _, err = executeSQL(ctx, sctx, "SET @@time_zone=%?", originalTimeZone); err != nil { + terror.Log(err) + r.Close() + return + } + s.pool.Put(r) } diff --git a/pkg/ttl/cache/split_test.go b/pkg/ttl/cache/split_test.go index 935b97d342151..5f5fbef1ca3c2 100644 --- a/pkg/ttl/cache/split_test.go +++ b/pkg/ttl/cache/split_test.go @@ -104,6 +104,10 @@ func (c *mockPDClient) GetStore(_ context.Context, storeID uint64) (*metapb.Stor }, nil } +func (c *mockPDClient) GetClusterID(_ context.Context) uint64 { + return 1 +} + type mockTiKVStore struct { t *testing.T helper.Storage diff --git a/pkg/types/mydecimal.go b/pkg/types/mydecimal.go index f2127bf4fbfe2..27a060fb807f1 100644 --- a/pkg/types/mydecimal.go +++ b/pkg/types/mydecimal.go @@ -51,8 +51,6 @@ const ( wordMax = wordBase - 1 notFixedDec = 31 - DivFracIncr = 4 - // Round up to the next integer if positive or down to the next integer if negative. ModeHalfUp RoundMode = 5 // Truncate just truncates the decimal. diff --git a/pkg/types/mydecimal_test.go b/pkg/types/mydecimal_test.go index 3bf7be1683997..697f8b18bb9c3 100644 --- a/pkg/types/mydecimal_test.go +++ b/pkg/types/mydecimal_test.go @@ -780,7 +780,7 @@ func TestDivModMyDecimal(t *testing.T) { require.NoError(t, err) err = b.FromString([]byte(tt.b)) require.NoError(t, err) - ec := DecimalDiv(&a, &b, &to, DivFracIncr) + ec := DecimalDiv(&a, &b, &to, 4) require.Equal(t, tt.err, ec) if tt.err == ErrDivByZero { continue diff --git a/pkg/util/cgmon/cgmon.go b/pkg/util/cgmon/cgmon.go index dbcae4e81f104..6d79502d410ef 100644 --- a/pkg/util/cgmon/cgmon.go +++ b/pkg/util/cgmon/cgmon.go @@ -49,9 +49,11 @@ func StartCgroupMonitor() { if started { return } + if runtime.GOOS != "linux" { + return + } started = true // Get configured maxprocs. - cfgMaxProcs = runtime.GOMAXPROCS(0) ctx, cancel = context.WithCancel(context.Background()) wg.Add(1) go refreshCgroupLoop() @@ -64,6 +66,9 @@ func StopCgroupMonitor() { if !started { return } + if runtime.GOOS != "linux" { + return + } started = false if cancel != nil { cancel() @@ -80,28 +85,39 @@ func refreshCgroupLoop() { }() defer util.Recover("cgmon", "refreshCgroupLoop", nil, false) - refreshCgroupCPU() - refreshCgroupMemory() + err := refreshCgroupCPU() + if err != nil { + log.Warn("failed to get cgroup cpu quota", zap.Error(err)) + } + err = refreshCgroupMemory() + if err != nil { + log.Warn("failed to get cgroup memory limit", zap.Error(err)) + } for { select { case <-ctx.Done(): return case <-ticker.C: - refreshCgroupCPU() - refreshCgroupMemory() + err = refreshCgroupCPU() + if err != nil { + log.Debug("failed to get cgroup cpu quota", zap.Error(err)) + } + err = refreshCgroupMemory() + if err != nil { + log.Debug("failed to get cgroup memory limit", zap.Error(err)) + } } } } -func refreshCgroupCPU() { +func refreshCgroupCPU() error { // Get the number of CPUs. quota := runtime.NumCPU() // Get CPU quota from cgroup. cpuPeriod, cpuQuota, err := cgroup.GetCPUPeriodAndQuota() if err != nil { - log.Warn("failed to get cgroup cpu quota", zap.Error(err)) - return + return err } if cpuPeriod > 0 && cpuQuota > 0 { ratio := float64(cpuQuota) / float64(cpuPeriod) @@ -110,8 +126,7 @@ func refreshCgroupCPU() { } } - if quota != lastMaxProcs && quota < cfgMaxProcs { - runtime.GOMAXPROCS(quota) + if quota != lastMaxProcs { log.Info("set the maxprocs", zap.Int("quota", quota)) metrics.MaxProcs.Set(float64(quota)) lastMaxProcs = quota @@ -120,18 +135,17 @@ func refreshCgroupCPU() { metrics.MaxProcs.Set(float64(cfgMaxProcs)) lastMaxProcs = cfgMaxProcs } + return nil } -func refreshCgroupMemory() { +func refreshCgroupMemory() error { memLimit, err := cgroup.GetMemoryLimit() if err != nil { - log.Warn("failed to get cgroup memory limit", zap.Error(err)) - return + return err } vmem, err := mem.VirtualMemory() if err != nil { - log.Warn("failed to get system memory size", zap.Error(err)) - return + return err } if memLimit > vmem.Total { memLimit = vmem.Total @@ -141,4 +155,5 @@ func refreshCgroupMemory() { metrics.MemoryLimit.Set(float64(memLimit)) lastMemoryLimit = memLimit } + return nil } diff --git a/pkg/util/cgroup/cgroup_memory.go b/pkg/util/cgroup/cgroup_memory.go index 012cac3cb52b0..664a9da5a7b1e 100644 --- a/pkg/util/cgroup/cgroup_memory.go +++ b/pkg/util/cgroup/cgroup_memory.go @@ -26,9 +26,26 @@ import ( "github.com/pingcap/log" ) +// Version represents the cgroup version. +type Version int + +// cgroup versions. +const ( + Unknown Version = 0 + V1 Version = 1 + V2 Version = 2 +) + // GetMemoryLimit attempts to retrieve the cgroup memory limit for the current // process. func GetMemoryLimit() (limit uint64, err error) { + limit, _, err = getCgroupMemLimit("/") + return +} + +// GetCgroupMemLimit attempts to retrieve the cgroup memory limit for the current +// process, and return cgroup version too. +func GetCgroupMemLimit() (uint64, Version, error) { return getCgroupMemLimit("/") } @@ -118,41 +135,46 @@ func getCgroupMemUsage(root string) (usage uint64, err error) { } // root is always "/" in the production. It will be changed for testing. -func getCgroupMemLimit(root string) (limit uint64, err error) { +func getCgroupMemLimit(root string) (limit uint64, version Version, err error) { + version = Unknown path, err := detectControlPath(filepath.Join(root, procPathCGroup), "memory") if err != nil { - return 0, err + return 0, version, err } if path == "" { log.Warn("no cgroup memory controller detected") - return 0, nil + return 0, version, nil } mount, ver, err := getCgroupDetails(filepath.Join(root, procPathMountInfo), path, "memory") if err != nil { - return 0, err + return 0, version, err } if len(ver) == 2 { + version = V1 limit, err = detectMemLimitInV1(filepath.Join(root, mount[0])) if err != nil { + version = V2 limit, err = detectMemLimitInV2(filepath.Join(root, mount[1], path)) } } else { switch ver[0] { case 1: // cgroupv1 + version = V1 limit, err = detectMemLimitInV1(filepath.Join(root, mount[0])) case 2: // cgroupv2 + version = V2 limit, err = detectMemLimitInV2(filepath.Join(root, mount[0], path)) default: limit, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) } } - return limit, err + return limit, version, err } func detectMemLimitInV1(cRoot string) (limit uint64, err error) { @@ -162,6 +184,12 @@ func detectMemLimitInV1(cRoot string) (limit uint64, err error) { // TODO(hawkingrei): this implementation was based on podman+criu environment. // It may cover not all the cases when v2 becomes more widely used in container // world. +// In K8S env, the value hold in memory.max is the memory limit defined in pod definition, +// the real memory limit should be the minimum value in the whole cgroup hierarchy +// as in cgroup V1, but cgroup V2 lacks the feature, see this patch for more details: +// https://lore.kernel.org/linux-kernel/ZctiM5RintD_D0Lt@host1.jankratochvil.net/T/ +// So, in cgroup V2, the value better be adjusted by some factor, like 0.9, to +// avoid OOM. see taskexecutor/manager.go too. func detectMemLimitInV2(cRoot string) (limit uint64, err error) { return readInt64Value(cRoot, cgroupV2MemLimit, 2) } diff --git a/pkg/util/cgroup/cgroup_mock_test.go b/pkg/util/cgroup/cgroup_mock_test.go index 01cc78a66d78f..590c415330fb0 100644 --- a/pkg/util/cgroup/cgroup_mock_test.go +++ b/pkg/util/cgroup/cgroup_mock_test.go @@ -211,14 +211,14 @@ func TestCgroupsGetMemoryInactiveFileUsage(t *testing.T) { func TestCgroupsGetMemoryLimit(t *testing.T) { for _, tc := range []struct { - name string - paths map[string]string - errMsg string - limit uint64 - warn string + name string + paths map[string]string + errMsg string + limit uint64 + warn string + version Version }{ { - errMsg: "failed to read memory cgroup from cgroups file:", }, { @@ -248,7 +248,8 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { "/proc/self/mountinfo": v1MountsWithMemController, "/sys/fs/cgroup/memory/memory.stat": v1MemoryStat, }, - limit: 2936016896, + limit: 2936016896, + version: V1, }, { paths: map[string]string{ @@ -256,7 +257,8 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { "/proc/self/mountinfo": v1MountsWithMemControllerNS, "/sys/fs/cgroup/memory/cgroup_test/memory.stat": v1MemoryStat, }, - limit: 2936016896, + limit: 2936016896, + version: V1, }, { paths: map[string]string{ @@ -279,7 +281,8 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { "/proc/self/mountinfo": v2Mounts, "/sys/fs/cgroup/machine.slice/libpod-f1c6b44c0d61f273952b8daecf154cee1be2d503b7e9184ebf7fcaf48e139810.scope/memory.max": "1073741824\n", }, - limit: 1073741824, + limit: 1073741824, + version: V2, }, { paths: map[string]string{ @@ -287,7 +290,8 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { "/proc/self/mountinfo": v2Mounts, "/sys/fs/cgroup/machine.slice/libpod-f1c6b44c0d61f273952b8daecf154cee1be2d503b7e9184ebf7fcaf48e139810.scope/memory.max": "max\n", }, - limit: 9223372036854775807, + limit: 9223372036854775807, + version: V2, }, { paths: map[string]string{ @@ -295,14 +299,18 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { "/proc/self/mountinfo": v1MountsWithEccentricMemController, "/sys/fs/cgroup/memory/memory.stat": v1MemoryStat, }, - limit: 2936016896, + limit: 2936016896, + version: V1, }, } { dir := createFiles(t, tc.paths) - limit, err := getCgroupMemLimit(dir) + limit, version, err := getCgroupMemLimit(dir) require.True(t, isError(err, tc.errMsg), "%v %v", err, tc.errMsg) require.Equal(t, tc.limit, limit) + if err == nil { + require.Equal(t, tc.version, version) + } } } diff --git a/pkg/util/chunk/BUILD.bazel b/pkg/util/chunk/BUILD.bazel index 5107e72c63d4f..7f3cfca2a24a6 100644 --- a/pkg/util/chunk/BUILD.bazel +++ b/pkg/util/chunk/BUILD.bazel @@ -65,7 +65,6 @@ go_test( deps = [ "//pkg/config", "//pkg/parser/mysql", - "//pkg/sessionctx/stmtctx", "//pkg/testkit/testsetup", "//pkg/types", "//pkg/util/collate", diff --git a/pkg/util/chunk/chunk_test.go b/pkg/util/chunk/chunk_test.go index 7ff2292db7f89..690f831001597 100644 --- a/pkg/util/chunk/chunk_test.go +++ b/pkg/util/chunk/chunk_test.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/stretchr/testify/require" ) @@ -544,8 +543,8 @@ func TestGetDecimalDatum(t *testing.T) { decType := types.NewFieldType(mysql.TypeNewDecimal) decType.SetFlen(4) decType.SetDecimal(2) - sc := stmtctx.NewStmtCtx() - decDatum, err := datum.ConvertTo(sc.TypeCtx(), decType) + typeCtx := types.DefaultStmtNoWarningContext + decDatum, err := datum.ConvertTo(typeCtx, decType) require.NoError(t, err) chk := NewChunkWithCapacity([]*types.FieldType{decType}, 32) diff --git a/pkg/util/chunk/mutrow_test.go b/pkg/util/chunk/mutrow_test.go index 96a69e224e184..c349a2b812166 100644 --- a/pkg/util/chunk/mutrow_test.go +++ b/pkg/util/chunk/mutrow_test.go @@ -19,7 +19,6 @@ import ( "time" "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/collate" "github.com/stretchr/testify/require" @@ -29,12 +28,12 @@ func TestMutRow(t *testing.T) { allTypes := newAllTypes() mutRow := MutRowFromTypes(allTypes) row := mutRow.ToRow() - sc := stmtctx.NewStmtCtx() + typeCtx := types.DefaultStmtNoWarningContext for i := 0; i < row.Len(); i++ { val := zeroValForType(allTypes[i]) d := row.GetDatum(i, allTypes[i]) d2 := types.NewDatum(val) - cmp, err := d.Compare(sc.TypeCtx(), &d2, collate.GetCollator(allTypes[i].GetCollate())) + cmp, err := d.Compare(typeCtx, &d2, collate.GetCollator(allTypes[i].GetCollate())) require.NoError(t, err) require.Equal(t, 0, cmp) } @@ -79,7 +78,7 @@ func TestMutRow(t *testing.T) { retTypes := []*types.FieldType{types.NewFieldType(mysql.TypeDuration)} chk := New(retTypes, 1, 1) - dur, _, err := types.ParseDuration(sc.TypeCtx(), "01:23:45", 0) + dur, _, err := types.ParseDuration(typeCtx, "01:23:45", 0) require.NoError(t, err) chk.AppendDuration(0, dur) mutRow = MutRowFromTypes(retTypes) diff --git a/pkg/util/codec/BUILD.bazel b/pkg/util/codec/BUILD.bazel index 02f96b5bad490..f2da06940be15 100644 --- a/pkg/util/codec/BUILD.bazel +++ b/pkg/util/codec/BUILD.bazel @@ -39,9 +39,9 @@ go_test( embed = [":codec"], flaky = True, deps = [ + "//pkg/errctx", "//pkg/parser/mysql", "//pkg/parser/terror", - "//pkg/sessionctx/stmtctx", "//pkg/testkit/testsetup", "//pkg/types", "//pkg/util/benchdaily", diff --git a/pkg/util/codec/codec_test.go b/pkg/util/codec/codec_test.go index c09b735185ebb..2cfcb7d5a0592 100644 --- a/pkg/util/codec/codec_test.go +++ b/pkg/util/codec/codec_test.go @@ -24,9 +24,9 @@ import ( "testing" "time" + "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/parser/terror" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/collate" @@ -72,20 +72,20 @@ func TestCodecKey(t *testing.T) { types.MakeDatums(uint64(1), uint64(1)), }, } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) for i, datums := range table { comment := fmt.Sprintf("%d %v", i, datums) - b, err := EncodeKey(sc.TimeZone(), nil, datums.Input...) + b, err := EncodeKey(typeCtx.Location(), nil, datums.Input...) require.NoError(t, err, comment) args, err := Decode(b, 1) require.NoError(t, err, comment) require.Equal(t, datums.Expect, args, comment) - b, err = EncodeValue(sc.TimeZone(), nil, datums.Input...) + b, err = EncodeValue(typeCtx.Location(), nil, datums.Input...) require.NoError(t, err, comment) - size, err := estimateValuesSize(sc, datums.Input) + size, err := estimateValuesSize(typeCtx, datums.Input) require.NoError(t, err, comment) require.Len(t, b, size, comment) @@ -96,14 +96,14 @@ func TestCodecKey(t *testing.T) { var raw types.Datum raw.SetRaw([]byte("raw")) - _, err := EncodeKey(sc.TimeZone(), nil, raw) + _, err := EncodeKey(typeCtx.Location(), nil, raw) require.Error(t, err) } -func estimateValuesSize(sc *stmtctx.StatementContext, vals []types.Datum) (int, error) { +func estimateValuesSize(typeCtx types.Context, vals []types.Datum) (int, error) { size := 0 for _, val := range vals { - length, err := EstimateValueSize(sc.TypeCtx(), val) + length, err := EstimateValueSize(typeCtx, val) if err != nil { return 0, err } @@ -214,12 +214,11 @@ func TestCodecKeyCompare(t *testing.T) { -1, }, } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) for _, datums := range table { - b1, err := EncodeKey(sc.TimeZone(), nil, datums.Left...) + b1, err := EncodeKey(time.Local, nil, datums.Left...) require.NoError(t, err) - b2, err := EncodeKey(sc.TimeZone(), nil, datums.Right...) + b2, err := EncodeKey(time.Local, nil, datums.Right...) require.NoError(t, err) comparedRes := bytes.Compare(b1, b2) @@ -519,8 +518,7 @@ func TestBytes(t *testing.T) { } func parseTime(t *testing.T, s string) types.Time { - sc := stmtctx.NewStmtCtxWithTimeZone(time.UTC) - m, err := types.ParseTime(sc.TypeCtx(), s, mysql.TypeDatetime, types.DefaultFsp) + m, err := types.ParseTime(types.DefaultStmtNoWarningContext, s, mysql.TypeDatetime, types.DefaultFsp) require.NoError(t, err) return m } @@ -537,11 +535,10 @@ func TestTime(t *testing.T) { "2011-01-01 00:00:00", "0001-01-01 00:00:00", } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) for _, timeDatum := range tbl { m := types.NewDatum(parseTime(t, timeDatum)) - b, err := EncodeKey(sc.TimeZone(), nil, m) + b, err := EncodeKey(time.Local, nil, m) require.NoError(t, err) v, err := Decode(b, 1) require.NoError(t, err) @@ -568,9 +565,9 @@ func TestTime(t *testing.T) { m1 := types.NewDatum(parseTime(t, timeData.Arg1)) m2 := types.NewDatum(parseTime(t, timeData.Arg2)) - b1, err := EncodeKey(sc.TimeZone(), nil, m1) + b1, err := EncodeKey(time.Local, nil, m1) require.NoError(t, err) - b2, err := EncodeKey(sc.TimeZone(), nil, m2) + b2, err := EncodeKey(time.Local, nil, m2) require.NoError(t, err) ret := bytes.Compare(b1, b2) @@ -584,11 +581,10 @@ func TestDuration(t *testing.T) { "00:00:00", "1 11:11:11", } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) for _, duration := range tbl { m := parseDuration(t, duration) - b, err := EncodeKey(sc.TimeZone(), nil, types.NewDatum(m)) + b, err := EncodeKey(time.Local, nil, types.NewDatum(m)) require.NoError(t, err) v, err := Decode(b, 1) require.NoError(t, err) @@ -610,9 +606,9 @@ func TestDuration(t *testing.T) { m1 := parseDuration(t, durations.Arg1) m2 := parseDuration(t, durations.Arg2) - b1, err := EncodeKey(sc.TimeZone(), nil, types.NewDatum(m1)) + b1, err := EncodeKey(time.Local, nil, types.NewDatum(m1)) require.NoError(t, err) - b2, err := EncodeKey(sc.TimeZone(), nil, types.NewDatum(m2)) + b2, err := EncodeKey(time.Local, nil, types.NewDatum(m2)) require.NoError(t, err) ret := bytes.Compare(b1, b2) @@ -637,13 +633,13 @@ func TestDecimal(t *testing.T) { "-12.340", "-0.1234", } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) for _, decimalNum := range tbl { dec := new(types.MyDecimal) err := dec.FromString([]byte(decimalNum)) require.NoError(t, err) - b, err := EncodeKey(sc.TimeZone(), nil, types.NewDatum(dec)) + b, err := EncodeKey(typeCtx.Location(), nil, types.NewDatum(dec)) require.NoError(t, err) v, err := Decode(b, 1) require.NoError(t, err) @@ -720,12 +716,12 @@ func TestDecimal(t *testing.T) { } for _, decimalNums := range tblCmp { d1 := types.NewDatum(decimalNums.Arg1) - dec1, err := d1.ToDecimal(sc.TypeCtxOrDefault()) + dec1, err := d1.ToDecimal(typeCtx) require.NoError(t, err) d1.SetMysqlDecimal(dec1) d2 := types.NewDatum(decimalNums.Arg2) - dec2, err := d2.ToDecimal(sc.TypeCtxOrDefault()) + dec2, err := d2.ToDecimal(typeCtx) require.NoError(t, err) d2.SetMysqlDecimal(dec2) @@ -734,17 +730,17 @@ func TestDecimal(t *testing.T) { d2.SetLength(30) d2.SetFrac(6) - b1, err := EncodeKey(sc.TimeZone(), nil, d1) + b1, err := EncodeKey(typeCtx.Location(), nil, d1) require.NoError(t, err) - b2, err := EncodeKey(sc.TimeZone(), nil, d2) + b2, err := EncodeKey(typeCtx.Location(), nil, d2) require.NoError(t, err) ret := bytes.Compare(b1, b2) require.Equalf(t, decimalNums.Ret, ret, "%v %x %x", decimalNums, b1, b2) - b1, err = EncodeValue(sc.TimeZone(), b1[:0], d1) + b1, err = EncodeValue(typeCtx.Location(), b1[:0], d1) require.NoError(t, err) - size, err := EstimateValueSize(sc.TypeCtx(), d1) + size, err := EstimateValueSize(typeCtx, d1) require.NoError(t, err) require.Len(t, b1, size) } @@ -761,7 +757,7 @@ func TestDecimal(t *testing.T) { b, err := EncodeDecimal(nil, d.GetMysqlDecimal(), d.Length(), d.Frac()) require.NoError(t, err) decs = append(decs, b) - size, err := EstimateValueSize(sc.TypeCtx(), d) + size, err := EstimateValueSize(typeCtx, d) require.NoError(t, err) // size - 1 because the flag occupy 1 bit. require.Len(t, b, size-1) @@ -778,19 +774,19 @@ func TestDecimal(t *testing.T) { _, err = EncodeDecimal(nil, d, 12, 10) require.Truef(t, terror.ErrorEqual(err, types.ErrOverflow), "err %v", err) - sc.SetTypeFlags(types.DefaultStmtFlags.WithIgnoreTruncateErr(true)) + errCtx := errctx.StrictNoWarningContext.WithErrGroupLevel(errctx.ErrGroupTruncate, errctx.LevelIgnore) decimalDatum := types.NewDatum(d) decimalDatum.SetLength(20) decimalDatum.SetFrac(5) - _, err = EncodeValue(sc.TimeZone(), nil, decimalDatum) - err = sc.HandleError(err) + _, err = EncodeValue(typeCtx.Location(), nil, decimalDatum) + err = errCtx.HandleError(err) require.NoError(t, err) - sc.SetTypeFlags(types.DefaultStmtFlags.WithTruncateAsWarning(true)) + errCtx = errctx.StrictNoWarningContext.WithErrGroupLevel(errctx.ErrGroupTruncate, errctx.LevelWarn) decimalDatum.SetLength(12) decimalDatum.SetFrac(10) - _, err = EncodeValue(sc.TimeZone(), nil, decimalDatum) - err = sc.HandleError(err) + _, err = EncodeValue(typeCtx.Location(), nil, decimalDatum) + err = errCtx.HandleError(err) require.NoError(t, err) } @@ -878,9 +874,8 @@ func TestCut(t *testing.T) { types.MakeDatums(types.CreateBinaryJSON(types.Opaque{TypeCode: mysql.TypeString, Buf: []byte("abc")})), }, } - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) for i, datums := range table { - b, err := EncodeKey(sc.TimeZone(), nil, datums.Input...) + b, err := EncodeKey(time.Local, nil, datums.Input...) require.NoErrorf(t, err, "%d %v", i, datums) var d []byte @@ -889,7 +884,7 @@ func TestCut(t *testing.T) { require.NoError(t, err) require.NotNil(t, d) - ed, err1 := EncodeKey(sc.TimeZone(), nil, e) + ed, err1 := EncodeKey(time.Local, nil, e) require.NoError(t, err1) require.Equalf(t, ed, d, "%d:%d %#v", i, j, e) } @@ -897,7 +892,7 @@ func TestCut(t *testing.T) { } for i, datums := range table { - b, err := EncodeValue(sc.TimeZone(), nil, datums.Input...) + b, err := EncodeValue(time.Local, nil, datums.Input...) require.NoErrorf(t, err, "%d %v", i, datums) var d []byte @@ -906,7 +901,7 @@ func TestCut(t *testing.T) { require.NoError(t, err) require.NotNil(t, d) - ed, err1 := EncodeValue(sc.TimeZone(), nil, e) + ed, err1 := EncodeValue(time.Local, nil, e) require.NoError(t, err1) require.Equalf(t, ed, d, "%d:%d %#v", i, j, e) } @@ -914,7 +909,7 @@ func TestCut(t *testing.T) { } input := 42 - b, err := EncodeValue(sc.TimeZone(), nil, types.NewDatum(input)) + b, err := EncodeValue(time.Local, nil, types.NewDatum(input)) require.NoError(t, err) rem, n, err := CutColumnID(b) require.NoError(t, err) @@ -935,9 +930,8 @@ func TestCutOneError(t *testing.T) { } func TestSetRawValues(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) datums := types.MakeDatums(1, "abc", 1.1, []byte("def")) - rowData, err := EncodeValue(sc.TimeZone(), nil, datums...) + rowData, err := EncodeValue(time.Local, nil, datums...) require.NoError(t, err) values := make([]types.Datum, 4) @@ -946,17 +940,17 @@ func TestSetRawValues(t *testing.T) { for i, rawVal := range values { require.IsType(t, types.KindRaw, rawVal.Kind()) - encoded, encodedErr := EncodeValue(sc.TimeZone(), nil, datums[i]) + encoded, encodedErr := EncodeValue(time.Local, nil, datums[i]) require.NoError(t, encodedErr) require.Equal(t, rawVal.GetBytes(), encoded) } } func TestDecodeOneToChunk(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) - datums, tps := datumsForTest(sc) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) + datums, tps := datumsForTest() rowCount := 3 - chk := chunkForTest(t, sc, datums, tps, rowCount) + chk := chunkForTest(t, typeCtx.Location(), datums, tps, rowCount) for colIdx, tp := range tps { for rowIdx := 0; rowIdx < rowCount; rowIdx++ { got := chk.GetRow(rowIdx).GetDatum(colIdx, tp) @@ -965,7 +959,7 @@ func TestDecodeOneToChunk(t *testing.T) { require.True(t, expect.IsNull()) } else { if got.Kind() != types.KindMysqlDecimal { - cmp, err := got.Compare(sc.TypeCtx(), &expect, collate.GetCollator(tp.GetCollate())) + cmp, err := got.Compare(typeCtx, &expect, collate.GetCollator(tp.GetCollate())) require.NoError(t, err) require.Equalf(t, 0, cmp, "expect: %v, got %v", expect, got) } else { @@ -977,7 +971,6 @@ func TestDecodeOneToChunk(t *testing.T) { } func TestHashGroup(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) tp := types.NewFieldType(mysql.TypeNewDecimal) tps := []*types.FieldType{tp} chk1 := chunk.New(tps, 3, 3) @@ -990,17 +983,17 @@ func TestHashGroup(t *testing.T) { tp1 := tp tp1.SetFlen(20) tp1.SetDecimal(5) - _, err := HashGroupKey(sc.TimeZone(), 3, chk1.Column(0), buf1, tp1) + _, err := HashGroupKey(time.Local, 3, chk1.Column(0), buf1, tp1) require.Error(t, err) tp2 := tp tp2.SetFlen(12) tp2.SetDecimal(10) - _, err = HashGroupKey(sc.TimeZone(), 3, chk1.Column(0), buf1, tp2) + _, err = HashGroupKey(time.Local, 3, chk1.Column(0), buf1, tp2) require.Error(t, err) } -func datumsForTest(_ *stmtctx.StatementContext) ([]types.Datum, []*types.FieldType) { +func datumsForTest() ([]types.Datum, []*types.FieldType) { decType := types.NewFieldType(mysql.TypeNewDecimal) decType.SetDecimal(2) _tp1 := types.NewFieldType(mysql.TypeEnum) @@ -1067,10 +1060,10 @@ func datumsForTest(_ *stmtctx.StatementContext) ([]types.Datum, []*types.FieldTy return datums, tps } -func chunkForTest(t *testing.T, sc *stmtctx.StatementContext, datums []types.Datum, tps []*types.FieldType, rowCount int) *chunk.Chunk { - decoder := NewDecoder(chunk.New(tps, 32, 32), sc.TimeZone()) +func chunkForTest(t *testing.T, tz *time.Location, datums []types.Datum, tps []*types.FieldType, rowCount int) *chunk.Chunk { + decoder := NewDecoder(chunk.New(tps, 32, 32), tz) for rowIdx := 0; rowIdx < rowCount; rowIdx++ { - encoded, err := EncodeValue(sc.TimeZone(), nil, datums...) + encoded, err := EncodeValue(tz, nil, datums...) require.NoError(t, err) decoder.buf = make([]byte, 0, len(encoded)) for colIdx, tp := range tps { @@ -1105,7 +1098,7 @@ func TestDecodeRange(t *testing.T) { } func testHashChunkRowEqual(t *testing.T, a, b any, equal bool) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) buf1 := make([]byte, 1) buf2 := make([]byte, 1) @@ -1124,10 +1117,10 @@ func testHashChunkRowEqual(t *testing.T, a, b any, equal bool) { chk2.AppendDatum(0, &d) h := crc32.NewIEEE() - err1 := HashChunkRow(sc.TypeCtx(), h, chk1.GetRow(0), []*types.FieldType{tp1}, []int{0}, buf1) + err1 := HashChunkRow(typeCtx, h, chk1.GetRow(0), []*types.FieldType{tp1}, []int{0}, buf1) sum1 := h.Sum32() h.Reset() - err2 := HashChunkRow(sc.TypeCtx(), h, chk2.GetRow(0), []*types.FieldType{tp2}, []int{0}, buf2) + err2 := HashChunkRow(typeCtx, h, chk2.GetRow(0), []*types.FieldType{tp2}, []int{0}, buf2) sum2 := h.Sum32() require.NoError(t, err1) require.NoError(t, err2) @@ -1136,7 +1129,7 @@ func testHashChunkRowEqual(t *testing.T, a, b any, equal bool) { } else { require.NotEqual(t, sum2, sum1) } - e, err := EqualChunkRow(sc.TypeCtx(), + e, err := EqualChunkRow(typeCtx, chk1.GetRow(0), []*types.FieldType{tp1}, []int{0}, chk2.GetRow(0), []*types.FieldType{tp2}, []int{0}) require.NoError(t, err) @@ -1148,26 +1141,26 @@ func testHashChunkRowEqual(t *testing.T, a, b any, equal bool) { } func TestHashChunkRow(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) buf := make([]byte, 1) - datums, tps := datumsForTest(sc) - chk := chunkForTest(t, sc, datums, tps, 1) + datums, tps := datumsForTest() + chk := chunkForTest(t, typeCtx.Location(), datums, tps, 1) colIdx := make([]int, len(tps)) for i := 0; i < len(tps); i++ { colIdx[i] = i } h := crc32.NewIEEE() - err1 := HashChunkRow(sc.TypeCtx(), h, chk.GetRow(0), tps, colIdx, buf) + err1 := HashChunkRow(typeCtx, h, chk.GetRow(0), tps, colIdx, buf) sum1 := h.Sum32() h.Reset() - err2 := HashChunkRow(sc.TypeCtx(), h, chk.GetRow(0), tps, colIdx, buf) + err2 := HashChunkRow(typeCtx, h, chk.GetRow(0), tps, colIdx, buf) sum2 := h.Sum32() require.NoError(t, err1) require.NoError(t, err2) require.Equal(t, sum2, sum1) - e, err := EqualChunkRow(sc.TypeCtx(), + e, err := EqualChunkRow(typeCtx, chk.GetRow(0), tps, colIdx, chk.GetRow(0), tps, colIdx) require.NoError(t, err) @@ -1236,10 +1229,10 @@ func TestValueSizeOfUnsignedInt(t *testing.T) { } func TestHashChunkColumns(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) buf := make([]byte, 1) - datums, tps := datumsForTest(sc) - chk := chunkForTest(t, sc, datums, tps, 4) + datums, tps := datumsForTest() + chk := chunkForTest(t, typeCtx.Location(), datums, tps, 4) colIdx := make([]int, len(tps)) for i := 0; i < len(tps); i++ { @@ -1257,10 +1250,10 @@ func TestHashChunkColumns(t *testing.T) { // Test hash value of the first 12 `Null` columns for i := 0; i < 12; i++ { require.True(t, chk.GetRow(0).IsNull(i)) - err1 := HashChunkSelected(sc.TypeCtx(), vecHash, chk, tps[i], i, buf, hasNull, sel, false) - err2 := HashChunkRow(sc.TypeCtx(), rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) - err3 := HashChunkRow(sc.TypeCtx(), rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) - err4 := HashChunkRow(sc.TypeCtx(), rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) + err1 := HashChunkSelected(typeCtx, vecHash, chk, tps[i], i, buf, hasNull, sel, false) + err2 := HashChunkRow(typeCtx, rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) + err3 := HashChunkRow(typeCtx, rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) + err4 := HashChunkRow(typeCtx, rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) require.NoError(t, err1) require.NoError(t, err2) require.NoError(t, err3) @@ -1282,10 +1275,10 @@ func TestHashChunkColumns(t *testing.T) { require.False(t, chk.GetRow(0).IsNull(i)) - err1 := HashChunkSelected(sc.TypeCtx(), vecHash, chk, tps[i], i, buf, hasNull, sel, false) - err2 := HashChunkRow(sc.TypeCtx(), rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) - err3 := HashChunkRow(sc.TypeCtx(), rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) - err4 := HashChunkRow(sc.TypeCtx(), rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) + err1 := HashChunkSelected(typeCtx, vecHash, chk, tps[i], i, buf, hasNull, sel, false) + err2 := HashChunkRow(typeCtx, rowHash[0], chk.GetRow(0), tps[i:i+1], colIdx[i:i+1], buf) + err3 := HashChunkRow(typeCtx, rowHash[1], chk.GetRow(1), tps[i:i+1], colIdx[i:i+1], buf) + err4 := HashChunkRow(typeCtx, rowHash[2], chk.GetRow(2), tps[i:i+1], colIdx[i:i+1], buf) require.NoError(t, err1) require.NoError(t, err2) diff --git a/pkg/util/codec/collation_test.go b/pkg/util/codec/collation_test.go index 1d8297e09ee18..0541ef1f30d80 100644 --- a/pkg/util/codec/collation_test.go +++ b/pkg/util/codec/collation_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" @@ -45,17 +44,16 @@ func prepareCollationData() (int, *chunk.Chunk, *chunk.Chunk) { } func TestHashGroupKeyCollation(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) tp := types.NewFieldType(mysql.TypeString) n, chk1, chk2 := prepareCollationData() tp.SetCollate("utf8_general_ci") buf1 := make([][]byte, n) buf2 := make([][]byte, n) - buf1, err := HashGroupKey(sc.TimeZone(), n, chk1.Column(0), buf1, tp) + buf1, err := HashGroupKey(time.Local, n, chk1.Column(0), buf1, tp) require.NoError(t, err) - buf2, err = HashGroupKey(sc.TimeZone(), n, chk2.Column(0), buf2, tp) + buf2, err = HashGroupKey(time.Local, n, chk2.Column(0), buf2, tp) require.NoError(t, err) for i := 0; i < n; i++ { @@ -68,9 +66,9 @@ func TestHashGroupKeyCollation(t *testing.T) { tp.SetCollate("utf8_unicode_ci") buf1 = make([][]byte, n) buf2 = make([][]byte, n) - buf1, err = HashGroupKey(sc.TimeZone(), n, chk1.Column(0), buf1, tp) + buf1, err = HashGroupKey(time.Local, n, chk1.Column(0), buf1, tp) require.NoError(t, err) - buf2, err = HashGroupKey(sc.TimeZone(), n, chk2.Column(0), buf2, tp) + buf2, err = HashGroupKey(time.Local, n, chk2.Column(0), buf2, tp) require.NoError(t, err) for i := 0; i < n; i++ { @@ -82,7 +80,7 @@ func TestHashGroupKeyCollation(t *testing.T) { } func TestHashChunkRowCollation(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) tp := types.NewFieldType(mysql.TypeString) tps := []*types.FieldType{tp} n, chk1, chk2 := prepareCollationData() @@ -93,8 +91,8 @@ func TestHashChunkRowCollation(t *testing.T) { for i := 0; i < n; i++ { h1 := crc32.NewIEEE() h2 := crc32.NewIEEE() - require.NoError(t, HashChunkRow(sc.TypeCtx(), h1, chk1.GetRow(i), tps, cols, buf)) - require.NoError(t, HashChunkRow(sc.TypeCtx(), h2, chk2.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h1, chk1.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h2, chk2.GetRow(i), tps, cols, buf)) require.NotEqual(t, h2.Sum32(), h1.Sum32()) h1.Reset() h2.Reset() @@ -104,8 +102,8 @@ func TestHashChunkRowCollation(t *testing.T) { for i := 0; i < n; i++ { h1 := crc32.NewIEEE() h2 := crc32.NewIEEE() - require.NoError(t, HashChunkRow(sc.TypeCtx(), h1, chk1.GetRow(i), tps, cols, buf)) - require.NoError(t, HashChunkRow(sc.TypeCtx(), h2, chk2.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h1, chk1.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h2, chk2.GetRow(i), tps, cols, buf)) require.Equal(t, h2.Sum32(), h1.Sum32()) h1.Reset() h2.Reset() @@ -115,8 +113,8 @@ func TestHashChunkRowCollation(t *testing.T) { for i := 0; i < n; i++ { h1 := crc32.NewIEEE() h2 := crc32.NewIEEE() - require.NoError(t, HashChunkRow(sc.TypeCtx(), h1, chk1.GetRow(i), tps, cols, buf)) - require.NoError(t, HashChunkRow(sc.TypeCtx(), h2, chk2.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h1, chk1.GetRow(i), tps, cols, buf)) + require.NoError(t, HashChunkRow(typeCtx, h2, chk2.GetRow(i), tps, cols, buf)) require.Equal(t, h2.Sum32(), h1.Sum32()) h1.Reset() h2.Reset() @@ -124,7 +122,7 @@ func TestHashChunkRowCollation(t *testing.T) { } func TestHashChunkColumnsCollation(t *testing.T) { - sc := stmtctx.NewStmtCtxWithTimeZone(time.Local) + typeCtx := types.DefaultStmtNoWarningContext.WithLocation(time.Local) tp := types.NewFieldType(mysql.TypeString) n, chk1, chk2 := prepareCollationData() buf := make([]byte, 1) @@ -133,8 +131,8 @@ func TestHashChunkColumnsCollation(t *testing.T) { h2s := []hash.Hash64{fnv.New64(), fnv.New64(), fnv.New64()} tp.SetCollate("binary") - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h1s, chk1, tp, 0, buf, hasNull)) - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h2s, chk2, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h1s, chk1, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h2s, chk2, tp, 0, buf, hasNull)) for i := 0; i < n; i++ { require.NotEqual(t, h2s[i].Sum64(), h1s[i].Sum64()) @@ -143,15 +141,15 @@ func TestHashChunkColumnsCollation(t *testing.T) { } tp.SetCollate("utf8_general_ci") - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h1s, chk1, tp, 0, buf, hasNull)) - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h2s, chk2, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h1s, chk1, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h2s, chk2, tp, 0, buf, hasNull)) for i := 0; i < n; i++ { require.Equal(t, h2s[i].Sum64(), h1s[i].Sum64()) } tp.SetCollate("utf8_unicode_ci") - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h1s, chk1, tp, 0, buf, hasNull)) - require.NoError(t, HashChunkColumns(sc.TypeCtx(), h2s, chk2, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h1s, chk1, tp, 0, buf, hasNull)) + require.NoError(t, HashChunkColumns(typeCtx, h2s, chk2, tp, 0, buf, hasNull)) for i := 0; i < n; i++ { require.Equal(t, h2s[i].Sum64(), h1s[i].Sum64()) } diff --git a/pkg/util/execdetails/execdetails.go b/pkg/util/execdetails/execdetails.go index 8258735b5df9b..b07a17a1219bd 100644 --- a/pkg/util/execdetails/execdetails.go +++ b/pkg/util/execdetails/execdetails.go @@ -752,20 +752,42 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu storeType: crs.storeType, BasicRuntimeStats: BasicRuntimeStats{ tiflashScanContext: TiFlashScanContext{ - totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(), - totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(), - totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(), - totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(), - totalDmfileRoughSetIndexCheckTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexCheckTimeMs(), - totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(), - totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(), - totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(), - totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum(), - totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(), - totalDisaggReadCacheHitSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheHitSize(), - totalDisaggReadCacheMissSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheMissSize()}}, threads: int32(summary.GetConcurrency()), + dmfileDataScannedRows: summary.GetTiflashScanContext().GetDmfileDataScannedRows(), + dmfileDataSkippedRows: summary.GetTiflashScanContext().GetDmfileDataSkippedRows(), + dmfileMvccScannedRows: summary.GetTiflashScanContext().GetDmfileMvccScannedRows(), + dmfileMvccSkippedRows: summary.GetTiflashScanContext().GetDmfileMvccSkippedRows(), + dmfileLmFilterScannedRows: summary.GetTiflashScanContext().GetDmfileLmFilterScannedRows(), + dmfileLmFilterSkippedRows: summary.GetTiflashScanContext().GetDmfileLmFilterSkippedRows(), + totalDmfileRsCheckMs: summary.GetTiflashScanContext().GetTotalDmfileRsCheckMs(), + totalDmfileReadMs: summary.GetTiflashScanContext().GetTotalDmfileReadMs(), + totalBuildSnapshotMs: summary.GetTiflashScanContext().GetTotalBuildSnapshotMs(), + localRegions: summary.GetTiflashScanContext().GetLocalRegions(), + remoteRegions: summary.GetTiflashScanContext().GetRemoteRegions(), + totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(), + disaggReadCacheHitBytes: summary.GetTiflashScanContext().GetDisaggReadCacheHitBytes(), + disaggReadCacheMissBytes: summary.GetTiflashScanContext().GetDisaggReadCacheMissBytes(), + segments: summary.GetTiflashScanContext().GetSegments(), + readTasks: summary.GetTiflashScanContext().GetReadTasks(), + deltaRows: summary.GetTiflashScanContext().GetDeltaRows(), + deltaBytes: summary.GetTiflashScanContext().GetDeltaBytes(), + mvccInputRows: summary.GetTiflashScanContext().GetMvccInputRows(), + mvccInputBytes: summary.GetTiflashScanContext().GetMvccInputBytes(), + mvccOutputRows: summary.GetTiflashScanContext().GetMvccOutputRows(), + lmSkipRows: summary.GetTiflashScanContext().GetLmSkipRows(), + totalBuildBitmapMs: summary.GetTiflashScanContext().GetTotalBuildBitmapMs(), + totalBuildInputStreamMs: summary.GetTiflashScanContext().GetTotalBuildInputstreamMs(), + staleReadRegions: summary.GetTiflashScanContext().GetStaleReadRegions(), + minLocalStreamMs: summary.GetTiflashScanContext().GetMinLocalStreamMs(), + maxLocalStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(), + minRemoteStreamMs: summary.GetTiflashScanContext().GetMinRemoteStreamMs(), + maxRemoteStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(), + regionsOfInstance: make(map[string]uint64), + }}, threads: int32(summary.GetConcurrency()), totalTasks: 1, } + for _, instance := range summary.GetTiflashScanContext().GetRegionsOfInstance() { + data.BasicRuntimeStats.tiflashScanContext.regionsOfInstance[instance.GetInstanceId()] = instance.GetRegionNum() + } data.BasicRuntimeStats.loop.Store(int32(*summary.NumIterations)) data.BasicRuntimeStats.consume.Store(int64(*summary.TimeProcessedNs)) data.BasicRuntimeStats.rows.Store(int64(*summary.NumProducedRows)) @@ -790,7 +812,9 @@ func (crs *CopRuntimeStats) GetTasks() (totalTasks int32) { // MergeBasicStats traverses basicCopRuntimeStats in the CopRuntimeStats and collects some useful information. func (crs *CopRuntimeStats) MergeBasicStats() (procTimes Percentile[Duration], totalTime time.Duration, totalTasks, totalLoops, totalThreads int32, totalTiFlashScanContext TiFlashScanContext) { - totalTiFlashScanContext = TiFlashScanContext{} + totalTiFlashScanContext = TiFlashScanContext{ + regionsOfInstance: make(map[string]uint64), + } for _, instanceStats := range crs.stats { procTimes.MergePercentile(&instanceStats.procTimes) totalTime += time.Duration(instanceStats.consume.Load()) @@ -896,60 +920,227 @@ type RuntimeStats interface { // TiFlashScanContext is used to express the table scan information in tiflash type TiFlashScanContext struct { - totalDmfileScannedPacks uint64 - totalDmfileScannedRows uint64 - totalDmfileSkippedPacks uint64 - totalDmfileSkippedRows uint64 - totalDmfileRoughSetIndexCheckTimeMs uint64 - totalDmfileReadTimeMs uint64 - totalCreateSnapshotTimeMs uint64 - totalLocalRegionNum uint64 - totalRemoteRegionNum uint64 - totalLearnerReadMs uint64 - totalDisaggReadCacheHitSize uint64 - totalDisaggReadCacheMissSize uint64 + dmfileDataScannedRows uint64 + dmfileDataSkippedRows uint64 + dmfileMvccScannedRows uint64 + dmfileMvccSkippedRows uint64 + dmfileLmFilterScannedRows uint64 + dmfileLmFilterSkippedRows uint64 + totalDmfileRsCheckMs uint64 + totalDmfileReadMs uint64 + totalBuildSnapshotMs uint64 + localRegions uint64 + remoteRegions uint64 + totalLearnerReadMs uint64 + disaggReadCacheHitBytes uint64 + disaggReadCacheMissBytes uint64 + segments uint64 + readTasks uint64 + deltaRows uint64 + deltaBytes uint64 + mvccInputRows uint64 + mvccInputBytes uint64 + mvccOutputRows uint64 + lmSkipRows uint64 + totalBuildBitmapMs uint64 + totalBuildInputStreamMs uint64 + staleReadRegions uint64 + minLocalStreamMs uint64 + maxLocalStreamMs uint64 + minRemoteStreamMs uint64 + maxRemoteStreamMs uint64 + regionsOfInstance map[string]uint64 } // Clone implements the deep copy of * TiFlashshScanContext func (context *TiFlashScanContext) Clone() TiFlashScanContext { - return TiFlashScanContext{ - totalDmfileScannedPacks: context.totalDmfileScannedPacks, - totalDmfileScannedRows: context.totalDmfileScannedRows, - totalDmfileSkippedPacks: context.totalDmfileSkippedPacks, - totalDmfileSkippedRows: context.totalDmfileSkippedRows, - totalDmfileRoughSetIndexCheckTimeMs: context.totalDmfileRoughSetIndexCheckTimeMs, - totalDmfileReadTimeMs: context.totalDmfileReadTimeMs, - totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs, - totalLocalRegionNum: context.totalLocalRegionNum, - totalRemoteRegionNum: context.totalRemoteRegionNum, - totalLearnerReadMs: context.totalLearnerReadMs, - totalDisaggReadCacheHitSize: context.totalDisaggReadCacheHitSize, - totalDisaggReadCacheMissSize: context.totalDisaggReadCacheMissSize, - } + newContext := TiFlashScanContext{ + dmfileDataScannedRows: context.dmfileDataScannedRows, + dmfileDataSkippedRows: context.dmfileDataSkippedRows, + dmfileMvccScannedRows: context.dmfileMvccScannedRows, + dmfileMvccSkippedRows: context.dmfileMvccSkippedRows, + dmfileLmFilterScannedRows: context.dmfileLmFilterScannedRows, + dmfileLmFilterSkippedRows: context.dmfileLmFilterSkippedRows, + totalDmfileRsCheckMs: context.totalDmfileRsCheckMs, + totalDmfileReadMs: context.totalDmfileReadMs, + totalBuildSnapshotMs: context.totalBuildSnapshotMs, + localRegions: context.localRegions, + remoteRegions: context.remoteRegions, + totalLearnerReadMs: context.totalLearnerReadMs, + disaggReadCacheHitBytes: context.disaggReadCacheHitBytes, + disaggReadCacheMissBytes: context.disaggReadCacheMissBytes, + segments: context.segments, + readTasks: context.readTasks, + deltaRows: context.deltaRows, + deltaBytes: context.deltaBytes, + mvccInputRows: context.mvccInputRows, + mvccInputBytes: context.mvccInputBytes, + mvccOutputRows: context.mvccOutputRows, + lmSkipRows: context.lmSkipRows, + totalBuildBitmapMs: context.totalBuildBitmapMs, + totalBuildInputStreamMs: context.totalBuildInputStreamMs, + staleReadRegions: context.staleReadRegions, + minLocalStreamMs: context.minLocalStreamMs, + maxLocalStreamMs: context.maxLocalStreamMs, + minRemoteStreamMs: context.minRemoteStreamMs, + maxRemoteStreamMs: context.maxRemoteStreamMs, + regionsOfInstance: make(map[string]uint64), + } + for k, v := range context.regionsOfInstance { + newContext.regionsOfInstance[k] = v + } + return newContext } + func (context *TiFlashScanContext) String() string { - return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_check_time: %dms, total_read_time: %dms, total_disagg_read_cache_hit_size: %d, total_disagg_read_cache_miss_size: %d}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d, total_learner_read_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexCheckTimeMs, context.totalDmfileReadTimeMs, context.totalDisaggReadCacheHitSize, context.totalDisaggReadCacheMissSize, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum, context.totalLearnerReadMs) + regionBalanceInfo := "none" + if len(context.regionsOfInstance) > 0 { + maxNum := uint64(0) + minNum := uint64(math.MaxUint64) + for _, v := range context.regionsOfInstance { + if v > maxNum { + maxNum = v + } + if v > 0 && v < minNum { + minNum = v + } + } + regionBalanceInfo = fmt.Sprintf("{instance_num: %d, max/min: %d/%d=%f}", + len(context.regionsOfInstance), + maxNum, + minNum, + float64(maxNum)/float64(minNum)) + } + dmfileDisaggInfo := "" + if context.disaggReadCacheHitBytes != 0 || context.disaggReadCacheMissBytes != 0 { + dmfileDisaggInfo = fmt.Sprintf(", disagg_cache_hit_bytes: %d, disagg_cache_miss_bytes: %d", + context.disaggReadCacheHitBytes, + context.disaggReadCacheMissBytes) + } + remoteStreamInfo := "" + if context.minRemoteStreamMs != 0 || context.maxRemoteStreamMs != 0 { + remoteStreamInfo = fmt.Sprintf("min_remote_stream:%dms, max_remote_stream:%dms, ", context.minRemoteStreamMs, context.maxRemoteStreamMs) + } + // note: "tot" is short for "total" + return fmt.Sprintf("tiflash_scan:{"+ + "mvcc_input_rows:%d, "+ + "mvcc_input_bytes:%d, "+ + "mvcc_output_rows:%d, "+ + "lm_skip_rows:%d, "+ + "local_regions:%d, "+ + "remote_regions:%d, "+ + "tot_learner_read:%dms, "+ + "region_balance:%s, "+ + "delta_rows:%d, "+ + "delta_bytes:%d, "+ + "segments:%d, "+ + "stale_read_regions:%d, "+ + "tot_build_snapshot:%dms, "+ + "tot_build_bitmap:%dms, "+ + "tot_build_inputstream:%dms, "+ + "min_local_stream:%dms, "+ + "max_local_stream:%dms, "+ + "%s"+ // remote stream info + "dtfile:{"+ + "data_scanned_rows:%d, "+ + "data_skipped_rows:%d, "+ + "mvcc_scanned_rows:%d, "+ + "mvcc_skipped_rows:%d, "+ + "lm_filter_scanned_rows:%d, "+ + "lm_filter_skipped_rows:%d, "+ + "tot_rs_index_check:%dms, "+ + "tot_read:%dms"+ + "%s}"+ // Disagg cache info of DMFile + "}", + context.mvccInputRows, + context.mvccInputBytes, + context.mvccOutputRows, + context.lmSkipRows, + context.localRegions, + context.remoteRegions, + context.totalLearnerReadMs, + regionBalanceInfo, + context.deltaRows, + context.deltaBytes, + context.segments, + context.staleReadRegions, + context.totalBuildSnapshotMs, + context.totalBuildBitmapMs, + context.totalBuildInputStreamMs, + context.minLocalStreamMs, + context.maxLocalStreamMs, + remoteStreamInfo, + context.dmfileDataScannedRows, + context.dmfileDataSkippedRows, + context.dmfileMvccScannedRows, + context.dmfileMvccSkippedRows, + context.dmfileLmFilterScannedRows, + context.dmfileLmFilterSkippedRows, + context.totalDmfileRsCheckMs, + context.totalDmfileReadMs, + dmfileDisaggInfo, + ) } // Merge make sum to merge the information in TiFlashScanContext func (context *TiFlashScanContext) Merge(other TiFlashScanContext) { - context.totalDmfileScannedPacks += other.totalDmfileScannedPacks - context.totalDmfileScannedRows += other.totalDmfileScannedRows - context.totalDmfileSkippedPacks += other.totalDmfileSkippedPacks - context.totalDmfileSkippedRows += other.totalDmfileSkippedRows - context.totalDmfileRoughSetIndexCheckTimeMs += other.totalDmfileRoughSetIndexCheckTimeMs - context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs - context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs - context.totalLocalRegionNum += other.totalLocalRegionNum - context.totalRemoteRegionNum += other.totalRemoteRegionNum + context.dmfileDataScannedRows += other.dmfileDataScannedRows + context.dmfileDataSkippedRows += other.dmfileDataSkippedRows + context.dmfileMvccScannedRows += other.dmfileMvccScannedRows + context.dmfileMvccSkippedRows += other.dmfileMvccSkippedRows + context.dmfileLmFilterScannedRows += other.dmfileLmFilterScannedRows + context.dmfileLmFilterSkippedRows += other.dmfileLmFilterSkippedRows + context.totalDmfileRsCheckMs += other.totalDmfileRsCheckMs + context.totalDmfileReadMs += other.totalDmfileReadMs + context.totalBuildSnapshotMs += other.totalBuildSnapshotMs + context.localRegions += other.localRegions + context.remoteRegions += other.remoteRegions context.totalLearnerReadMs += other.totalLearnerReadMs - context.totalDisaggReadCacheHitSize += other.totalDisaggReadCacheHitSize - context.totalDisaggReadCacheMissSize += other.totalDisaggReadCacheMissSize + context.disaggReadCacheHitBytes += other.disaggReadCacheHitBytes + context.disaggReadCacheMissBytes += other.disaggReadCacheMissBytes + context.segments += other.segments + context.readTasks += other.readTasks + context.deltaRows += other.deltaRows + context.deltaBytes += other.deltaBytes + context.mvccInputRows += other.mvccInputRows + context.mvccInputBytes += other.mvccInputBytes + context.mvccOutputRows += other.mvccOutputRows + context.lmSkipRows += other.lmSkipRows + context.totalBuildBitmapMs += other.totalBuildBitmapMs + context.totalBuildInputStreamMs += other.totalBuildInputStreamMs + context.staleReadRegions += other.staleReadRegions + + if context.minLocalStreamMs == 0 || other.minLocalStreamMs < context.minLocalStreamMs { + context.minLocalStreamMs = other.minLocalStreamMs + } + if other.maxLocalStreamMs > context.maxLocalStreamMs { + context.maxLocalStreamMs = other.maxLocalStreamMs + } + if context.minRemoteStreamMs == 0 || other.minRemoteStreamMs < context.minRemoteStreamMs { + context.minRemoteStreamMs = other.minRemoteStreamMs + } + if other.maxRemoteStreamMs > context.maxRemoteStreamMs { + context.maxRemoteStreamMs = other.maxRemoteStreamMs + } + + if context.regionsOfInstance == nil { + context.regionsOfInstance = make(map[string]uint64) + } + for k, v := range other.regionsOfInstance { + context.regionsOfInstance[k] += v + } } // Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty func (context *TiFlashScanContext) Empty() bool { - res := context.totalDmfileScannedPacks == 0 && context.totalDmfileSkippedPacks == 0 && context.totalLocalRegionNum == 0 && context.totalRemoteRegionNum == 0 + res := context.dmfileDataScannedRows == 0 && + context.dmfileDataSkippedRows == 0 && + context.dmfileMvccScannedRows == 0 && + context.dmfileMvccSkippedRows == 0 && + context.dmfileLmFilterScannedRows == 0 && + context.dmfileLmFilterSkippedRows == 0 && + context.localRegions == 0 && + context.remoteRegions == 0 return res } diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index faac7100815fc..018f0fa05cfcc 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -136,20 +136,18 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration NumIterations: &NumIterations, XXX_unrecognized: nil} } -func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexCheckTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs, totalLocalRegionNum, totalRemoteRegionNum, totalLearnerReadMs, totalDisaggReadCacheHitSize, totalDisaggReadCacheMissSize uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { +func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, dmfileScannedRows, dmfileSkippedRows, totalDmfileRsCheckMs, totalDmfileReadTimeMs, totalBuildSnapshotMs, localRegions, remoteRegions, totalLearnerReadMs, disaggReadCacheHitBytes, disaggReadCacheMissBytes uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { tiflashScanContext := tipb.TiFlashScanContext{ - TotalDmfileScannedPacks: &totalDmfileScannedPacks, - TotalDmfileSkippedPacks: &totalDmfileSkippedPacks, - TotalDmfileScannedRows: &totalDmfileScannedRows, - TotalDmfileSkippedRows: &totalDmfileSkippedRows, - TotalDmfileRoughSetIndexCheckTimeMs: &totalDmfileRoughSetIndexCheckTimeMs, - TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, - TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, - TotalLocalRegionNum: &totalLocalRegionNum, - TotalRemoteRegionNum: &totalRemoteRegionNum, - TotalLearnerReadMs: &totalLearnerReadMs, - TotalDisaggReadCacheHitSize: &totalDisaggReadCacheHitSize, - TotalDisaggReadCacheMissSize: &totalDisaggReadCacheMissSize, + DmfileDataScannedRows: &dmfileScannedRows, + DmfileDataSkippedRows: &dmfileSkippedRows, + TotalDmfileRsCheckMs: &totalDmfileRsCheckMs, + TotalDmfileReadMs: &totalDmfileReadTimeMs, + TotalBuildSnapshotMs: &totalBuildSnapshotMs, + LocalRegions: &localRegions, + RemoteRegions: &remoteRegions, + TotalLearnerReadMs: &totalLearnerReadMs, + DisaggReadCacheHitBytes: &disaggReadCacheHitBytes, + DisaggReadCacheMissBytes: &disaggReadCacheMissBytes, } fmt.Println("tiflashScanContext is ", tiflashScanContext.String()) return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows, @@ -215,10 +213,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 8192, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 12000, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 8192, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID))) scanDetail := &util.ScanDetail{ TotalKeys: 10, ProcessedKeys: 10, @@ -232,15 +230,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetOrCreateCopStats(tableScanID, "tiflash") - require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 202ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", cop.String()) + require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:10, remote_regions:4, tot_learner_read:1ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:40ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:8192, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:15ms, tot_read:202ms, disagg_cache_hit_bytes: 100, disagg_cache_miss_bytes: 50}}", cop.String()) copStats := cop.stats["8.8.8.8"] require.NotNil(t, copStats) copStats.SetRowNum(10) copStats.Record(time.Second, 10) - require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 200ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", copStats.String()) - expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_check_time: 100ms, total_read_time: 3000ms, total_disagg_read_cache_hit_size: 20, total_disagg_read_cache_miss_size: 0}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2, total_learner_read_time: 0ms}" + require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:10, remote_regions:4, tot_learner_read:1ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:40ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:8192, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:15ms, tot_read:200ms, disagg_cache_hit_bytes: 100, disagg_cache_miss_bytes: 50}}", copStats.String()) + expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:6, remote_regions:2, tot_learner_read:0ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:50ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:20192, data_skipped_rows:86000, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:100ms, tot_read:3000ms, disagg_cache_hit_bytes: 20, disagg_cache_miss_bytes: 0}}" require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String()) rootStats := stats.GetRootStats(tableReaderID) diff --git a/pkg/util/fastrand/BUILD.bazel b/pkg/util/fastrand/BUILD.bazel index 84df5c88e852a..ba17841cba1f3 100644 --- a/pkg/util/fastrand/BUILD.bazel +++ b/pkg/util/fastrand/BUILD.bazel @@ -2,7 +2,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "fastrand", - srcs = ["random.go"], + srcs = [ + "random.go", + "runtime.go", + "runtime_1.22.go", + ], importpath = "github.com/pingcap/tidb/pkg/util/fastrand", visibility = ["//visibility:public"], ) diff --git a/pkg/util/fastrand/random.go b/pkg/util/fastrand/random.go index 33458a3d87678..d823876ad6c1a 100644 --- a/pkg/util/fastrand/random.go +++ b/pkg/util/fastrand/random.go @@ -16,7 +16,6 @@ package fastrand import ( "math/bits" - _ "unsafe" // required by go:linkname ) // wyrand is a fast PRNG. See https://github.com/wangyi-fudan/wyhash @@ -48,11 +47,6 @@ func Buf(size int) []byte { return buf } -// Uint32 returns a lock free uint32 value. -// -//go:linkname Uint32 runtime.fastrand -func Uint32() uint32 - // Uint32N returns, as an uint32, a pseudo-random number in [0,n). func Uint32N(n uint32) uint32 { // This is similar to Uint32() % n, but faster. diff --git a/pkg/util/fastrand/runtime.go b/pkg/util/fastrand/runtime.go new file mode 100644 index 0000000000000..dda6549090de2 --- /dev/null +++ b/pkg/util/fastrand/runtime.go @@ -0,0 +1,26 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.22 + +package fastrand + +import ( + _ "unsafe" // required by go:linkname +) + +// Uint32 returns a lock free uint32 value. +// +//go:linkname Uint32 runtime.fastrand +func Uint32() uint32 diff --git a/pkg/util/fastrand/runtime_1.22.go b/pkg/util/fastrand/runtime_1.22.go new file mode 100644 index 0000000000000..0e8cf2a87a396 --- /dev/null +++ b/pkg/util/fastrand/runtime_1.22.go @@ -0,0 +1,26 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.22 + +package fastrand + +import ( + _ "unsafe" // required by go:linkname +) + +// Uint32 returns a lock free uint32 value. +// +//go:linkname Uint32 runtime.cheaprand +func Uint32() uint32 diff --git a/pkg/util/hint/hint_processor.go b/pkg/util/hint/hint_processor.go index 67f143aea18e6..728d0b06f0f3a 100644 --- a/pkg/util/hint/hint_processor.go +++ b/pkg/util/hint/hint_processor.go @@ -18,7 +18,6 @@ import ( "fmt" "strings" - "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/format" @@ -355,30 +354,30 @@ func nodeType4Stmt(node ast.StmtNode) NodeType { return TypeInvalid } -// CheckBindingFromHistoryBindable checks whether the ast and hint string from history is bindable. -// Not support: +// CheckBindingFromHistoryComplete checks whether the ast and hint string from history is complete. +// For these complex queries, the auto-generated binding might be not complete: // 1. query use tiFlash engine // 2. query with sub query // 3. query with more than 2 table join -func CheckBindingFromHistoryBindable(node ast.Node, hintStr string) error { +func CheckBindingFromHistoryComplete(node ast.Node, hintStr string) (complete bool, reason string) { // check tiflash contain := strings.Contains(hintStr, "tiflash") if contain { - return errors.New("can't create binding for query with tiflash engine") + return false, "auto-generated hint for queries accessing TiFlash might not be complete, the plan might change even after creating this binding" } checker := bindableChecker{ - bindable: true, + complete: true, tables: make(map[model.CIStr]struct{}, 2), } node.Accept(&checker) - return checker.reason + return checker.complete, checker.reason } // bindableChecker checks whether a binding from history can be created. type bindableChecker struct { - bindable bool - reason error + complete bool + reason string tables map[model.CIStr]struct{} } @@ -386,16 +385,16 @@ type bindableChecker struct { func (checker *bindableChecker) Enter(in ast.Node) (out ast.Node, skipChildren bool) { switch node := in.(type) { case *ast.ExistsSubqueryExpr, *ast.SubqueryExpr: - checker.bindable = false - checker.reason = errors.New("can't create binding for query with sub query") + checker.complete = false + checker.reason = "auto-generated hint for queries with sub queries might not be complete, the plan might change even after creating this binding" return in, true case *ast.TableName: if _, ok := checker.tables[node.Schema]; !ok { checker.tables[node.Name] = struct{}{} } if len(checker.tables) >= 3 { - checker.bindable = false - checker.reason = errors.New("can't create binding for query with more than two table join") + checker.complete = false + checker.reason = "auto-generated hint for queries with more than 3 table join might not be complete, the plan might change even after creating this binding" return in, true } } @@ -404,5 +403,5 @@ func (checker *bindableChecker) Enter(in ast.Node) (out ast.Node, skipChildren b // Leave implements Visitor interface. func (checker *bindableChecker) Leave(in ast.Node) (out ast.Node, ok bool) { - return in, checker.bindable + return in, checker.complete } diff --git a/pkg/util/logutil/BUILD.bazel b/pkg/util/logutil/BUILD.bazel index b6893b1563238..4ee4b14c6da44 100644 --- a/pkg/util/logutil/BUILD.bazel +++ b/pkg/util/logutil/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "logutil", srcs = [ + "general_logger.go", "hex.go", "log.go", "slow_query_logger.go", diff --git a/pkg/util/logutil/consistency/BUILD.bazel b/pkg/util/logutil/consistency/BUILD.bazel index a45516fa0762c..dc321bba7ee9d 100644 --- a/pkg/util/logutil/consistency/BUILD.bazel +++ b/pkg/util/logutil/consistency/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/util/dbterror", "//pkg/util/logutil", "//pkg/util/redact", + "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/kvrpcpb", "@com_github_tikv_client_go_v2//tikv", "@org_uber_go_zap//:zap", diff --git a/pkg/util/logutil/consistency/reporter.go b/pkg/util/logutil/consistency/reporter.go index a69cfc3485c92..4271a1dfc0b37 100644 --- a/pkg/util/logutil/consistency/reporter.go +++ b/pkg/util/logutil/consistency/reporter.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/pkg/errno" "github.com/pingcap/tidb/pkg/kv" @@ -186,7 +187,7 @@ func decodeMvccRecordValue(bs []byte, colMap map[int64]*types.FieldType, tb *mod // ReportLookupInconsistent reports inconsistent when index rows is more than record rows. func (r *Reporter) ReportLookupInconsistent(ctx context.Context, idxCnt, tblCnt int, missHd, fullHd []kv.Handle, missRowIdx []RecordData) error { - rmode := r.Sctx.GetSessionVars().EnableRedactNew + rmode := r.Sctx.GetSessionVars().EnableRedactLog const maxFullHandleCnt = 50 displayFullHdCnt := min(len(fullHd), maxFullHandleCnt) @@ -197,7 +198,7 @@ func (r *Reporter) ReportLookupInconsistent(ctx context.Context, idxCnt, tblCnt zap.String("missing_handles", redact.String(rmode, fmt.Sprint(missHd))), zap.String("total_handles", redact.String(rmode, fmt.Sprint(fullHd[:displayFullHdCnt]))), } - if rmode != "ON" { + if rmode != errors.RedactLogEnable { store, ok := r.Sctx.GetStore().(helper.Storage) if ok { for i, hd := range missHd { @@ -215,7 +216,7 @@ func (r *Reporter) ReportLookupInconsistent(ctx context.Context, idxCnt, tblCnt // ReportAdminCheckInconsistentWithColInfo reports inconsistent when the value of index row is different from record row. func (r *Reporter) ReportAdminCheckInconsistentWithColInfo(ctx context.Context, handle kv.Handle, colName string, idxDat, tblDat fmt.Stringer, err error, idxRow *RecordData) error { - rmode := r.Sctx.GetSessionVars().EnableRedactNew + rmode := r.Sctx.GetSessionVars().EnableRedactLog fs := []zap.Field{ zap.String("table_name", r.Tbl.Name.O), zap.String("index_name", r.Idx.Name.O), @@ -224,7 +225,7 @@ func (r *Reporter) ReportAdminCheckInconsistentWithColInfo(ctx context.Context, zap.Stringer("idxDatum", redact.Stringer(rmode, idxDat)), zap.Stringer("rowDatum", redact.Stringer(rmode, tblDat)), } - if rmode != "ON" { + if rmode != errors.RedactLogEnable { store, ok := r.Sctx.GetStore().(helper.Storage) if ok { fs = append(fs, zap.String("row_mvcc", redact.String(rmode, GetMvccByKey(store, r.HandleEncode(handle), DecodeRowMvccData(r.Tbl))))) @@ -252,7 +253,7 @@ func (r *RecordData) String() string { // ReportAdminCheckInconsistent reports inconsistent when single index row not found in record rows. func (r *Reporter) ReportAdminCheckInconsistent(ctx context.Context, handle kv.Handle, idxRow, tblRow *RecordData) error { - rmode := r.Sctx.GetSessionVars().EnableRedactNew + rmode := r.Sctx.GetSessionVars().EnableRedactLog fs := []zap.Field{ zap.String("table_name", r.Tbl.Name.O), zap.String("index_name", r.Idx.Name.O), @@ -260,7 +261,7 @@ func (r *Reporter) ReportAdminCheckInconsistent(ctx context.Context, handle kv.H zap.Stringer("index", redact.Stringer(rmode, idxRow)), zap.Stringer("row", redact.Stringer(rmode, tblRow)), } - if rmode != "ON" { + if rmode != errors.RedactLogEnable { store, ok := r.Sctx.GetStore().(helper.Storage) if ok { fs = append(fs, zap.String("row_mvcc", redact.String(rmode, GetMvccByKey(store, r.HandleEncode(handle), DecodeRowMvccData(r.Tbl))))) diff --git a/pkg/util/logutil/general_logger.go b/pkg/util/logutil/general_logger.go new file mode 100644 index 0000000000000..0e295c9c54d1c --- /dev/null +++ b/pkg/util/logutil/general_logger.go @@ -0,0 +1,44 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/log" + "go.uber.org/zap" +) + +func newGeneralLogger(cfg *LogConfig) (*zap.Logger, *log.ZapProperties, error) { + // create the general logger + sqLogger, prop, err := log.InitLogger(newGeneralLogConfig(cfg)) + if err != nil { + return nil, nil, errors.Trace(err) + } + return sqLogger, prop, nil +} + +func newGeneralLogConfig(cfg *LogConfig) *log.Config { + // copy the global log config to general log config + // if the filename of general log config is empty, general log will behave the same as global log. + sqConfig := cfg.Config + // level of the global log config doesn't affect the general logger which determines whether to + // log by execution duration. + sqConfig.Level = LogConfig{}.Level + if len(cfg.GeneralLogFile) != 0 { + sqConfig.File = cfg.File + sqConfig.File.Filename = cfg.GeneralLogFile + } + return &sqConfig +} diff --git a/pkg/util/logutil/log.go b/pkg/util/logutil/log.go index 997071483476d..1f77023f8a76a 100644 --- a/pkg/util/logutil/log.go +++ b/pkg/util/logutil/log.go @@ -81,10 +81,13 @@ type LogConfig struct { // SlowQueryFile filename, default to File log config on empty. SlowQueryFile string + + // GeneralLogFile filenanme, default to File log config on empty. + GeneralLogFile string } // NewLogConfig creates a LogConfig. -func NewLogConfig(level, format, slowQueryFile string, fileCfg FileLogConfig, disableTimestamp bool, opts ...func(*log.Config)) *LogConfig { +func NewLogConfig(level, format, slowQueryFile string, generalLogFile string, fileCfg FileLogConfig, disableTimestamp bool, opts ...func(*log.Config)) *LogConfig { c := &LogConfig{ Config: log.Config{ Level: level, @@ -92,7 +95,8 @@ func NewLogConfig(level, format, slowQueryFile string, fileCfg FileLogConfig, di DisableTimestamp: disableTimestamp, File: fileCfg.FileLogConfig, }, - SlowQueryFile: slowQueryFile, + SlowQueryFile: slowQueryFile, + GeneralLogFile: generalLogFile, } for _, opt := range opts { opt(&c.Config) @@ -113,6 +117,9 @@ const ( // SlowQueryLogger is used to log slow query, InitLogger will modify it according to config file. var SlowQueryLogger = log.L() +// GeneralLogger is used to log general log, InitLogger will modify it according to config file. +var GeneralLogger = log.L() + // InitLogger initializes a logger with cfg. func InitLogger(cfg *LogConfig, opts ...zap.Option) error { opts = append(opts, zap.AddStacktrace(zapcore.FatalLevel)) @@ -128,6 +135,12 @@ func InitLogger(cfg *LogConfig, opts ...zap.Option) error { return errors.Trace(err) } + // init dedicated logger for general log + GeneralLogger, _, err = newGeneralLogger(cfg) + if err != nil { + return errors.Trace(err) + } + initGRPCLogger(gl) tikv.SetLogContextKey(CtxLogKey) return nil @@ -174,6 +187,11 @@ func ReplaceLogger(cfg *LogConfig, opts ...zap.Option) error { return errors.Trace(err) } + GeneralLogger, _, err = newGeneralLogger(cfg) + if err != nil { + return errors.Trace(err) + } + log.S().Infof("replaced global logger with config: %s", string(cfgJSON)) return nil diff --git a/pkg/util/logutil/log_test.go b/pkg/util/logutil/log_test.go index d0c08d5554e0a..0092aa5763a6a 100644 --- a/pkg/util/logutil/log_test.go +++ b/pkg/util/logutil/log_test.go @@ -57,7 +57,7 @@ func TestZapLoggerWithKeys(t *testing.T) { } fileCfg := FileLogConfig{log.FileLogConfig{Filename: fmt.Sprintf("zap_log_%s", uuid.NewString()), MaxSize: 4096}} - conf := NewLogConfig("info", DefaultLogFormat, "", fileCfg, false) + conf := NewLogConfig("info", DefaultLogFormat, "", "", fileCfg, false) err := InitLogger(conf) require.NoError(t, err) connID := uint64(123) @@ -66,7 +66,7 @@ func TestZapLoggerWithKeys(t *testing.T) { err = os.Remove(fileCfg.Filename) require.NoError(t, err) - conf = NewLogConfig("info", DefaultLogFormat, "", fileCfg, false) + conf = NewLogConfig("info", DefaultLogFormat, "", "", fileCfg, false) err = InitLogger(conf) require.NoError(t, err) ctx = WithConnID(context.Background(), connID) @@ -124,7 +124,7 @@ func TestZapLoggerWithCore(t *testing.T) { } fileCfg := FileLogConfig{log.FileLogConfig{Filename: "zap_log", MaxSize: 4096}} - conf := NewLogConfig("info", DefaultLogFormat, "", fileCfg, false) + conf := NewLogConfig("info", DefaultLogFormat, "", "", fileCfg, false) opt := zap.WrapCore(func(core zapcore.Core) zapcore.Core { return core.With([]zap.Field{zap.String("coreKey", "coreValue")}) @@ -165,7 +165,7 @@ func testZapLogger(ctx context.Context, t *testing.T, fileName, pattern string) } func TestSetLevel(t *testing.T) { - conf := NewLogConfig("info", DefaultLogFormat, "", EmptyFileLogConfig, false) + conf := NewLogConfig("info", DefaultLogFormat, "", "", EmptyFileLogConfig, false) err := InitLogger(conf) require.NoError(t, err) require.Equal(t, zap.InfoLevel, log.GetLevel()) @@ -181,37 +181,80 @@ func TestSetLevel(t *testing.T) { require.Equal(t, zap.DebugLevel, log.GetLevel()) } -func TestSlowQueryLoggerCreation(t *testing.T) { - level := "Error" - conf := NewLogConfig(level, DefaultLogFormat, "", EmptyFileLogConfig, false) - _, prop, err := newSlowQueryLogger(conf) - // assert after init slow query logger, the original conf is not changed - require.Equal(t, conf.Level, level) - require.NoError(t, err) - // slow query logger doesn't use the level of the global log config, and the - // level should be less than WarnLevel which is used by it to log slow query. - require.NotEqual(t, conf.Level, prop.Level.String()) - require.True(t, prop.Level.Level() <= zapcore.WarnLevel) +func TestSlowQueryLoggerAndGeneralLoggerCreation(t *testing.T) { + var prop *log.ZapProperties + var err error + for i := 0; i < 2; i++ { + level := "Error" + conf := NewLogConfig(level, DefaultLogFormat, "", "", EmptyFileLogConfig, false) + if i == 0 { + _, prop, err = newSlowQueryLogger(conf) + } else { + _, prop, err = newGeneralLogger(conf) + } + // assert after init logger, the original conf is not changed + require.Equal(t, conf.Level, level) + require.NoError(t, err) + // logger doesn't use the level of the global log config, and the + // level should be equals to InfoLevel. + require.NotEqual(t, conf.Level, prop.Level.String()) + require.True(t, prop.Level.Level() == zapcore.InfoLevel) + + level = "warn" + name := "test.log" + fileConf := FileLogConfig{ + log.FileLogConfig{ + Filename: name, + MaxSize: 10, + MaxDays: 10, + MaxBackups: 10, + }, + } + conf = NewLogConfig(level, DefaultLogFormat, name, "", fileConf, false) + if i == 0 { + slowQueryConf := newSlowQueryLogConfig(conf) + // slowQueryConf.MaxDays/MaxSize/MaxBackups should be same with global config. + require.Equal(t, fileConf.FileLogConfig, slowQueryConf.File) + } else { + generalConf := newGeneralLogConfig(conf) + // generalConf.MaxDays/MaxSize/MaxBackups should be same with global config. + require.Equal(t, fileConf.FileLogConfig, generalConf.File) + } + } +} - level = "warn" - slowQueryFn := "slow-query.log" +func TestCompressedLog(t *testing.T) { + level := "warn" fileConf := FileLogConfig{ log.FileLogConfig{ - Filename: slowQueryFn, - MaxSize: 10, - MaxDays: 10, - MaxBackups: 10, + Filename: "test.log", + MaxSize: 10, + MaxDays: 10, + MaxBackups: 10, + Compression: "xxx", }, } - conf = NewLogConfig(level, DefaultLogFormat, slowQueryFn, fileConf, false) - slowQueryConf := newSlowQueryLogConfig(conf) - // slowQueryConf.MaxDays/MaxSize/MaxBackups should be same with global config. - require.Equal(t, fileConf.FileLogConfig, slowQueryConf.File) + conf := NewLogConfig(level, DefaultLogFormat, "test.log", "", fileConf, false) + err := InitLogger(conf) + require.Error(t, err) + + fileConf = FileLogConfig{ + log.FileLogConfig{ + Filename: "test.log", + MaxSize: 10, + MaxDays: 10, + MaxBackups: 10, + Compression: "gzip", + }, + } + conf = NewLogConfig(level, DefaultLogFormat, "test.log", "", fileConf, false) + err = InitLogger(conf) + require.NoError(t, err) } func TestGlobalLoggerReplace(t *testing.T) { fileCfg := FileLogConfig{log.FileLogConfig{Filename: "zap_log", MaxDays: 0, MaxSize: 4096}} - conf := NewLogConfig("info", DefaultLogFormat, "", fileCfg, false) + conf := NewLogConfig("info", DefaultLogFormat, "", "", fileCfg, false) err := InitLogger(conf) require.NoError(t, err) diff --git a/pkg/util/misc.go b/pkg/util/misc.go index 81d6bba617443..a149eaf0ffcf5 100644 --- a/pkg/util/misc.go +++ b/pkg/util/misc.go @@ -167,12 +167,10 @@ func SyntaxWarn(err error) error { } logutil.BgLogger().Debug("syntax error", zap.Error(err)) - // If the warn is already a terror with stack, pass it through. - if errors.HasStack(err) { - cause := errors.Cause(err) - if _, ok := cause.(*terror.Error); ok { - return err - } + // If the "err" is already a terror, pass it through. + cause := errors.Cause(err) + if _, ok := cause.(*terror.Error); ok { + return err } return parser.ErrParse.FastGenByArgs(syntaxErrorPrefix, err.Error()) diff --git a/pkg/util/mock/context.go b/pkg/util/mock/context.go index 19ff217a80aad..fb1eb185e8a10 100644 --- a/pkg/util/mock/context.go +++ b/pkg/util/mock/context.go @@ -544,6 +544,7 @@ func NewContext() *Context { vars.MinPagingSize = variable.DefMinPagingSize vars.CostModelVersion = variable.DefTiDBCostModelVer vars.EnableChunkRPC = true + vars.DivPrecisionIncrement = variable.DefDivPrecisionIncrement if err := sctx.GetSessionVars().SetSystemVar(variable.MaxAllowedPacket, "67108864"); err != nil { panic(err) } diff --git a/pkg/util/redact/BUILD.bazel b/pkg/util/redact/BUILD.bazel index 17e67c298cc2b..efc000e401a2d 100644 --- a/pkg/util/redact/BUILD.bazel +++ b/pkg/util/redact/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = ["redact.go"], importpath = "github.com/pingcap/tidb/pkg/util/redact", visibility = ["//visibility:public"], + deps = ["@com_github_pingcap_errors//:errors"], ) go_test( diff --git a/pkg/util/redact/redact.go b/pkg/util/redact/redact.go index 2bae8a8991f6c..9fdb39a560fee 100644 --- a/pkg/util/redact/redact.go +++ b/pkg/util/redact/redact.go @@ -15,8 +15,15 @@ package redact import ( + "bufio" + "bytes" "fmt" + "io" + "os" + "path/filepath" "strings" + + "github.com/pingcap/errors" ) var ( @@ -60,3 +67,108 @@ func (s redactStringer) String() string { func Stringer(mode string, input fmt.Stringer) redactStringer { return redactStringer{mode, input} } + +// DeRedactFile will deredact the input file, either removing marked contents, or remove the marker. It works line by line. +func DeRedactFile(remove bool, input string, output string) error { + ifile, err := os.Open(filepath.Clean(input)) + if err != nil { + return errors.WithStack(err) + } + defer ifile.Close() + + var ofile io.Writer + if output == "-" { + ofile = os.Stdout + } else { + //nolint: gosec + file, err := os.OpenFile(filepath.Clean(output), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return errors.WithStack(err) + } + defer file.Close() + ofile = file + } + + return DeRedact(remove, ifile, ofile, "\n") +} + +// DeRedact is similar to DeRedactFile, but act on reader/writer, it works line by line. +func DeRedact(remove bool, input io.Reader, output io.Writer, sep string) error { + sc := bufio.NewScanner(input) + out := bufio.NewWriter(output) + defer out.Flush() + buf := bytes.NewBuffer(nil) + s := bufio.NewReader(nil) + + for sc.Scan() { + s.Reset(strings.NewReader(sc.Text())) + start := false + for { + ch, _, err := s.ReadRune() + if err == io.EOF { + break + } + if err != nil { + return errors.WithStack(err) + } + if ch == '‹' { + if start { + // must be '<' + pch, _, err := s.ReadRune() + if err != nil { + return errors.WithStack(err) + } + if pch == ch { + _, _ = buf.WriteRune(ch) + } else { + _, _ = buf.WriteRune(ch) + _, _ = buf.WriteRune(pch) + } + } else { + start = true + buf.Reset() + } + } else if ch == '›' { + if start { + // peek the next + pch, _, err := s.ReadRune() + if err != nil && err != io.EOF { + return errors.WithStack(err) + } + if pch == ch { + _, _ = buf.WriteRune(ch) + } else { + start = false + if err != io.EOF { + // unpeek it + if err := s.UnreadRune(); err != nil { + return errors.WithStack(err) + } + } + if remove { + _ = out.WriteByte('?') + } else { + _, err = io.Copy(out, buf) + if err != nil { + return errors.WithStack(err) + } + } + } + } else { + _, _ = out.WriteRune(ch) + } + } else if start { + _, _ = buf.WriteRune(ch) + } else { + _, _ = out.WriteRune(ch) + } + } + if start { + _, _ = out.WriteRune('‹') + _, _ = out.WriteString(buf.String()) + } + _, _ = out.WriteString(sep) + } + + return nil +} diff --git a/pkg/util/redact/redact_test.go b/pkg/util/redact/redact_test.go index 4f1afeb7e3faf..3f39c29bf9b19 100644 --- a/pkg/util/redact/redact_test.go +++ b/pkg/util/redact/redact_test.go @@ -15,6 +15,8 @@ package redact import ( + "bytes" + "strings" "testing" "github.com/stretchr/testify/require" @@ -44,3 +46,32 @@ func TestRedact(t *testing.T) { require.Equal(t, c.output, Stringer(c.mode, &testStringer{c.input}).String()) } } + +func TestDeRedact(t *testing.T) { + for _, c := range []struct { + remove bool + input string + output string + }{ + {true, "‹fxcv›ggg", "?ggg"}, + {false, "‹fxcv›ggg", "fxcvggg"}, + {true, "fxcv", "fxcv"}, + {false, "fxcv", "fxcv"}, + {true, "‹fxcv›ggg‹fxcv›eee", "?ggg?eee"}, + {false, "‹fxcv›ggg‹fxcv›eee", "fxcvgggfxcveee"}, + {true, "‹›", "?"}, + {false, "‹›", ""}, + {true, "gg‹ee", "gg‹ee"}, + {false, "gg‹ee", "gg‹ee"}, + {true, "gg›ee", "gg›ee"}, + {false, "gg›ee", "gg›ee"}, + {true, "gg‹ee‹ee", "gg‹ee‹ee"}, + {false, "gg‹ee‹gg", "gg‹ee‹gg"}, + {true, "gg›ee›gg", "gg›ee›gg"}, + {false, "gg›ee›ee", "gg›ee›ee"}, + } { + w := bytes.NewBuffer(nil) + require.NoError(t, DeRedact(c.remove, strings.NewReader(c.input), w, "")) + require.Equal(t, c.output, w.String()) + } +} diff --git a/tests/integrationtest/r/ddl/bdr_mode.result b/tests/integrationtest/r/ddl/bdr_mode.result index bdfd9a9c361ba..ed65449e03c18 100644 --- a/tests/integrationtest/r/ddl/bdr_mode.result +++ b/tests/integrationtest/r/ddl/bdr_mode.result @@ -479,5 +479,9 @@ alter table t add column c int null; alter table t add column d int not null; Error 8263 (HY000): The operation is not allowed while the bdr role of this cluster is set to primary. alter table t add column d int not null default 10; +alter table t add column e int default 10; +alter table t add column f int comment "test"; +alter table t add column g int default 10 comment "test"; +alter table t add column h int as (a + 1) virtual; admin unset bdr role; drop database bdr_mode; diff --git a/tests/integrationtest/r/ddl/default_as_expression.result b/tests/integrationtest/r/ddl/default_as_expression.result index 848011b4dfb42..460f7b5eadbfc 100644 --- a/tests/integrationtest/r/ddl/default_as_expression.result +++ b/tests/integrationtest/r/ddl/default_as_expression.result @@ -15,29 +15,36 @@ Error 1674 (HY000): Statement is unsafe because it uses a system function that m SET @x := NOW(); insert into t0(c) values (1); insert into t0 values (2, default); -SELECT * FROM t0 WHERE c = date_format(@x,'%Y-%m') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m'); -c c1 +SELECT count(1) FROM t0 WHERE c1 = date_format(@x,'%Y-%m'); +count(1) +2 insert into t1(c) values (1); insert into t1 values (2, default); -SELECT * FROM t1 WHERE c = date_format(@x,'%Y-%m-%d'); -c c1 +SELECT count(1) FROM t1 WHERE c1 = date_format(@x,'%Y-%m-%d'); +count(1) +2 +SET @x := NOW(); insert into t2(c) values (1); insert into t2 values (2, default); -SELECT * FROM t2 WHERE c = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); -c c1 +SELECT count(1) FROM t2 WHERE c1 = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c1 = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); +count(1) +2 SET @x := NOW(); insert into t3(c) values (1); insert into t3 values (2, default); -SELECT * FROM t3 WHERE c = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); -c c1 +SELECT count(1) FROM t3 WHERE c1 = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c1 = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); +count(1) +2 insert into t4(c) values (1); insert into t4 values (2, default); -SELECT * FROM t4 WHERE c = date_format(@x,'%Y-%m-%d %H:%i:%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H:%i:%s'); -c c1 +SELECT count(1) FROM t4 WHERE c1 = date_format(@x,'%Y-%m-%d'); +count(1) +2 insert into t5(c) values (1); insert into t5 values (2, default); -SELECT * FROM t5 WHERE c = date_format(@x,'%Y-%m-%d %H:%i:%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H:%i:%s'); -c c1 +SELECT count(1) FROM t5 WHERE c1 = date_format(@x,'%Y-%m-%d'); +count(1) +2 show create table t0; Table Create Table t0 CREATE TABLE `t0` ( @@ -132,11 +139,11 @@ create table t (c int(10), c1 varchar(256) default (REPLACE(UPPER(UUID()), '-', create table t1 (c int(10), c1 int default (REPLACE(UPPER(UUID()), '-', '')), index idx(c1)); create table t2 (c int(10), c1 varchar(256) default (REPLACE(CONVERT(UPPER(UUID()) USING UTF8MB4), '-', '')), index idx(c1)); create table t1 (c int(10), c1 varchar(256) default (REPLACE('xdfj-jfj', '-', ''))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `REPLACE`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `REPLACE with disallowed args`. create table t1 (c int(10), c1 varchar(256) default (UPPER(UUID()))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `UPPER`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `UPPER with disallowed args`. create table t1 (c int(10), c1 varchar(256) default (REPLACE(UPPER('dfdkj-kjkl-d'), '-', ''))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `REPLACE`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `REPLACE with disallowed args`. alter table t add column c2 varchar(32) default (REPLACE(UPPER(UUID()), '-', '')); Error 1674 (HY000): Statement is unsafe because it uses a system function that may return a different value on the slave alter table t add column c3 int default (UPPER(UUID())); @@ -219,9 +226,9 @@ create table t2 (c int(10), c1 blob default (str_to_date('1980-01-01','%Y-%m-%d' create table t5 (c int(10), c1 json default (str_to_date('9999-01-01','%Y-%m-%d')), c2 timestamp default (str_to_date('1980-01-01','%Y-%m-%d'))); set session sql_mode=@sqlMode; create table t6 (c int(10), c1 varchar(32) default (str_to_date(upper('1980-01-01'),'%Y-%m-%d'))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `str_to_date with these args`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `str_to_date with disallowed args`. create table t6 (c int(10), c1 varchar(32) default (str_to_date('1980-01-01',upper('%Y-%m-%d')))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `str_to_date with these args`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `str_to_date with disallowed args`. alter table t0 add column c3 varchar(32) default (str_to_date('1980-01-01','%Y-%m-%d')); Error 1674 (HY000): Statement is unsafe because it uses a system function that may return a different value on the slave alter table t0 add column c4 int default (str_to_date('1980-01-01','%Y-%m-%d')); @@ -356,6 +363,17 @@ c c1 c2 2 1980-01-01 NULL 3 1980-01-01 NULL 4 1980-01-01 NULL +alter table t0 drop column c1; +Error 8200 (HY000): can't drop column c1 with composite index covered or Primary Key covered now +alter table t0 drop column c2; +show create table t0; +Table Create Table +t0 CREATE TABLE `t0` ( + `c` int(10) DEFAULT NULL, + `c1` varchar(32), + KEY `idx` (`c`,`c1`), + KEY `idx1` (`c1`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin SELECT column_default, extra FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema='test' AND TABLE_NAME='t1' AND COLUMN_NAME='c1'; column_default extra str_to_date(_utf8mb4'1980-01-01', _utf8mb4'%Y-%m-%d') DEFAULT_GENERATED @@ -365,7 +383,7 @@ create table t1 (c int(10), c1 int default (upper(substring_index(user(),_utf8mb create table t2 (c int(10), c1 varchar(256) default (substring_index(user(),'@',1))); Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `substring_index`. create table t2 (c int(10), c1 varchar(256) default (upper(substring_index('fjks@jkkl','@',1)))); -Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `upper`. +Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `upper with disallowed args`. create table t2 (c int(10), c1 varchar(256) default (upper(substring_index(user(),'x',1)))); Error 3770 (HY000): Default value expression of column 'c1' contains a disallowed function: `KindString x`. alter table t add column c2 varchar(32) default (upper(substring_index(user(),'@',1))); @@ -747,27 +765,177 @@ Error 8200 (HY000): can't change column constraint (PRIMARY KEY) ALTER TABLE t0 ALTER COLUMN c SET DEFAULT(str_to_date('1980-01-01','%Y-%m-%d')); insert into t0(id) values (2); drop table t0; -CREATE TABLE t1 (i INT, b int DEFAULT (str_to_date('1980-01-01','%Y-%m-%d')), c INT GENERATED ALWAYS AS (b+2)); +CREATE TABLE t1 (i INT, b int DEFAULT (str_to_date('1980-01-01','%Y-%m-%d')), c INT GENERATED ALWAYS AS (b+2), d INT GENERATED ALWAYS AS (b+10) STORED); +INSERT INTO t1(i) VALUES (1); +CREATE INDEX idx1 ON t1 ((b+1)); +CREATE INDEX idx2 ON t1 ((c+1)); +CREATE INDEX idx3 ON t1 ((d+1)); SHOW COLUMNS FROM t1; Field Type Null Key Default Extra i int(11) YES NULL b int(11) YES str_to_date(_utf8mb4'1980-01-01', _utf8mb4'%Y-%m-%d') DEFAULT_GENERATED c int(11) YES NULL VIRTUAL GENERATED +d int(11) YES NULL STORED GENERATED show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `i` int(11) DEFAULT NULL, `b` int(11) DEFAULT str_to_date(_utf8mb4'1980-01-01', _utf8mb4'%Y-%m-%d'), - `c` int(11) GENERATED ALWAYS AS (`b` + 2) VIRTUAL + `c` int(11) GENERATED ALWAYS AS (`b` + 2) VIRTUAL, + `d` int(11) GENERATED ALWAYS AS (`b` + 10) STORED, + KEY `idx1` ((`b` + 1)), + KEY `idx2` ((`c` + 1)), + KEY `idx3` ((`d` + 1)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin -INSERT INTO t1(i) VALUES (1); INSERT INTO t1(i, b) VALUES (2, DEFAULT); INSERT INTO t1(i, b) VALUES (3, 123); INSERT INTO t1(i, b) VALUES (NULL, NULL); SELECT * FROM t1; -i b c -1 19800101 19800103 -2 19800101 19800103 -3 123 125 -NULL NULL NULL +i b c d +1 19800101 19800103 19800111 +2 19800101 19800103 19800111 +3 123 125 133 +NULL NULL NULL NULL drop table t1; +create table t0 (c int(10), c1 int default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c, c1)); +REPLACE INTO t0 VALUES (1, DEFAULT); +SELECT * FROM t0; +c c1 +1 19800101 +show columns from test.t0 where field='c1'; +Field Type Null Key Default Extra +c1 int(11) NO PRI str_to_date(_utf8mb4'1980-01-01', _utf8mb4'%Y-%m-%d') DEFAULT_GENERATED +create table t1 (c int(10), c1 BLOB default (date_format(now(),'%Y-%m-%d')), c2 JSON default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c1(32), c2)); +Error 3152 (42000): JSON column 'c2' cannot be used in key specification. +create table t1 (c int(10), c1 BLOB default (date_format(now(),'%Y-%m-%d')), c2 JSON default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c1(32))); +SET @x := NOW(); +REPLACE INTO t1 VALUES (1, DEFAULT, '[1,1,2]'); +CREATE INDEX idx ON t1 ((cast(c2 as signed array))); +REPLACE INTO t1 VALUES (1, DEFAULT, '[3, 4]'); +SELECT count(1) FROM t1 WHERE c1 = date_format(@x,'%Y-%m-%d'); +count(1) +1 +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(10) DEFAULT NULL, + `c1` blob NOT NULL DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d'), + `c2` json DEFAULT str_to_date(_utf8mb4'1980-01-01', _utf8mb4'%Y-%m-%d'), + PRIMARY KEY (`c1`(32)) /*T![clustered_index] NONCLUSTERED */, + KEY `idx` ((cast(`c2` as signed array))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +drop table t0, t1; +CREATE TABLE t0( +id INT NOT NULL, +c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), +d datetime default (date_format(now(),'%Y-%m-%d %H:%i:%s')), +unique key idx(id, c), +key idx1(id, c, d) +) +PARTITION BY RANGE (YEAR(c)) ( +PARTITION p0 VALUES LESS THAN (1991), +PARTITION p1 VALUES LESS THAN (1996), +PARTITION p2 VALUES LESS THAN (2001), +PARTITION p3 VALUES LESS THAN MAXVALUE +); +INSERT INTO t0 VALUES(1, default, '1998-05-04 10:10:10'), (2, '1990-05-04 10:10:10', default),(3, default, '1991-05-04 10:10:10'), (4, '2000-05-04 10:10:10', '1991-05-04 10:10:10'),(5, default, '2002-05-04 10:10:10'); +select id from t0 order by c, d; +id +2 +4 +3 +1 +5 +show create table t0; +Table Create Table +t0 CREATE TABLE `t0` ( + `id` int(11) NOT NULL, + `c` date DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d %H:%i:%s'), + `d` datetime DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d %H:%i:%s'), + UNIQUE KEY `idx` (`id`,`c`), + KEY `idx1` (`id`,`c`,`d`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +PARTITION BY RANGE (YEAR(`c`)) +(PARTITION `p0` VALUES LESS THAN (1991), + PARTITION `p1` VALUES LESS THAN (1996), + PARTITION `p2` VALUES LESS THAN (2001), + PARTITION `p3` VALUES LESS THAN (MAXVALUE)) +drop table t0; +CREATE TEMPORARY TABLE t0( +id BIGINT, +c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), +PRIMARY KEY(id, c) +); +show create table t0; +Table Create Table +t0 CREATE TEMPORARY TABLE `t0` ( + `id` bigint(20) NOT NULL, + `c` date NOT NULL DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d %H:%i:%s'), + PRIMARY KEY (`id`,`c`) /*T![clustered_index] NONCLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +SET @x := NOW(); +INSERT INTO t0 VALUES(1, default); +SELECT count(1) FROM t0 WHERE c = date_format(@x,'%Y-%m-%d'); +count(1) +1 +show create table t0; +Table Create Table +t0 CREATE TEMPORARY TABLE `t0` ( + `id` bigint(20) NOT NULL, + `c` date NOT NULL DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d %H:%i:%s'), + PRIMARY KEY (`id`,`c`) /*T![clustered_index] NONCLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +drop table t0; +CREATE TABLE t0( +id BIGINT, +c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), +PRIMARY KEY(id, c) +); +SET @x := NOW(); +INSERT INTO t0 VALUES(1, default); +ALTER TABLE t0 CACHE; +INSERT INTO t0 VALUES(2, default); +SELECT count(1) FROM t0 WHERE c = date_format(@x,'%Y-%m-%d'); +count(1) +2 +show create table t0; +Table Create Table +t0 CREATE TABLE `t0` ( + `id` bigint(20) NOT NULL, + `c` date NOT NULL DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d %H:%i:%s'), + PRIMARY KEY (`id`,`c`) /*T![clustered_index] NONCLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /* CACHED ON */ +ALTER TABLE t0 NOCACHE; +drop table t0; +CREATE TABLE parent ( +id INT, +c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), +primary key(c) +); +CREATE TABLE child ( +id INT, +cc date default (date_format(now(),'%Y-%m-%d')), +INDEX idx (cc), +FOREIGN KEY (cc) REFERENCES parent(c) ON DELETE CASCADE +); +SET @x := NOW(); +INSERT INTO parent VALUES(1, default); +INSERT INTO child VALUES(1, default); +alter table child add foreign key fk_2(cc) references parent(c); +INSERT INTO parent VALUES(2, default); +alter table child drop foreign key fk_2; +SELECT count(1) FROM parent WHERE c = date_format(@x,'%Y-%m-%d'); +count(1) +1 +SELECT count(1) FROM child WHERE cc = date_format(@x,'%Y-%m-%d'); +count(1) +1 +show create table child; +Table Create Table +child CREATE TABLE `child` ( + `id` int(11) DEFAULT NULL, + `cc` date DEFAULT date_format(now(), _utf8mb4'%Y-%m-%d'), + KEY `idx` (`cc`), + CONSTRAINT `fk_1` FOREIGN KEY (`cc`) REFERENCES `test`.`parent` (`c`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +drop table parent, child; diff --git a/tests/integrationtest/r/ddl/integration.result b/tests/integrationtest/r/ddl/integration.result index 31d3984d97e3b..ab768bbf028b0 100644 --- a/tests/integrationtest/r/ddl/integration.result +++ b/tests/integrationtest/r/ddl/integration.result @@ -99,3 +99,38 @@ create table t (d int default '18446744073709551616' ); Level Code Message Warning 1690 BIGINT value is out of range in '18446744073709551616' set sql_mode=DEFAULT; +drop table if exists t; +create table t(a int not null, b int, primary key(a), unique idx_b(b)); +drop table if exists t2; +create table t2(a int not null, b int, primary key(a) nonclustered, unique idx_b(b)); +drop table if exists t3; +set tidb_enable_global_index=1; +create table t3(a int not null, b int, primary key(a) nonclustered, unique idx_b(b)) partition by hash(a) partitions 3; +drop table if exists t4; +create table t4(a int not null, b int, primary key(a)) partition by hash(a) partitions 3; +alter table t partition by hash(a) partitions 3; +Error 8214 (HY000): global index is not supported yet for alter table partitioning +alter table t partition by key() partitions 3; +Error 8214 (HY000): global index is not supported yet for alter table partitioning +alter table t partition by hash(b) partitions 3; +Error 1503 (HY000): A PRIMARY must include all columns in the table's partitioning function +alter table t2 partition by hash(b) partitions 3; +Error 8214 (HY000): global index is not supported yet for alter table partitioning +alter table t3 partition by key(a) partitions 3; +Error 8214 (HY000): global index is not supported yet for alter table partitioning +alter table t4 partition by hash(b) partitions 3; +Error 1503 (HY000): A PRIMARY must include all columns in the table's partitioning function +set tidb_enable_global_index=0; +alter table t partition by hash(a) partitions 3; +Error 1503 (HY000): A UNIQUE INDEX must include all columns in the table's partitioning function +alter table t partition by key() partitions 3; +Error 1503 (HY000): A UNIQUE INDEX must include all columns in the table's partitioning function +alter table t partition by hash(b) partitions 3; +Error 1503 (HY000): A PRIMARY must include all columns in the table's partitioning function +alter table t2 partition by hash(b) partitions 3; +Error 1503 (HY000): A PRIMARY must include all columns in the table's partitioning function +alter table t3 partition by key(a) partitions 3; +Error 1503 (HY000): A UNIQUE INDEX must include all columns in the table's partitioning function +alter table t4 partition by hash(b) partitions 3; +Error 1503 (HY000): A PRIMARY must include all columns in the table's partitioning function +drop table t, t2, t3, t4; diff --git a/tests/integrationtest/r/ddl/partition.result b/tests/integrationtest/r/ddl/partition.result index f3aa462cc7e9a..44b97995d520a 100644 --- a/tests/integrationtest/r/ddl/partition.result +++ b/tests/integrationtest/r/ddl/partition.result @@ -457,3 +457,12 @@ drop table t; set character_set_connection=DEFAULT; create table a (col1 int, col2 int, unique key (col1, col2)) partition by range columns (col1, col2) (partition p0 values less than (NULL, 1 )); Error 1566 (HY000): Not allowed to use NULL value in VALUES LESS THAN +drop table if exists parent, child, child_with_partition; +create table parent (id int unique); +create table child (id int, parent_id int, foreign key (parent_id) references parent(id)); +create table child_with_partition(id int, parent_id int) partition by range(id) (partition p1 values less than (100)); +alter table child_with_partition exchange partition p1 with table child; +Error 1740 (HY000): Table to exchange with partition has foreign key references: 'child' +alter table child drop foreign key fk_1; +alter table child drop key fk_1; +alter table child_with_partition exchange partition p1 with table child; diff --git a/tests/integrationtest/r/executor/partition/issues.result b/tests/integrationtest/r/executor/partition/issues.result index c6750ed300647..d564ce2855965 100644 --- a/tests/integrationtest/r/executor/partition/issues.result +++ b/tests/integrationtest/r/executor/partition/issues.result @@ -307,7 +307,7 @@ id estRows task access object operator info IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:executor__partition__issues.t.txn_account_id, executor__partition__issues.t.serial_id, inner key:executor__partition__issues.c.txt_account_id, executor__partition__issues.c.serial_id, equal cond:eq(executor__partition__issues.t.serial_id, executor__partition__issues.c.serial_id), eq(executor__partition__issues.t.txn_account_id, executor__partition__issues.c.txt_account_id) ├─TableReader_25(Build) 0.80 root data:Selection_24 │ └─Selection_24 0.80 cop[tikv] eq(executor__partition__issues.t.broker, "0009"), not(isnull(executor__partition__issues.t.serial_id)) -│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false +│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false, stats:partial[serial_id:missing] └─TableReader_19(Probe) 0.80 root partition:all data:Selection_18 └─Selection_18 0.80 cop[tikv] eq(executor__partition__issues.c.occur_trade_date, 2022-11-17 00:00:00.000000) └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(executor__partition__issues.c.txt_account_id, executor__partition__issues.t.txn_account_id) eq(executor__partition__issues.c.serial_id, executor__partition__issues.t.serial_id) eq(executor__partition__issues.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false diff --git a/tests/integrationtest/r/explain_complex_stats.result b/tests/integrationtest/r/explain_complex_stats.result index bb300a673b109..9183896d7aefb 100644 --- a/tests/integrationtest/r/explain_complex_stats.result +++ b/tests/integrationtest/r/explain_complex_stats.result @@ -115,16 +115,34 @@ bm tinyint(1) DEFAULT '0' comment '[[set=0,1]]', PRIMARY KEY (aid,dic) ); load stats 's/explain_complex_stats_rr.json'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dt' and column_name = 'cm'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'gad' and column_name = 't'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dd' and column_name = 'ip'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dd' and column_name = 't'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'sdk' and column_name = 't'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'st' and column_name = 't'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'uid'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'ppt'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'ps'; +Db_name Table_name Partition_name Column_name Is_index Update_time Distinct_count Null_count Avg_col_size Correlation Load_status Total_mem_usage Hist_mem_usage Topn_mem_usage Cms_mem_usage explain format = 'brief' SELECT ds, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(dic) as install_device FROM dt use index (cm) WHERE (ds >= '2016-09-01') AND (ds <= '2016-11-03') AND (cm IN ('1062', '1086', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1709', '1719', '1720', '1843', '2813', '2814', '2815', '2816', '2817', '2818', '2819', '2820', '2821', '2822', '2823', '2824', '2825', '2826', '2827', '2828', '2829', '2830', '2831', '2832', '2833', '2834', '2835', '2836', '2837', '2838', '2839', '2840', '2841', '2842', '2843', '2844', '2845', '2846', '2847', '2848', '2849', '2850', '2851', '2852', '2853', '2854', '2855', '2856', '2857', '2858', '2859', '2860', '2861', '2862', '2863', '2864', '2865', '2866', '2867', '2868', '2869', '2870', '2871', '2872', '3139', '3140', '3141', '3142', '3143', '3144', '3145', '3146', '3147', '3148', '3149', '3150', '3151', '3152', '3153', '3154', '3155', '3156', '3157', '3158', '3386', '3387', '3388', '3389', '3390', '3391', '3392', '3393', '3394', '3395', '3664', '3665', '3666', '3667', '3668', '3670', '3671', '3672', '3673', '3674', '3676', '3677', '3678', '3679', '3680', '3681', '3682', '3683', '3684', '3685', '3686', '3687', '3688', '3689', '3690', '3691', '3692', '3693', '3694', '3695', '3696', '3697', '3698', '3699', '3700', '3701', '3702', '3703', '3704', '3705', '3706', '3707', '3708', '3709', '3710', '3711', '3712', '3713', '3714', '3715', '3960', '3961', '3962', '3963', '3964', '3965', '3966', '3967', '3968', '3978', '3979', '3980', '3981', '3982', '3983', '3984', '3985', '3986', '3987', '4208', '4209', '4210', '4211', '4212', '4304', '4305', '4306', '4307', '4308', '4866', '4867', '4868', '4869', '4870', '4871', '4872', '4873', '4874', '4875')) GROUP BY ds, p1, p2, p3, p4, p5, p6_md5, p7_md5 ORDER BY ds2 DESC; id estRows task access object operator info Projection 21.47 root explain_complex_stats.dt.ds, explain_complex_stats.dt.p1, explain_complex_stats.dt.p2, explain_complex_stats.dt.p3, explain_complex_stats.dt.p4, explain_complex_stats.dt.p5, explain_complex_stats.dt.p6_md5, explain_complex_stats.dt.p7_md5, Column#21->Column#30 └─Sort 21.47 root explain_complex_stats.dt.ds2:desc └─HashAgg 21.47 root group by:explain_complex_stats.dt.ds, explain_complex_stats.dt.p1, explain_complex_stats.dt.p2, explain_complex_stats.dt.p3, explain_complex_stats.dt.p4, explain_complex_stats.dt.p5, explain_complex_stats.dt.p6_md5, explain_complex_stats.dt.p7_md5, funcs:count(Column#32)->Column#21, funcs:firstrow(explain_complex_stats.dt.ds)->explain_complex_stats.dt.ds, funcs:firstrow(Column#34)->explain_complex_stats.dt.ds2, funcs:firstrow(explain_complex_stats.dt.p1)->explain_complex_stats.dt.p1, funcs:firstrow(explain_complex_stats.dt.p2)->explain_complex_stats.dt.p2, funcs:firstrow(explain_complex_stats.dt.p3)->explain_complex_stats.dt.p3, funcs:firstrow(explain_complex_stats.dt.p4)->explain_complex_stats.dt.p4, funcs:firstrow(explain_complex_stats.dt.p5)->explain_complex_stats.dt.p5, funcs:firstrow(explain_complex_stats.dt.p6_md5)->explain_complex_stats.dt.p6_md5, funcs:firstrow(explain_complex_stats.dt.p7_md5)->explain_complex_stats.dt.p7_md5 └─IndexLookUp 21.47 root - ├─IndexRangeScan(Build) 128.00 cop[tikv] table:dt, index:cm(cm) range:[1062,1062], [1086,1086], [1423,1423], [1424,1424], [1425,1425], [1426,1426], [1427,1427], [1428,1428], [1429,1429], [1430,1430], [1431,1431], [1432,1432], [1433,1433], [1434,1434], [1435,1435], [1436,1436], [1437,1437], [1438,1438], [1439,1439], [1440,1440], [1441,1441], [1442,1442], [1443,1443], [1444,1444], [1445,1445], [1446,1446], [1447,1447], [1448,1448], [1449,1449], [1450,1450], [1451,1451], [1452,1452], [1488,1488], [1489,1489], [1490,1490], [1491,1491], [1492,1492], [1493,1493], [1494,1494], [1495,1495], [1496,1496], [1497,1497], [1550,1550], [1551,1551], [1552,1552], [1553,1553], [1554,1554], [1555,1555], [1556,1556], [1557,1557], [1558,1558], [1559,1559], [1597,1597], [1598,1598], [1599,1599], [1600,1600], [1601,1601], [1602,1602], [1603,1603], [1604,1604], [1605,1605], [1606,1606], [1607,1607], [1608,1608], [1609,1609], [1610,1610], [1611,1611], [1612,1612], [1613,1613], [1614,1614], [1615,1615], [1616,1616], [1623,1623], [1624,1624], [1625,1625], [1626,1626], [1627,1627], [1628,1628], [1629,1629], [1630,1630], [1631,1631], [1632,1632], [1709,1709], [1719,1719], [1720,1720], [1843,1843], [2813,2813], [2814,2814], [2815,2815], [2816,2816], [2817,2817], [2818,2818], [2819,2819], [2820,2820], [2821,2821], [2822,2822], [2823,2823], [2824,2824], [2825,2825], [2826,2826], [2827,2827], [2828,2828], [2829,2829], [2830,2830], [2831,2831], [2832,2832], [2833,2833], [2834,2834], [2835,2835], [2836,2836], [2837,2837], [2838,2838], [2839,2839], [2840,2840], [2841,2841], [2842,2842], [2843,2843], [2844,2844], [2845,2845], [2846,2846], [2847,2847], [2848,2848], [2849,2849], [2850,2850], [2851,2851], [2852,2852], [2853,2853], [2854,2854], [2855,2855], [2856,2856], [2857,2857], [2858,2858], [2859,2859], [2860,2860], [2861,2861], [2862,2862], [2863,2863], [2864,2864], [2865,2865], [2866,2866], [2867,2867], [2868,2868], [2869,2869], [2870,2870], [2871,2871], [2872,2872], [3139,3139], [3140,3140], [3141,3141], [3142,3142], [3143,3143], [3144,3144], [3145,3145], [3146,3146], [3147,3147], [3148,3148], [3149,3149], [3150,3150], [3151,3151], [3152,3152], [3153,3153], [3154,3154], [3155,3155], [3156,3156], [3157,3157], [3158,3158], [3386,3386], [3387,3387], [3388,3388], [3389,3389], [3390,3390], [3391,3391], [3392,3392], [3393,3393], [3394,3394], [3395,3395], [3664,3664], [3665,3665], [3666,3666], [3667,3667], [3668,3668], [3670,3670], [3671,3671], [3672,3672], [3673,3673], [3674,3674], [3676,3676], [3677,3677], [3678,3678], [3679,3679], [3680,3680], [3681,3681], [3682,3682], [3683,3683], [3684,3684], [3685,3685], [3686,3686], [3687,3687], [3688,3688], [3689,3689], [3690,3690], [3691,3691], [3692,3692], [3693,3693], [3694,3694], [3695,3695], [3696,3696], [3697,3697], [3698,3698], [3699,3699], [3700,3700], [3701,3701], [3702,3702], [3703,3703], [3704,3704], [3705,3705], [3706,3706], [3707,3707], [3708,3708], [3709,3709], [3710,3710], [3711,3711], [3712,3712], [3713,3713], [3714,3714], [3715,3715], [3960,3960], [3961,3961], [3962,3962], [3963,3963], [3964,3964], [3965,3965], [3966,3966], [3967,3967], [3968,3968], [3978,3978], [3979,3979], [3980,3980], [3981,3981], [3982,3982], [3983,3983], [3984,3984], [3985,3985], [3986,3986], [3987,3987], [4208,4208], [4209,4209], [4210,4210], [4211,4211], [4212,4212], [4304,4304], [4305,4305], [4306,4306], [4307,4307], [4308,4308], [4866,4866], [4867,4867], [4868,4868], [4869,4869], [4870,4870], [4871,4871], [4872,4872], [4873,4873], [4874,4874], [4875,4875], keep order:false + ├─IndexRangeScan(Build) 128.00 cop[tikv] table:dt, index:cm(cm) range:[1062,1062], [1086,1086], [1423,1423], [1424,1424], [1425,1425], [1426,1426], [1427,1427], [1428,1428], [1429,1429], [1430,1430], [1431,1431], [1432,1432], [1433,1433], [1434,1434], [1435,1435], [1436,1436], [1437,1437], [1438,1438], [1439,1439], [1440,1440], [1441,1441], [1442,1442], [1443,1443], [1444,1444], [1445,1445], [1446,1446], [1447,1447], [1448,1448], [1449,1449], [1450,1450], [1451,1451], [1452,1452], [1488,1488], [1489,1489], [1490,1490], [1491,1491], [1492,1492], [1493,1493], [1494,1494], [1495,1495], [1496,1496], [1497,1497], [1550,1550], [1551,1551], [1552,1552], [1553,1553], [1554,1554], [1555,1555], [1556,1556], [1557,1557], [1558,1558], [1559,1559], [1597,1597], [1598,1598], [1599,1599], [1600,1600], [1601,1601], [1602,1602], [1603,1603], [1604,1604], [1605,1605], [1606,1606], [1607,1607], [1608,1608], [1609,1609], [1610,1610], [1611,1611], [1612,1612], [1613,1613], [1614,1614], [1615,1615], [1616,1616], [1623,1623], [1624,1624], [1625,1625], [1626,1626], [1627,1627], [1628,1628], [1629,1629], [1630,1630], [1631,1631], [1632,1632], [1709,1709], [1719,1719], [1720,1720], [1843,1843], [2813,2813], [2814,2814], [2815,2815], [2816,2816], [2817,2817], [2818,2818], [2819,2819], [2820,2820], [2821,2821], [2822,2822], [2823,2823], [2824,2824], [2825,2825], [2826,2826], [2827,2827], [2828,2828], [2829,2829], [2830,2830], [2831,2831], [2832,2832], [2833,2833], [2834,2834], [2835,2835], [2836,2836], [2837,2837], [2838,2838], [2839,2839], [2840,2840], [2841,2841], [2842,2842], [2843,2843], [2844,2844], [2845,2845], [2846,2846], [2847,2847], [2848,2848], [2849,2849], [2850,2850], [2851,2851], [2852,2852], [2853,2853], [2854,2854], [2855,2855], [2856,2856], [2857,2857], [2858,2858], [2859,2859], [2860,2860], [2861,2861], [2862,2862], [2863,2863], [2864,2864], [2865,2865], [2866,2866], [2867,2867], [2868,2868], [2869,2869], [2870,2870], [2871,2871], [2872,2872], [3139,3139], [3140,3140], [3141,3141], [3142,3142], [3143,3143], [3144,3144], [3145,3145], [3146,3146], [3147,3147], [3148,3148], [3149,3149], [3150,3150], [3151,3151], [3152,3152], [3153,3153], [3154,3154], [3155,3155], [3156,3156], [3157,3157], [3158,3158], [3386,3386], [3387,3387], [3388,3388], [3389,3389], [3390,3390], [3391,3391], [3392,3392], [3393,3393], [3394,3394], [3395,3395], [3664,3664], [3665,3665], [3666,3666], [3667,3667], [3668,3668], [3670,3670], [3671,3671], [3672,3672], [3673,3673], [3674,3674], [3676,3676], [3677,3677], [3678,3678], [3679,3679], [3680,3680], [3681,3681], [3682,3682], [3683,3683], [3684,3684], [3685,3685], [3686,3686], [3687,3687], [3688,3688], [3689,3689], [3690,3690], [3691,3691], [3692,3692], [3693,3693], [3694,3694], [3695,3695], [3696,3696], [3697,3697], [3698,3698], [3699,3699], [3700,3700], [3701,3701], [3702,3702], [3703,3703], [3704,3704], [3705,3705], [3706,3706], [3707,3707], [3708,3708], [3709,3709], [3710,3710], [3711,3711], [3712,3712], [3713,3713], [3714,3714], [3715,3715], [3960,3960], [3961,3961], [3962,3962], [3963,3963], [3964,3964], [3965,3965], [3966,3966], [3967,3967], [3968,3968], [3978,3978], [3979,3979], [3980,3980], [3981,3981], [3982,3982], [3983,3983], [3984,3984], [3985,3985], [3986,3986], [3987,3987], [4208,4208], [4209,4209], [4210,4210], [4211,4211], [4212,4212], [4304,4304], [4305,4305], [4306,4306], [4307,4307], [4308,4308], [4866,4866], [4867,4867], [4868,4868], [4869,4869], [4870,4870], [4871,4871], [4872,4872], [4873,4873], [4874,4874], [4875,4875], keep order:false, stats:partial[cm:missing] └─HashAgg(Probe) 21.47 cop[tikv] group by:explain_complex_stats.dt.ds, explain_complex_stats.dt.p1, explain_complex_stats.dt.p2, explain_complex_stats.dt.p3, explain_complex_stats.dt.p4, explain_complex_stats.dt.p5, explain_complex_stats.dt.p6_md5, explain_complex_stats.dt.p7_md5, funcs:count(explain_complex_stats.dt.dic)->Column#32, funcs:firstrow(explain_complex_stats.dt.ds2)->Column#34 └─Selection 21.50 cop[tikv] ge(explain_complex_stats.dt.ds, 2016-09-01 00:00:00.000000), le(explain_complex_stats.dt.ds, 2016-11-03 00:00:00.000000) - └─TableRowIDScan 128.00 cop[tikv] table:dt keep order:false + └─TableRowIDScan 128.00 cop[tikv] table:dt keep order:false, stats:partial[cm:missing] explain format = 'brief' select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext, gad.t as gtime from st gad join (select id, aid, pt, dic, ip, t from dd where pt = 'android' and bm = 0 and t > 1478143908) sdk on gad.aid = sdk.aid and gad.ip = sdk.ip and sdk.t > gad.t where gad.t > 1478143908 and gad.bm = 0 and gad.pt = 'android' group by gad.aid, sdk.dic limit 2500; id estRows task access object operator info Projection 424.00 root explain_complex_stats.st.id, explain_complex_stats.dd.id, explain_complex_stats.st.aid, explain_complex_stats.st.cm, explain_complex_stats.dd.dic, explain_complex_stats.dd.ip, explain_complex_stats.dd.t, explain_complex_stats.st.p1, explain_complex_stats.st.p2, explain_complex_stats.st.p3, explain_complex_stats.st.p4, explain_complex_stats.st.p5, explain_complex_stats.st.p6_md5, explain_complex_stats.st.p7_md5, explain_complex_stats.st.ext, explain_complex_stats.st.t @@ -133,10 +151,10 @@ Projection 424.00 root explain_complex_stats.st.id, explain_complex_stats.dd.id └─HashJoin 424.00 root inner join, equal:[eq(explain_complex_stats.st.aid, explain_complex_stats.dd.aid) eq(explain_complex_stats.st.ip, explain_complex_stats.dd.ip)], other cond:gt(explain_complex_stats.dd.t, explain_complex_stats.st.t) ├─TableReader(Build) 424.00 root data:Selection │ └─Selection 424.00 cop[tikv] eq(explain_complex_stats.st.bm, 0), eq(explain_complex_stats.st.pt, "android"), gt(explain_complex_stats.st.t, 1478143908), not(isnull(explain_complex_stats.st.ip)) - │ └─TableFullScan 1999.00 cop[tikv] table:gad keep order:false + │ └─TableFullScan 1999.00 cop[tikv] table:gad keep order:false, stats:partial[t:missing] └─TableReader(Probe) 450.56 root data:Selection └─Selection 450.56 cop[tikv] eq(explain_complex_stats.dd.bm, 0), eq(explain_complex_stats.dd.pt, "android"), gt(explain_complex_stats.dd.t, 1478143908), not(isnull(explain_complex_stats.dd.ip)), not(isnull(explain_complex_stats.dd.t)) - └─TableFullScan 2000.00 cop[tikv] table:dd keep order:false + └─TableFullScan 2000.00 cop[tikv] table:dd keep order:false, stats:partial[ip:missing, t:missing] explain format = 'brief' select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000; id estRows task access object operator info Projection 170.34 root explain_complex_stats.st.id, explain_complex_stats.dd.id, explain_complex_stats.st.aid, explain_complex_stats.st.cm, explain_complex_stats.dd.dic, explain_complex_stats.dd.ip, explain_complex_stats.dd.t, explain_complex_stats.st.p1, explain_complex_stats.st.p2, explain_complex_stats.st.p3, explain_complex_stats.st.p4, explain_complex_stats.st.p5, explain_complex_stats.st.p6_md5, explain_complex_stats.st.p7_md5, explain_complex_stats.st.ext @@ -144,19 +162,19 @@ Projection 170.34 root explain_complex_stats.st.id, explain_complex_stats.dd.id └─IndexJoin 170.34 root inner join, inner:IndexLookUp, outer key:explain_complex_stats.st.aid, inner key:explain_complex_stats.dd.aid, equal cond:eq(explain_complex_stats.st.aid, explain_complex_stats.dd.aid), eq(explain_complex_stats.st.dic, explain_complex_stats.dd.mac), other cond:lt(explain_complex_stats.st.t, explain_complex_stats.dd.t) ├─TableReader(Build) 170.34 root data:Selection │ └─Selection 170.34 cop[tikv] eq(explain_complex_stats.st.bm, 0), eq(explain_complex_stats.st.dit, "mac"), eq(explain_complex_stats.st.pt, "ios"), gt(explain_complex_stats.st.t, 1477971479), not(isnull(explain_complex_stats.st.dic)) - │ └─TableFullScan 1999.00 cop[tikv] table:gad keep order:false + │ └─TableFullScan 1999.00 cop[tikv] table:gad keep order:false, stats:partial[t:missing] └─IndexLookUp(Probe) 170.34 root - ├─IndexRangeScan(Build) 669.25 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(explain_complex_stats.dd.aid, explain_complex_stats.st.aid)], keep order:false + ├─IndexRangeScan(Build) 669.25 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(explain_complex_stats.dd.aid, explain_complex_stats.st.aid)], keep order:false, stats:partial[t:missing] └─Selection(Probe) 170.34 cop[tikv] eq(explain_complex_stats.dd.bm, 0), eq(explain_complex_stats.dd.pt, "ios"), gt(explain_complex_stats.dd.t, 1477971479), not(isnull(explain_complex_stats.dd.mac)), not(isnull(explain_complex_stats.dd.t)) - └─TableRowIDScan 669.25 cop[tikv] table:sdk keep order:false + └─TableRowIDScan 669.25 cop[tikv] table:sdk keep order:false, stats:partial[t:missing] explain format = 'brief' SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5; id estRows task access object operator info Projection 39.28 root explain_complex_stats.st.cm, explain_complex_stats.st.p1, explain_complex_stats.st.p2, explain_complex_stats.st.p3, explain_complex_stats.st.p4, explain_complex_stats.st.p5, explain_complex_stats.st.p6_md5, explain_complex_stats.st.p7_md5, Column#20, Column#21 └─HashAgg 39.28 root group by:explain_complex_stats.st.cm, explain_complex_stats.st.p1, explain_complex_stats.st.p2, explain_complex_stats.st.p3, explain_complex_stats.st.p4, explain_complex_stats.st.p5, explain_complex_stats.st.p6_md5, explain_complex_stats.st.p7_md5, funcs:count(1)->Column#20, funcs:count(distinct explain_complex_stats.st.ip)->Column#21, funcs:firstrow(explain_complex_stats.st.cm)->explain_complex_stats.st.cm, funcs:firstrow(explain_complex_stats.st.p1)->explain_complex_stats.st.p1, funcs:firstrow(explain_complex_stats.st.p2)->explain_complex_stats.st.p2, funcs:firstrow(explain_complex_stats.st.p3)->explain_complex_stats.st.p3, funcs:firstrow(explain_complex_stats.st.p4)->explain_complex_stats.st.p4, funcs:firstrow(explain_complex_stats.st.p5)->explain_complex_stats.st.p5, funcs:firstrow(explain_complex_stats.st.p6_md5)->explain_complex_stats.st.p6_md5, funcs:firstrow(explain_complex_stats.st.p7_md5)->explain_complex_stats.st.p7_md5 └─IndexLookUp 39.38 root - ├─IndexRangeScan(Build) 160.23 cop[tikv] table:st, index:t(t) range:[1478188800,1478275200], keep order:false + ├─IndexRangeScan(Build) 160.23 cop[tikv] table:st, index:t(t) range:[1478188800,1478275200], keep order:false, stats:partial[t:missing] └─Selection(Probe) 39.38 cop[tikv] eq(explain_complex_stats.st.aid, "cn.sbkcq"), eq(explain_complex_stats.st.pt, "android") - └─TableRowIDScan 160.23 cop[tikv] table:st keep order:false + └─TableRowIDScan 160.23 cop[tikv] table:st keep order:false, stats:partial[t:missing] explain format = 'brief' select dt.id as id, dt.aid as aid, dt.pt as pt, dt.dic as dic, dt.cm as cm, rr.gid as gid, rr.acd as acd, rr.t as t,dt.p1 as p1, dt.p2 as p2, dt.p3 as p3, dt.p4 as p4, dt.p5 as p5, dt.p6_md5 as p6, dt.p7_md5 as p7 from dt dt join rr rr on (rr.pt = 'ios' and rr.t > 1478185592 and dt.aid = rr.aid and dt.dic = rr.dic) where dt.pt = 'ios' and dt.t > 1478185592 and dt.bm = 0 limit 2000; id estRows task access object operator info Projection 428.32 root explain_complex_stats.dt.id, explain_complex_stats.dt.aid, explain_complex_stats.dt.pt, explain_complex_stats.dt.dic, explain_complex_stats.dt.cm, explain_complex_stats.rr.gid, explain_complex_stats.rr.acd, explain_complex_stats.rr.t, explain_complex_stats.dt.p1, explain_complex_stats.dt.p2, explain_complex_stats.dt.p3, explain_complex_stats.dt.p4, explain_complex_stats.dt.p5, explain_complex_stats.dt.p6_md5, explain_complex_stats.dt.p7_md5 @@ -174,9 +192,9 @@ id estRows task access object operator info Projection 207.02 root explain_complex_stats.pp.pc, explain_complex_stats.pp.cr, Column#22, Column#23, Column#24 └─HashAgg 207.02 root group by:explain_complex_stats.pp.cr, explain_complex_stats.pp.pc, funcs:count(distinct explain_complex_stats.pp.uid)->Column#22, funcs:count(explain_complex_stats.pp.oid)->Column#23, funcs:sum(explain_complex_stats.pp.am)->Column#24, funcs:firstrow(explain_complex_stats.pp.pc)->explain_complex_stats.pp.pc, funcs:firstrow(explain_complex_stats.pp.cr)->explain_complex_stats.pp.cr └─IndexLookUp 207.02 root - ├─IndexRangeScan(Build) 627.00 cop[tikv] table:pp, index:ps(ps) range:[2,2], keep order:false + ├─IndexRangeScan(Build) 627.00 cop[tikv] table:pp, index:ps(ps) range:[2,2], keep order:false, stats:partial[uid:missing, ppt:missing, ps:missing] └─Selection(Probe) 207.02 cop[tikv] ge(explain_complex_stats.pp.ppt, 1478188800), in(explain_complex_stats.pp.pi, 510017, 520017), in(explain_complex_stats.pp.uid, 18089709, 18090780), lt(explain_complex_stats.pp.ppt, 1478275200) - └─TableRowIDScan 627.00 cop[tikv] table:pp keep order:false + └─TableRowIDScan 627.00 cop[tikv] table:pp keep order:false, stats:partial[uid:missing, ppt:missing, ps:missing] drop table if exists tbl_001; CREATE TABLE tbl_001 (a int, b int); load stats 's/explain_complex_stats_tbl_001.json'; diff --git a/tests/integrationtest/r/explain_easy_stats.result b/tests/integrationtest/r/explain_easy_stats.result index 1d375da7ac63c..8c26bcf04b8be 100644 --- a/tests/integrationtest/r/explain_easy_stats.result +++ b/tests/integrationtest/r/explain_easy_stats.result @@ -44,13 +44,13 @@ TableReader 1999.00 root data:TableRangeScan explain format = 'brief' select t1.c1, t1.c2 from t1 where t1.c2 = 1; id estRows task access object operator info IndexReader 0.00 root index:IndexRangeScan -└─IndexRangeScan 0.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false +└─IndexRangeScan 0.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:partial[c2:missing] explain format = 'brief' select * from t1 left join t2 on t1.c2 = t2.c1 where t1.c1 > 1; id estRows task access object operator info HashJoin 2481.25 root left outer join, equal:[eq(explain_easy_stats.t1.c2, explain_easy_stats.t2.c1)] ├─TableReader(Build) 1985.00 root data:Selection │ └─Selection 1985.00 cop[tikv] not(isnull(explain_easy_stats.t2.c1)) -│ └─TableFullScan 1985.00 cop[tikv] table:t2 keep order:false +│ └─TableFullScan 1985.00 cop[tikv] table:t2 keep order:false, stats:partial[c1:missing] └─TableReader(Probe) 1998.00 root data:TableRangeScan └─TableRangeScan 1998.00 cop[tikv] table:t1 range:(1,+inf], keep order:false explain format = 'brief' update t1 set t1.c2 = 2 where t1.c1 = 1; @@ -61,8 +61,8 @@ explain format = 'brief' delete from t1 where t1.c2 = 1; id estRows task access object operator info Delete N/A root N/A └─IndexLookUp 0.00 root - ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false - └─TableRowIDScan(Probe) 0.00 cop[tikv] table:t1 keep order:false + ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:partial[c2:missing] + └─TableRowIDScan(Probe) 0.00 cop[tikv] table:t1 keep order:false, stats:partial[c2:missing] explain format = 'brief' select count(b.c2) from t1 a, t2 b where a.c1 = b.c2 group by a.c1; id estRows task access object operator info Projection 1985.00 root Column#7 @@ -82,9 +82,9 @@ TopN 1.00 root explain_easy_stats.t2.c2, offset:0, count:1 explain format = 'brief' select * from t1 where c1 > 1 and c2 = 1 and c3 < 1; id estRows task access object operator info IndexLookUp 0.00 root -├─IndexRangeScan(Build) 0.00 cop[tikv] table:t1, index:c2(c2) range:(1 1,1 +inf], keep order:false +├─IndexRangeScan(Build) 0.00 cop[tikv] table:t1, index:c2(c2) range:(1 1,1 +inf], keep order:false, stats:partial[c2:missing] └─Selection(Probe) 0.00 cop[tikv] lt(explain_easy_stats.t1.c3, 1) - └─TableRowIDScan 0.00 cop[tikv] table:t1 keep order:false + └─TableRowIDScan 0.00 cop[tikv] table:t1 keep order:false, stats:partial[c2:missing] explain format = 'brief' select * from t1 where c1 = 1 and c2 > 1; id estRows task access object operator info Selection 0.50 root gt(explain_easy_stats.t1.c2, 1) diff --git a/tests/integrationtest/r/expression/json.result b/tests/integrationtest/r/expression/json.result index a2092ffe22376..2f1cfdece1660 100644 --- a/tests/integrationtest/r/expression/json.result +++ b/tests/integrationtest/r/expression/json.result @@ -628,3 +628,24 @@ a cast(a as signed) "-1" -1 "18446744073709551615" -1 "18446744073709552000" -1 +select cast(binary 'aa' as json); +cast(binary 'aa' as json) +"base64:type254:YWE=" +drop table if exists t; +create table t (vb VARBINARY(10), b BINARY(10), vc VARCHAR(10), c CHAR(10)); +insert into t values ('1', '1', '1', '1'); +select cast(vb as json), cast(b as json), cast(vc as json), cast(c as json) from t; +cast(vb as json) cast(b as json) cast(vc as json) cast(c as json) +"base64:type15:MQ==" "base64:type254:MQAAAAAAAAAAAA==" 1 1 +select 1 from t where cast(vb as json) = '1'; +1 +select 1 from t where cast(b as json) = '1'; +1 +select 1 from t where cast(vc as json) = '1'; +1 +select 1 from t where cast(c as json) = '1'; +1 +select 1 from t where cast(BINARY vc as json) = '1'; +1 +select 1 from t where cast(BINARY c as json) = '1'; +1 diff --git a/tests/integrationtest/r/expression/misc.result b/tests/integrationtest/r/expression/misc.result index e55f8fc4fe65d..c6e95ea94f305 100644 --- a/tests/integrationtest/r/expression/misc.result +++ b/tests/integrationtest/r/expression/misc.result @@ -242,7 +242,7 @@ Warning 8061 Optimizer hint unknown_hint is not supported by TiDB and is ignored select 1 from /*+ test1() */ t; 1 Level Code Message -Warning 1064 You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use [parser:8066]Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. +Warning 8066 Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. drop table if exists t; create table t(a bigint, b double, c decimal, d varchar(20), e datetime, f time, g json); insert into t values(1, 1.1, 2.2, "abc", "2018-10-24", NOW(), "12"); diff --git a/tests/integrationtest/r/infoschema/v2.result b/tests/integrationtest/r/infoschema/v2.result new file mode 100644 index 0000000000000..6d9f0339d2b94 --- /dev/null +++ b/tests/integrationtest/r/infoschema/v2.result @@ -0,0 +1,15 @@ +set @@global.tidb_schema_cache_size = 1024; +use infoschema__v2; +drop table if exists t1; +create table t1 (id int); +rename table t1 to t2; +show tables; +Tables_in_infoschema__v2 +t2 +select * from t2; +id +select * from t1; +Error 1146 (42S02): Table 'infoschema__v2.t1' doesn't exist +show create table t1; +Error 1146 (42S02): Table 'infoschema__v2.t1' doesn't exist +set @@global.tidb_schema_cache_size = default; diff --git a/tests/integrationtest/r/planner/core/casetest/hint/hint.result b/tests/integrationtest/r/planner/core/casetest/hint/hint.result index a47430e2f2a85..1fd67eb399eed 100644 --- a/tests/integrationtest/r/planner/core/casetest/hint/hint.result +++ b/tests/integrationtest/r/planner/core/casetest/hint/hint.result @@ -8,12 +8,12 @@ select count(*) /*+ stream_agg() */ from t where a > 1; count(*) /*+ stream_agg() 0 Level Code Message -Warning 1064 You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use [parser:8066]Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. +Warning 8066 Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. select count(*) from (select count(*) as a /*+ stream_agg() */ from t where b > 1 group by b) t1 where a > 1; count(*) 0 Level Code Message -Warning 1064 You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use [parser:8066]Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. +Warning 8066 Optimizer hint can only be followed by certain keywords like SELECT, INSERT, etc. set tidb_cost_model_version=2; drop table if exists t, t1, th; drop view if exists v, v1; diff --git a/tests/integrationtest/r/planner/core/casetest/index/index.result b/tests/integrationtest/r/planner/core/casetest/index/index.result index 9fa10d38fdc10..42fa7873de187 100644 --- a/tests/integrationtest/r/planner/core/casetest/index/index.result +++ b/tests/integrationtest/r/planner/core/casetest/index/index.result @@ -54,11 +54,10 @@ IndexMerge_9 0.20 root type: union └─TableRowIDScan_7 0.20 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where ( json_contains(a, '[1, 2, 3]') and c=1 and d=2) or (2 member of (b) and c=3 and d=2); -- 4: OR index merge from multi complicated mv index (memberof)，make full use of DNF item's condition even if the predicate is intersection case (json_contains); id estRows task access object operator info -IndexMerge_9 0.01 root type: union -├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[1,1], keep order:false, stats:pseudo -├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[2 3,2 3], keep order:false, stats:pseudo -└─Selection_8(Probe) 0.01 cop[tikv] or(and(json_contains(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) - └─TableRowIDScan_7 10.10 cop[tikv] table:t2 keep order:false, stats:pseudo +IndexLookUp_11 8.00 root +├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t2, index:idx3(c, d) range:[1 2,1 2], [3 2,3 2], keep order:false, stats:pseudo +└─Selection_10(Probe) 8.00 cop[tikv] or(and(json_contains(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) + └─TableRowIDScan_9 10.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where ( json_overlaps(a, '[1, 2, 3]') and c=1 and d=2) or (2 member of (b) and c=3 and d=2); -- 5: OR index merge from multi complicated mv index (memberof)，make full use of DNF item's condition even if the predicate is intersection case (json_contains); id estRows task access object operator info Selection_5 0.32 root or(and(json_overlaps(planner__core__casetest__index__index.t2.a, cast("[1, 2, 3]", json BINARY)), and(eq(planner__core__casetest__index__index.t2.c, 1), eq(planner__core__casetest__index__index.t2.d, 2))), and(json_memberof(cast(2, json BINARY), planner__core__casetest__index__index.t2.b), and(eq(planner__core__casetest__index__index.t2.c, 3), eq(planner__core__casetest__index__index.t2.d, 2)))) @@ -90,9 +89,9 @@ TableReader_7 10.18 root data:Selection_6 explain select /*+ use_index_merge(t2, idx2, idx, idx4) */ * from t2 where (1 member of (a) and 1 member of (b) and c=3) or (3 member of (b) and c=4) or d=1; -- 9: OR index merge from multi complicated mv index (memberof), each DNF item should be strict or lax used as index partial path, specify the index in index merge hint; id estRows task access object operator info IndexMerge_10 0.01 root type: union -├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t2, index:idx4(d) range:[1,1], keep order:false, stats:pseudo -├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[3 1,3 1], keep order:false, stats:pseudo -├─IndexRangeScan_7(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[3 4,3 4], keep order:false, stats:pseudo +├─IndexRangeScan_5(Build) 0.10 cop[tikv] table:t2, index:idx(c, cast(`a` as signed array)) range:[3 1,3 1], keep order:false, stats:pseudo +├─IndexRangeScan_6(Build) 0.10 cop[tikv] table:t2, index:idx2(cast(`b` as signed array), c) range:[3 4,3 4], keep order:false, stats:pseudo +├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t2, index:idx4(d) range:[1,1], keep order:false, stats:pseudo └─Selection_9(Probe) 0.01 cop[tikv] or(and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.a), and(json_memberof(cast(1, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 3))), or(and(json_memberof(cast(3, json BINARY), planner__core__casetest__index__index.t2.b), eq(planner__core__casetest__index__index.t2.c, 4)), eq(planner__core__casetest__index__index.t2.d, 1))) └─TableRowIDScan_8 10.20 cop[tikv] table:t2 keep order:false, stats:pseudo drop table if exists t1, t2; diff --git a/tests/integrationtest/r/planner/core/indexmerge_path.result b/tests/integrationtest/r/planner/core/indexmerge_path.result index 4fc1903e018e2..77f45330efa03 100644 --- a/tests/integrationtest/r/planner/core/indexmerge_path.result +++ b/tests/integrationtest/r/planner/core/indexmerge_path.result @@ -737,3 +737,209 @@ IndexMerge 11.00 root type: union ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, index:PRIMARY(a, b) range:[1 2,1 2], keep order:false, stats:pseudo ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi(cast(`j` as unsigned array)) range:[1,1], keep order:false, stats:pseudo └─TableRowIDScan(Probe) 11.00 cop[tikv] table:t1 keep order:false, stats:pseudo +drop table if exists t, t1; +create table t (a int, b varchar(30), c float, j json, pk int primary key, +key mvi1(c, (cast(j->'$.a' as unsigned array)), b), +key mvi2(a, (cast(j->'$.c' as unsigned array))), +key mvi3((cast(j->'$.d' as unsigned array)), c), +key idx(b, c) +); +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +json_overlaps(j->'$.a', '[4,5,6]') or +(2 member of (j->'$.a')) +) and +c = 10 and +a = 20; +id estRows task access object operator info +Selection 0.01 root or(json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)), json_memberof(cast(2, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a"))) +└─IndexMerge 0.40 root type: union + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 4,10 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 5,10 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 6,10 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 2,10 2], keep order:false, stats:pseudo + └─Selection(Probe) 0.40 cop[tikv] eq(planner__core__indexmerge_path.t.a, 20), eq(planner__core__indexmerge_path.t.c, 10) + └─TableRowIDScan 0.40 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +c = 1 or +c = 2 or +c = 3 +) and +json_overlaps(j->'$.a', '[4,5,6]'); +id estRows task access object operator info +Selection 0.72 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)) +└─IndexMerge 0.90 root type: union + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 6,1 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[2 4,2 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[2 5,2 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[2 6,2 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 4,3 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 5,3 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 6,3 6], keep order:false, stats:pseudo + └─Selection(Probe) 0.90 cop[tikv] or(eq(planner__core__indexmerge_path.t.c, 1), or(eq(planner__core__indexmerge_path.t.c, 2), eq(planner__core__indexmerge_path.t.c, 3))) + └─TableRowIDScan 0.90 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +c = 1 or +c = 2 or +c = 3 +) and +(json_contains(j->'$.a', '[4,5,6]')); +id estRows task access object operator info +TableReader 24.00 root data:Selection +└─Selection 24.00 cop[tikv] json_contains(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)), or(eq(planner__core__indexmerge_path.t.c, 1), or(eq(planner__core__indexmerge_path.t.c, 2), eq(planner__core__indexmerge_path.t.c, 3))) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +c = 1 or +c = 2 or +c = 3 +) and +json_contains(j->'$.a', '[2]'); +id estRows task access object operator info +IndexMerge 0.30 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 2,1 2], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[2 2,2 2], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 2,3 2], keep order:false, stats:pseudo +└─Selection(Probe) 0.30 cop[tikv] json_contains(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[2]", json BINARY)), or(eq(planner__core__indexmerge_path.t.c, 1), or(eq(planner__core__indexmerge_path.t.c, 2), eq(planner__core__indexmerge_path.t.c, 3))) + └─TableRowIDScan 0.30 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +1 member of (j->'$.a') or +2 member of (j->'$.a') +) and +c = 10 and +a = 20; +id estRows task access object operator info +IndexMerge 0.20 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 1,10 1], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 2,10 2], keep order:false, stats:pseudo +└─Selection(Probe) 0.20 cop[tikv] eq(planner__core__indexmerge_path.t.a, 20), eq(planner__core__indexmerge_path.t.c, 10), or(json_memberof(cast(1, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a")), json_memberof(cast(2, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a"))) + └─TableRowIDScan 0.20 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( +1 member of (j->'$.a') or +2 member of (j->'$.d') +) and +a = 20; +id estRows task access object operator info +TableReader 8.00 root data:Selection +└─Selection 8.00 cop[tikv] eq(planner__core__indexmerge_path.t.a, 20), or(json_memberof(cast(1, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a")), json_memberof(cast(2, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.d"))) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi3) */ * from t where +c = 5 and +( +1 member of (j->'$.a') or +2 member of (j->'$.d') +) and +a = 20; +id estRows task access object operator info +IndexMerge 0.20 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[5 1,5 1], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi3(cast(json_extract(`j`, _utf8mb4'$.d') as unsigned array), c) range:[2 5,2 5], keep order:false, stats:pseudo +└─Selection(Probe) 0.20 cop[tikv] eq(planner__core__indexmerge_path.t.a, 20), eq(planner__core__indexmerge_path.t.c, 5), or(json_memberof(cast(1, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a")), json_memberof(cast(2, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.d"))) + └─TableRowIDScan 0.20 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, primary) */ * from t where +( +pk = 2 or +json_overlaps(j->'$.a', '[4,5,6]') or +json_contains(j->'$.c', '[3]') +) and +a = 1 and +b = '2' and +c = 3; +id estRows task access object operator info +Selection 0.00 root or(eq(planner__core__indexmerge_path.t.pk, 2), or(json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)), json_contains(json_extract(planner__core__indexmerge_path.t.j, "$.c"), cast("[3]", json BINARY)))) +└─IndexMerge 0.00 root type: union + ├─TableRangeScan(Build) 1.00 cop[tikv] table:t range:[2,2], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 4 "2",3 4 "2"], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 5 "2",3 5 "2"], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 6 "2",3 6 "2"], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo + └─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), eq(planner__core__indexmerge_path.t.b, "2"), eq(planner__core__indexmerge_path.t.c, 3) + └─TableRowIDScan 1.10 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, primary) */ * from t where +a = 1 and +b = '2' and +c = 3 and +( +pk = 2 or +(3 member of (j->'$.a')) or +(3 member of (j->'$.c')) +); +id estRows task access object operator info +IndexMerge 0.00 root type: union +├─TableRangeScan(Build) 1.00 cop[tikv] table:t range:[2,2], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[3 3 "2",3 3 "2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), eq(planner__core__indexmerge_path.t.b, "2"), eq(planner__core__indexmerge_path.t.c, 3), or(eq(planner__core__indexmerge_path.t.pk, 2), or(json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a")), json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.c")))) + └─TableRowIDScan 1.10 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b = '2' and +( +c = 20 or +(c = 10 and 3 member of (j->'$.a')) or +3 member of (j->'$.c') +); +id estRows task access object operator info +IndexMerge 0.20 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:idx(b, c) range:["2" 20,"2" 20], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 3 "2",10 3 "2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo +└─Selection(Probe) 0.20 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), eq(planner__core__indexmerge_path.t.b, "2"), or(eq(planner__core__indexmerge_path.t.c, 20), or(and(eq(planner__core__indexmerge_path.t.c, 10), json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.a"))), json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.c")))) + └─TableRowIDScan 0.20 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +(json_overlaps(j->'$.a', '[4,5,6]')) and +( +(b = '2' and c > 20) or +c = 10 or +3 member of (j->'$.c') +); +id estRows task access object operator info +Selection 6.41 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)) +└─IndexMerge 0.03 root type: union + ├─IndexRangeScan(Build) 33.33 cop[tikv] table:t, index:idx(b, c) range:("2" 20,"2" +inf], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 4,10 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 5,10 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 6,10 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo + └─Selection(Probe) 0.03 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), or(and(eq(planner__core__indexmerge_path.t.b, "2"), gt(planner__core__indexmerge_path.t.c, 20)), or(eq(planner__core__indexmerge_path.t.c, 10), json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.c")))) + └─TableRowIDScan 33.73 cop[tikv] table:t keep order:false, stats:pseudo +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +(json_overlaps(j->'$.a', '[4,5,6]')) and +( +(b > '2' and c = 20) or +(c = 10) or +(3 member of (j->'$.c')) +); +id estRows task access object operator info +Selection 0.56 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)) +└─IndexMerge 0.70 root type: union + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[20 4,20 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[20 5,20 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[20 6,20 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 4,10 4], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 5,10 5], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 6,10 6], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo + └─Selection(Probe) 0.70 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), or(and(gt(planner__core__indexmerge_path.t.b, "2"), eq(planner__core__indexmerge_path.t.c, 20)), or(eq(planner__core__indexmerge_path.t.c, 10), json_memberof(cast(3, json BINARY), json_extract(planner__core__indexmerge_path.t.j, "$.c")))) + └─TableRowIDScan 0.70 cop[tikv] table:t keep order:false, stats:pseudo +create table t1 (a int, b int, c int, d int, j json, key kb(b, (cast(j as unsigned array))), key(d, c)); +explain format=brief select * from t1 where (c=1 or b=1) and (1 member of (j)); +id estRows task access object operator info +TableReader 15.99 root data:Selection +└─Selection 15.99 cop[tikv] json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t1.j), or(eq(planner__core__indexmerge_path.t1.c, 1), eq(planner__core__indexmerge_path.t1.b, 1)) + └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain format=brief select * from t1 where (c=1 or b=1) and (1 member of (j)) and d=1; +id estRows task access object operator info +IndexMerge 0.20 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t1, index:d(d, c) range:[1 1,1 1], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t1, index:kb(b, cast(`j` as unsigned array)) range:[1 1,1 1], keep order:false, stats:pseudo +└─Selection(Probe) 0.20 cop[tikv] eq(planner__core__indexmerge_path.t1.d, 1), json_memberof(cast(1, json BINARY), planner__core__indexmerge_path.t1.j), or(eq(planner__core__indexmerge_path.t1.c, 1), eq(planner__core__indexmerge_path.t1.b, 1)) + └─TableRowIDScan 0.20 cop[tikv] table:t1 keep order:false, stats:pseudo diff --git a/tests/integrationtest/r/planner/core/integration.result b/tests/integrationtest/r/planner/core/integration.result index 89e11b5df3bf4..56aa43bc98f91 100644 --- a/tests/integrationtest/r/planner/core/integration.result +++ b/tests/integrationtest/r/planner/core/integration.result @@ -305,54 +305,6 @@ c2 c0 0 0 1 1 0 1 -drop table if exists t; -create table t(a int, b int, unique index i_a (a) invisible, unique index i_b(b)); -insert into t values (1,2); -admin check table t; -select a from t order by a; -a -1 -explain select a from t order by a; -id estRows task access object operator info -Sort_4 10000.00 root planner__core__integration.t.a -└─TableReader_8 10000.00 root data:TableFullScan_7 - └─TableFullScan_7 10000.00 cop[tikv] table:t keep order:false, stats:pseudo -select a from t where a > 0; -a -1 -explain select a from t where a > 1; -id estRows task access object operator info -TableReader_7 3333.33 root data:Selection_6 -└─Selection_6 3333.33 cop[tikv] gt(planner__core__integration.t.a, 1) - └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo -select * from t use index(i_a); -Error 1176 (42000): Key 'i_a' doesn't exist in table 't' -select * from t force index(i_a); -Error 1176 (42000): Key 'i_a' doesn't exist in table 't' -select * from t ignore index(i_a); -Error 1176 (42000): Key 'i_a' doesn't exist in table 't' -select /*+ USE_INDEX(t, i_a) */ * from t; -a b -1 2 -Level Code Message -Warning 1176 Key 'i_a' doesn't exist in table 't' -select /*+ IGNORE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a; -a -1 -Level Code Message -Warning 1176 Key 'i_a' doesn't exist in table 't' -select /*+ FORCE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a; -a -1 -Level Code Message -Warning 1176 Key 'i_a' doesn't exist in table 't' -select /*+ FORCE_INDEX(aaa) */ * from t; -a b -1 2 -Level Code Message -Warning 1815 force_index(planner__core__integration.aaa) is inapplicable, check whether the table(planner__core__integration.aaa) exists -admin check table t; -admin check index t i_a; select max(t.col) from (select 'a' as col union all select '' as col) as t; max(t.col) a diff --git a/tests/integrationtest/r/planner/core/issuetest/planner_issue.result b/tests/integrationtest/r/planner/core/issuetest/planner_issue.result index fa909e1c03283..b7a0a40154fd7 100644 --- a/tests/integrationtest/r/planner/core/issuetest/planner_issue.result +++ b/tests/integrationtest/r/planner/core/issuetest/planner_issue.result @@ -335,3 +335,28 @@ id value 3 0 4 4 5 5 +create table A(a int primary key, b int); +create table B(b int primary key); +create table C(c int primary key, b int); +insert into A values (2, 1), (3, 2); +insert into B values (1), (2); +select b.b +from A a +left join ( +B b +left join C c on b.b = c.b) +on b.b = a.b +where a.a in (2, 3); +b +1 +2 +select b.b +from A a +left join ( +B b +left join C c on b.b = c.b) +on b.b = a.b +where a.a in (2, 3, null); +b +1 +2 diff --git a/tests/integrationtest/r/table/index.result b/tests/integrationtest/r/table/index.result index 7cec18d6bd831..fa9a834826d9e 100644 --- a/tests/integrationtest/r/table/index.result +++ b/tests/integrationtest/r/table/index.result @@ -31,3 +31,50 @@ Error 1062 (23000): Duplicate entry 'q' for key 't.idx_a' update ignore t set a = 'qcc' where a = 'rcc'; Level Code Message Warning 1062 Duplicate entry 'q' for key 't.idx_a' +drop table if exists t; +create table t (id int, a varchar(64), b varchar(64), c varchar(64), index idx_a(a(64))); +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `id` int(11) DEFAULT NULL, + `a` varchar(64) DEFAULT NULL, + `b` varchar(64) DEFAULT NULL, + `c` varchar(64) DEFAULT NULL, + KEY `idx_a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +alter table t add index idx_b(b(64)); +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `id` int(11) DEFAULT NULL, + `a` varchar(64) DEFAULT NULL, + `b` varchar(64) DEFAULT NULL, + `c` varchar(64) DEFAULT NULL, + KEY `idx_a` (`a`), + KEY `idx_b` (`b`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +alter table t add index idx_c(c(32)); +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `id` int(11) DEFAULT NULL, + `a` varchar(64) DEFAULT NULL, + `b` varchar(64) DEFAULT NULL, + `c` varchar(64) DEFAULT NULL, + KEY `idx_a` (`a`), + KEY `idx_b` (`b`), + KEY `idx_c` (`c`(32)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +alter table t modify column c varchar(32); +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `id` int(11) DEFAULT NULL, + `a` varchar(64) DEFAULT NULL, + `b` varchar(64) DEFAULT NULL, + `c` varchar(32) DEFAULT NULL, + KEY `idx_a` (`a`), + KEY `idx_b` (`b`), + KEY `idx_c` (`c`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +drop table t; diff --git a/tests/integrationtest/t/ddl/bdr_mode.test b/tests/integrationtest/t/ddl/bdr_mode.test index cecf570994016..4e79a72072028 100644 --- a/tests/integrationtest/t/ddl/bdr_mode.test +++ b/tests/integrationtest/t/ddl/bdr_mode.test @@ -564,5 +564,9 @@ alter table t add column c int null; -- error 8263 alter table t add column d int not null; alter table t add column d int not null default 10; +alter table t add column e int default 10; +alter table t add column f int comment "test"; +alter table t add column g int default 10 comment "test"; +alter table t add column h int as (a + 1) virtual; admin unset bdr role; drop database bdr_mode; diff --git a/tests/integrationtest/t/ddl/default_as_expression.test b/tests/integrationtest/t/ddl/default_as_expression.test index e6c67dba4cf5a..6c52f2c9602b1 100644 --- a/tests/integrationtest/t/ddl/default_as_expression.test +++ b/tests/integrationtest/t/ddl/default_as_expression.test @@ -18,23 +18,24 @@ alter table t0 add column c2 date default (date_format(now(),'%Y-%m')); SET @x := NOW(); insert into t0(c) values (1); insert into t0 values (2, default); -SELECT * FROM t0 WHERE c = date_format(@x,'%Y-%m') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m'); +SELECT count(1) FROM t0 WHERE c1 = date_format(@x,'%Y-%m'); insert into t1(c) values (1); insert into t1 values (2, default); -SELECT * FROM t1 WHERE c = date_format(@x,'%Y-%m-%d'); +SELECT count(1) FROM t1 WHERE c1 = date_format(@x,'%Y-%m-%d'); +SET @x := NOW(); insert into t2(c) values (1); insert into t2 values (2, default); -SELECT * FROM t2 WHERE c = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); +SELECT count(1) FROM t2 WHERE c1 = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c1 = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); SET @x := NOW(); insert into t3(c) values (1); insert into t3 values (2, default); -SELECT * FROM t3 WHERE c = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); +SELECT count(1) FROM t3 WHERE c1 = date_format(@x,'%Y-%m-%d %H.%i.%s') OR c1 = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H.%i.%s'); insert into t4(c) values (1); insert into t4 values (2, default); -SELECT * FROM t4 WHERE c = date_format(@x,'%Y-%m-%d %H:%i:%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H:%i:%s'); +SELECT count(1) FROM t4 WHERE c1 = date_format(@x,'%Y-%m-%d'); insert into t5(c) values (1); insert into t5 values (2, default); -SELECT * FROM t5 WHERE c = date_format(@x,'%Y-%m-%d %H:%i:%s') OR c = date_format(DATE_ADD(@x, INTERVAL 1 SECOND), '%Y-%m-%d %H:%i:%s'); +SELECT count(1) FROM t5 WHERE c1 = date_format(@x,'%Y-%m-%d'); show create table t0; show create table t1; @@ -172,7 +173,7 @@ show create table t0; show create table t1; show create table t2; -# test modify column, set default value, add index +# test modify column, set default value, add index, drop column alter table t0 add index idx1(c1); alter table t1 add unique index idx1(c, c1); insert into t0 values (5, default, default); @@ -194,6 +195,10 @@ select * from t0 where c < 6; select c, c1 from t0 where c = 6 and c2 = date_format(now(),'%Y-%m-%d');; select * from t1; select * from t2; +-- error 8200 +alter table t0 drop column c1; +alter table t0 drop column c2; +show create table t0; SELECT column_default, extra FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema='test' AND TABLE_NAME='t1' AND COLUMN_NAME='c1'; # TestDefaultColumnWithUpper @@ -397,13 +402,103 @@ ALTER TABLE t0 ALTER COLUMN c SET DEFAULT(str_to_date('1980-01-01','%Y-%m-%d')); insert into t0(id) values (2); drop table t0; -# test generated column -CREATE TABLE t1 (i INT, b int DEFAULT (str_to_date('1980-01-01','%Y-%m-%d')), c INT GENERATED ALWAYS AS (b+2)); +# test generated column and expression index +CREATE TABLE t1 (i INT, b int DEFAULT (str_to_date('1980-01-01','%Y-%m-%d')), c INT GENERATED ALWAYS AS (b+2), d INT GENERATED ALWAYS AS (b+10) STORED); +INSERT INTO t1(i) VALUES (1); +CREATE INDEX idx1 ON t1 ((b+1)); +CREATE INDEX idx2 ON t1 ((c+1)); +CREATE INDEX idx3 ON t1 ((d+1)); SHOW COLUMNS FROM t1; show create table t1; -INSERT INTO t1(i) VALUES (1); INSERT INTO t1(i, b) VALUES (2, DEFAULT); INSERT INTO t1(i, b) VALUES (3, 123); INSERT INTO t1(i, b) VALUES (NULL, NULL); SELECT * FROM t1; drop table t1; + +# clustered index, multi-valued index and replace into +create table t0 (c int(10), c1 int default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c, c1)); +REPLACE INTO t0 VALUES (1, DEFAULT); +SELECT * FROM t0; +show columns from test.t0 where field='c1'; +-- error 3152 +create table t1 (c int(10), c1 BLOB default (date_format(now(),'%Y-%m-%d')), c2 JSON default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c1(32), c2)); +create table t1 (c int(10), c1 BLOB default (date_format(now(),'%Y-%m-%d')), c2 JSON default (str_to_date('1980-01-01','%Y-%m-%d')), primary key(c1(32))); +SET @x := NOW(); +REPLACE INTO t1 VALUES (1, DEFAULT, '[1,1,2]'); +CREATE INDEX idx ON t1 ((cast(c2 as signed array))); +REPLACE INTO t1 VALUES (1, DEFAULT, '[3, 4]'); +SELECT count(1) FROM t1 WHERE c1 = date_format(@x,'%Y-%m-%d'); +show create table t1; +drop table t0, t1; + +# partition table and global index +CREATE TABLE t0( + id INT NOT NULL, + c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), + d datetime default (date_format(now(),'%Y-%m-%d %H:%i:%s')), + unique key idx(id, c), + key idx1(id, c, d) +) +PARTITION BY RANGE (YEAR(c)) ( + PARTITION p0 VALUES LESS THAN (1991), + PARTITION p1 VALUES LESS THAN (1996), + PARTITION p2 VALUES LESS THAN (2001), + PARTITION p3 VALUES LESS THAN MAXVALUE +); +INSERT INTO t0 VALUES(1, default, '1998-05-04 10:10:10'), (2, '1990-05-04 10:10:10', default),(3, default, '1991-05-04 10:10:10'), (4, '2000-05-04 10:10:10', '1991-05-04 10:10:10'),(5, default, '2002-05-04 10:10:10'); +select id from t0 order by c, d; +show create table t0; +drop table t0; + +# temporary table +CREATE TEMPORARY TABLE t0( + id BIGINT, + c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), + PRIMARY KEY(id, c) +); +show create table t0; +SET @x := NOW(); +INSERT INTO t0 VALUES(1, default); +SELECT count(1) FROM t0 WHERE c = date_format(@x,'%Y-%m-%d'); +show create table t0; +drop table t0; + +# cache table +CREATE TABLE t0( + id BIGINT, + c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), + PRIMARY KEY(id, c) +); +SET @x := NOW(); +INSERT INTO t0 VALUES(1, default); +ALTER TABLE t0 CACHE; +INSERT INTO t0 VALUES(2, default); +SELECT count(1) FROM t0 WHERE c = date_format(@x,'%Y-%m-%d'); +show create table t0; +ALTER TABLE t0 NOCACHE; +drop table t0; + +# foreign key +CREATE TABLE parent ( + id INT, + c date default (date_format(now(),'%Y-%m-%d %H:%i:%s')), + primary key(c) +); +CREATE TABLE child ( + id INT, + cc date default (date_format(now(),'%Y-%m-%d')), + INDEX idx (cc), + FOREIGN KEY (cc) REFERENCES parent(c) ON DELETE CASCADE +); +SET @x := NOW(); +INSERT INTO parent VALUES(1, default); +INSERT INTO child VALUES(1, default); +alter table child add foreign key fk_2(cc) references parent(c); +-- error 0,1062 +INSERT INTO parent VALUES(2, default); +alter table child drop foreign key fk_2; +SELECT count(1) FROM parent WHERE c = date_format(@x,'%Y-%m-%d'); +SELECT count(1) FROM child WHERE cc = date_format(@x,'%Y-%m-%d'); +show create table child; +drop table parent, child; diff --git a/tests/integrationtest/t/ddl/integration.test b/tests/integrationtest/t/ddl/integration.test index 4104c1f824c67..09eb605b3d5a9 100644 --- a/tests/integrationtest/t/ddl/integration.test +++ b/tests/integrationtest/t/ddl/integration.test @@ -82,3 +82,40 @@ set sql_mode=''; create table t (d int default '18446744073709551616' ); -- disable_warnings set sql_mode=DEFAULT; + +# Test alter non-partition table to partition with global index needed. +drop table if exists t; +create table t(a int not null, b int, primary key(a), unique idx_b(b)); +drop table if exists t2; +create table t2(a int not null, b int, primary key(a) nonclustered, unique idx_b(b)); +drop table if exists t3; +set tidb_enable_global_index=1; +create table t3(a int not null, b int, primary key(a) nonclustered, unique idx_b(b)) partition by hash(a) partitions 3; +drop table if exists t4; +create table t4(a int not null, b int, primary key(a)) partition by hash(a) partitions 3; +-- error 8214 +alter table t partition by hash(a) partitions 3; +-- error 8214 +alter table t partition by key() partitions 3; +-- error 1503 +alter table t partition by hash(b) partitions 3; +-- error 8214 +alter table t2 partition by hash(b) partitions 3; +-- error 8214 +alter table t3 partition by key(a) partitions 3; +-- error 1503 +alter table t4 partition by hash(b) partitions 3; +set tidb_enable_global_index=0; +-- error 1503 +alter table t partition by hash(a) partitions 3; +-- error 1503 +alter table t partition by key() partitions 3; +-- error 1503 +alter table t partition by hash(b) partitions 3; +-- error 1503 +alter table t2 partition by hash(b) partitions 3; +-- error 1503 +alter table t3 partition by key(a) partitions 3; +-- error 1503 +alter table t4 partition by hash(b) partitions 3; +drop table t, t2, t3, t4; diff --git a/tests/integrationtest/t/ddl/partition.test b/tests/integrationtest/t/ddl/partition.test index 25f5964c71034..0e243251ab599 100644 --- a/tests/integrationtest/t/ddl/partition.test +++ b/tests/integrationtest/t/ddl/partition.test @@ -259,3 +259,13 @@ set character_set_connection=DEFAULT; -- error 1566 create table a (col1 int, col2 int, unique key (col1, col2)) partition by range columns (col1, col2) (partition p0 values less than (NULL, 1 )); +# TestExchangePartitionAfterDropForeignKey +drop table if exists parent, child, child_with_partition; +create table parent (id int unique); +create table child (id int, parent_id int, foreign key (parent_id) references parent(id)); +create table child_with_partition(id int, parent_id int) partition by range(id) (partition p1 values less than (100)); +-- error 1740 +alter table child_with_partition exchange partition p1 with table child; +alter table child drop foreign key fk_1; +alter table child drop key fk_1; +alter table child_with_partition exchange partition p1 with table child; diff --git a/tests/integrationtest/t/explain_complex_stats.test b/tests/integrationtest/t/explain_complex_stats.test index 84a87eb08e818..2e792981109b0 100644 --- a/tests/integrationtest/t/explain_complex_stats.test +++ b/tests/integrationtest/t/explain_complex_stats.test @@ -120,6 +120,17 @@ CREATE TABLE rr ( ); load stats 's/explain_complex_stats_rr.json'; +# The following ones doesn't have its column stats. +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dt' and column_name = 'cm'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'gad' and column_name = 't'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dd' and column_name = 'ip'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'dd' and column_name = 't'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'sdk' and column_name = 't'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'st' and column_name = 't'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'uid'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'ppt'; +show stats_histograms where db_name = 'explain_complext_stats' and table_name = 'pp' and column_name = 'ps'; + explain format = 'brief' SELECT ds, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(dic) as install_device FROM dt use index (cm) WHERE (ds >= '2016-09-01') AND (ds <= '2016-11-03') AND (cm IN ('1062', '1086', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1709', '1719', '1720', '1843', '2813', '2814', '2815', '2816', '2817', '2818', '2819', '2820', '2821', '2822', '2823', '2824', '2825', '2826', '2827', '2828', '2829', '2830', '2831', '2832', '2833', '2834', '2835', '2836', '2837', '2838', '2839', '2840', '2841', '2842', '2843', '2844', '2845', '2846', '2847', '2848', '2849', '2850', '2851', '2852', '2853', '2854', '2855', '2856', '2857', '2858', '2859', '2860', '2861', '2862', '2863', '2864', '2865', '2866', '2867', '2868', '2869', '2870', '2871', '2872', '3139', '3140', '3141', '3142', '3143', '3144', '3145', '3146', '3147', '3148', '3149', '3150', '3151', '3152', '3153', '3154', '3155', '3156', '3157', '3158', '3386', '3387', '3388', '3389', '3390', '3391', '3392', '3393', '3394', '3395', '3664', '3665', '3666', '3667', '3668', '3670', '3671', '3672', '3673', '3674', '3676', '3677', '3678', '3679', '3680', '3681', '3682', '3683', '3684', '3685', '3686', '3687', '3688', '3689', '3690', '3691', '3692', '3693', '3694', '3695', '3696', '3697', '3698', '3699', '3700', '3701', '3702', '3703', '3704', '3705', '3706', '3707', '3708', '3709', '3710', '3711', '3712', '3713', '3714', '3715', '3960', '3961', '3962', '3963', '3964', '3965', '3966', '3967', '3968', '3978', '3979', '3980', '3981', '3982', '3983', '3984', '3985', '3986', '3987', '4208', '4209', '4210', '4211', '4212', '4304', '4305', '4306', '4307', '4308', '4866', '4867', '4868', '4869', '4870', '4871', '4872', '4873', '4874', '4875')) GROUP BY ds, p1, p2, p3, p4, p5, p6_md5, p7_md5 ORDER BY ds2 DESC; explain format = 'brief' select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext, gad.t as gtime from st gad join (select id, aid, pt, dic, ip, t from dd where pt = 'android' and bm = 0 and t > 1478143908) sdk on gad.aid = sdk.aid and gad.ip = sdk.ip and sdk.t > gad.t where gad.t > 1478143908 and gad.bm = 0 and gad.pt = 'android' group by gad.aid, sdk.dic limit 2500; diff --git a/tests/integrationtest/t/explain_easy_stats.test b/tests/integrationtest/t/explain_easy_stats.test index ddfb8e419ef94..e8ebdba4aaeab 100644 --- a/tests/integrationtest/t/explain_easy_stats.test +++ b/tests/integrationtest/t/explain_easy_stats.test @@ -15,6 +15,7 @@ set @@session.tidb_hashagg_partial_concurrency = 1; set @@session.tidb_hashagg_final_concurrency = 1; +# t1 has no column stats for c2 and t2 has no column stats for c1; explain format = 'brief' select * from t3 where exists (select s.a from t3 s having sum(s.a) = t3.a ); explain format = 'brief' select * from t1; explain format = 'brief' select * from t1 order by c2; diff --git a/tests/integrationtest/t/expression/json.test b/tests/integrationtest/t/expression/json.test index 005b3754d2553..714aab87f2edd 100644 --- a/tests/integrationtest/t/expression/json.test +++ b/tests/integrationtest/t/expression/json.test @@ -380,3 +380,16 @@ insert into t values ('"18446744073709552000"'); select a, cast(a as unsigned) from t; -- sorted_result select a, cast(a as signed) from t; + +# TestCastBinaryStringToJSON +select cast(binary 'aa' as json); +drop table if exists t; +create table t (vb VARBINARY(10), b BINARY(10), vc VARCHAR(10), c CHAR(10)); +insert into t values ('1', '1', '1', '1'); +select cast(vb as json), cast(b as json), cast(vc as json), cast(c as json) from t; +select 1 from t where cast(vb as json) = '1'; +select 1 from t where cast(b as json) = '1'; +select 1 from t where cast(vc as json) = '1'; +select 1 from t where cast(c as json) = '1'; +select 1 from t where cast(BINARY vc as json) = '1'; +select 1 from t where cast(BINARY c as json) = '1'; diff --git a/tests/integrationtest/t/infoschema/v2.test b/tests/integrationtest/t/infoschema/v2.test new file mode 100644 index 0000000000000..384c311e6f865 --- /dev/null +++ b/tests/integrationtest/t/infoschema/v2.test @@ -0,0 +1,16 @@ +set @@global.tidb_schema_cache_size = 1024; + +# TestRenameTable +use infoschema__v2; +drop table if exists t1; +create table t1 (id int); +rename table t1 to t2; +show tables; +select * from t2; +-- error 1146 +select * from t1; +-- error 1146 +show create table t1; + + +set @@global.tidb_schema_cache_size = default; \ No newline at end of file diff --git a/tests/integrationtest/t/planner/core/indexmerge_path.test b/tests/integrationtest/t/planner/core/indexmerge_path.test index 5d71c82f75e60..32b51d4958a94 100644 --- a/tests/integrationtest/t/planner/core/indexmerge_path.test +++ b/tests/integrationtest/t/planner/core/indexmerge_path.test @@ -263,3 +263,118 @@ select * from t force index(mvi) where isnull(i) or json_contains(j, '1'); create table t1 (j json, a bigint(20), b int, primary key(a,b), key mvi((cast(j as unsigned array)))); explain format=brief select /*+ use_index_merge(t1, mvi, primary) */ * from t1 where a = 1 or json_contains(j, '1'); explain format=brief select /*+ use_index_merge(t1, mvi, primary) */ * from t1 where (a = 1 and b = 2) or json_contains(j, '1'); + +# TestExpandANDListWithNestedORList +drop table if exists t, t1; +create table t (a int, b varchar(30), c float, j json, pk int primary key, +key mvi1(c, (cast(j->'$.a' as unsigned array)), b), +key mvi2(a, (cast(j->'$.c' as unsigned array))), +key mvi3((cast(j->'$.d' as unsigned array)), c), +key idx(b, c) +); + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + json_overlaps(j->'$.a', '[4,5,6]') or + (2 member of (j->'$.a')) +) and +c = 10 and +a = 20; + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + c = 1 or + c = 2 or + c = 3 +) and +json_overlaps(j->'$.a', '[4,5,6]'); + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + c = 1 or + c = 2 or + c = 3 +) and +(json_contains(j->'$.a', '[4,5,6]')); + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + c = 1 or + c = 2 or + c = 3 +) and +json_contains(j->'$.a', '[2]'); + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + 1 member of (j->'$.a') or + 2 member of (j->'$.a') +) and +c = 10 and +a = 20; + +explain format=brief select /*+ use_index_merge(t, mvi1) */ * from t where +( + 1 member of (j->'$.a') or + 2 member of (j->'$.d') +) and +a = 20; + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi3) */ * from t where +c = 5 and +( + 1 member of (j->'$.a') or + 2 member of (j->'$.d') +) and +a = 20; + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, primary) */ * from t where +( + pk = 2 or + json_overlaps(j->'$.a', '[4,5,6]') or + json_contains(j->'$.c', '[3]') +) and +a = 1 and +b = '2' and +c = 3; + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, primary) */ * from t where +a = 1 and +b = '2' and +c = 3 and +( + pk = 2 or + (3 member of (j->'$.a')) or + (3 member of (j->'$.c')) +); + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b = '2' and +( + c = 20 or + (c = 10 and 3 member of (j->'$.a')) or + 3 member of (j->'$.c') +); + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +(json_overlaps(j->'$.a', '[4,5,6]')) and +( + (b = '2' and c > 20) or + c = 10 or + 3 member of (j->'$.c') +); + +explain format=brief select /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +(json_overlaps(j->'$.a', '[4,5,6]')) and +( + (b > '2' and c = 20) or + (c = 10) or + (3 member of (j->'$.c')) +); + +create table t1 (a int, b int, c int, d int, j json, key kb(b, (cast(j as unsigned array))), key(d, c)); +explain format=brief select * from t1 where (c=1 or b=1) and (1 member of (j)); +explain format=brief select * from t1 where (c=1 or b=1) and (1 member of (j)) and d=1; diff --git a/tests/integrationtest/t/planner/core/integration.test b/tests/integrationtest/t/planner/core/integration.test index e8ac4efeac49d..7dbd75208f0ab 100644 --- a/tests/integrationtest/t/planner/core/integration.test +++ b/tests/integrationtest/t/planner/core/integration.test @@ -234,31 +234,6 @@ INSERT INTO t0(c0) VALUES (3); SELECT /*+ MERGE_JOIN(t1, t0, v0)*/v0.c2, t1.c0 FROM v0, t0 CROSS JOIN t1 ORDER BY -v0.c1; -# TestInvisibleIndex -drop table if exists t; -create table t(a int, b int, unique index i_a (a) invisible, unique index i_b(b)); -insert into t values (1,2); -admin check table t; -select a from t order by a; -explain select a from t order by a; -select a from t where a > 0; -explain select a from t where a > 1; ---error 1176 -select * from t use index(i_a); ---error 1176 -select * from t force index(i_a); ---error 1176 -select * from t ignore index(i_a); ---enable_warnings -select /*+ USE_INDEX(t, i_a) */ * from t; -select /*+ IGNORE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a; -select /*+ FORCE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a; -select /*+ FORCE_INDEX(aaa) */ * from t; ---disable_warnings -admin check table t; -admin check index t i_a; - - # TestTopNByConstFunc select max(t.col) from (select 'a' as col union all select '' as col) as t; diff --git a/tests/integrationtest/t/planner/core/issuetest/planner_issue.test b/tests/integrationtest/t/planner/core/issuetest/planner_issue.test index f41f312cd82e4..1c461f5616784 100644 --- a/tests/integrationtest/t/planner/core/issuetest/planner_issue.test +++ b/tests/integrationtest/t/planner/core/issuetest/planner_issue.test @@ -186,3 +186,29 @@ where test2.id in select * from v1 ); select * from test2; + +# https://github.com/pingcap/tidb/issues/51560 +create table A(a int primary key, b int); +create table B(b int primary key); +create table C(c int primary key, b int); + +insert into A values (2, 1), (3, 2); +insert into B values (1), (2); + +# Returns data as expected +select b.b +from A a +left join ( + B b + left join C c on b.b = c.b) +on b.b = a.b +where a.a in (2, 3); + +# Returns the same. +select b.b +from A a +left join ( + B b + left join C c on b.b = c.b) +on b.b = a.b +where a.a in (2, 3, null); diff --git a/tests/integrationtest/t/table/index.test b/tests/integrationtest/t/table/index.test index 3d760a3a7dbf6..0eb979f46998b 100644 --- a/tests/integrationtest/t/table/index.test +++ b/tests/integrationtest/t/table/index.test @@ -33,3 +33,15 @@ update t set a = 'qcc' where a = 'rcc'; --enable_warnings; update ignore t set a = 'qcc' where a = 'rcc'; --disable_warnings; + +# Test Issue 48295. +drop table if exists t; +create table t (id int, a varchar(64), b varchar(64), c varchar(64), index idx_a(a(64))); +show create table t; +alter table t add index idx_b(b(64)); +show create table t; +alter table t add index idx_c(c(32)); +show create table t; +alter table t modify column c varchar(32); +show create table t; +drop table t; diff --git a/tests/realtikvtest/importintotest/import_into_test.go b/tests/realtikvtest/importintotest/import_into_test.go index 4b600debe5a30..8b8114773bb81 100644 --- a/tests/realtikvtest/importintotest/import_into_test.go +++ b/tests/realtikvtest/importintotest/import_into_test.go @@ -44,7 +44,6 @@ import ( "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/testkit" - "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" "github.com/pingcap/tidb/pkg/util/sem" "github.com/stretchr/testify/require" @@ -958,11 +957,6 @@ func (s *mockGCSSuite) TestRegisterTask() { }() // wait for the task to be registered <-importinto.TestSyncChan - // cannot run 2 import job at the same time - tk2 := testkit.NewTestKit(s.T(), s.store) - err = tk2.QueryToErr(sql) - s.ErrorIs(err, exeerrors.ErrLoadDataPreCheckFailed) - s.ErrorContains(err, "there's pending or running jobs") client, err := importer.GetEtcdClient() s.NoError(err) diff --git a/tests/realtikvtest/importintotest4/global_sort_test.go b/tests/realtikvtest/importintotest4/global_sort_test.go index 6e212655e9075..1c7d898d4d0b8 100644 --- a/tests/realtikvtest/importintotest4/global_sort_test.go +++ b/tests/realtikvtest/importintotest4/global_sort_test.go @@ -149,7 +149,7 @@ func (s *mockGCSSuite) TestGlobalSortMultiFiles() { // 1 subtask, encoding 10 files using 4 threads. sortStorageURI := fmt.Sprintf("gs://sorted/gs_multi_files?endpoint=%s", gcsEndpoint) importSQL := fmt.Sprintf(`import into t FROM 'gs://gs-multi-files/t.*.csv?endpoint=%s' - with thread=4, cloud_storage_uri='%s'`, gcsEndpoint, sortStorageURI) + with cloud_storage_uri='%s'`, gcsEndpoint, sortStorageURI) s.tk.MustQuery(importSQL) s.tk.MustQuery("select * from t").Sort().Check(testkit.Rows(allData...)) } diff --git a/tests/realtikvtest/importintotest4/main_test.go b/tests/realtikvtest/importintotest4/main_test.go index 4e77eda549e9d..575a824826ac7 100644 --- a/tests/realtikvtest/importintotest4/main_test.go +++ b/tests/realtikvtest/importintotest4/main_test.go @@ -49,6 +49,7 @@ func TestImportInto(t *testing.T) { } func (s *mockGCSSuite) SetupSuite() { + testkit.EnableFailPoint(s.T(), "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", `return(32)`) s.Require().True(*realtikvtest.WithRealTiKV) testutil.ReduceCheckInterval(s.T()) var err error diff --git a/tests/realtikvtest/pipelineddmltest/BUILD.bazel b/tests/realtikvtest/pipelineddmltest/BUILD.bazel index 6a51a7e57dcb4..18c9dd1b214a7 100644 --- a/tests/realtikvtest/pipelineddmltest/BUILD.bazel +++ b/tests/realtikvtest/pipelineddmltest/BUILD.bazel @@ -13,6 +13,7 @@ go_test( "//pkg/config", "//pkg/kv", "//pkg/sessionctx/binloginfo", + "//pkg/sessionctx/variable", "//pkg/testkit", "//tests/realtikvtest", "@com_github_pingcap_failpoint//:failpoint", diff --git a/tests/realtikvtest/pipelineddmltest/pipelineddml_test.go b/tests/realtikvtest/pipelineddmltest/pipelineddml_test.go index 00241ce8dcc61..bd3b51789d6d2 100644 --- a/tests/realtikvtest/pipelineddmltest/pipelineddml_test.go +++ b/tests/realtikvtest/pipelineddmltest/pipelineddml_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/sessionctx/binloginfo" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/tests/realtikvtest" "github.com/stretchr/testify/require" @@ -47,7 +48,7 @@ func TestVariable(t *testing.T) { // We limit this feature only for cases meet all the following conditions: // 1. tidb_dml_type is set to bulk for the current session // 2. the session is running an auto-commit txn -// 3. pessimistic-auto-commit is turned off +// 3. (removed, it is now overridden by tidb_dml_type=bulk) pessimistic-auto-commit ~~if off~~ // 4. binlog is disabled // 5. the statement is not running inside a transaction // 6. the session is external used @@ -55,7 +56,7 @@ func TestVariable(t *testing.T) { func TestPipelinedDMLPositive(t *testing.T) { // the test is a little tricky, only when pipelined dml is enabled, the failpoint panics and the panic message will be returned as error // TODO: maybe save the pipelined DML usage into TxnInfo, so we can check from it. - require.NoError(t, failpoint.Enable("tikvclient/pipelinedCommitFail", `panic("pipelined memdb is be enabled")`)) + require.NoError(t, failpoint.Enable("tikvclient/pipelinedCommitFail", `panic("pipelined memdb is enabled")`)) defer func() { require.NoError(t, failpoint.Disable("tikvclient/pipelinedCommitFail")) }() @@ -83,7 +84,7 @@ func TestPipelinedDMLPositive(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("drop table if exists t") - tk.MustExec("create table t (a int primary key, b int)") + tk.MustExec("create table t (a int, b int)") tk.MustExec("insert into t values(1, 1)") tk.MustExec("set session tidb_dml_type = bulk") for _, stmt := range stmts { @@ -93,7 +94,7 @@ func TestPipelinedDMLPositive(t *testing.T) { return err }) require.Error(t, err, stmt) - require.True(t, strings.Contains(err.Error(), "pipelined memdb is be enabled"), err.Error(), stmt) + require.True(t, strings.Contains(err.Error(), "pipelined memdb is enabled"), err.Error(), stmt) // binary protocol ctx := context.Background() parsedStmts, err := tk.Session().Parse(ctx, stmt) @@ -103,8 +104,38 @@ func TestPipelinedDMLPositive(t *testing.T) { return err }) require.Error(t, err, stmt) - require.True(t, strings.Contains(err.Error(), "pipelined memdb is be enabled"), err.Error(), stmt) + require.True(t, strings.Contains(err.Error(), "pipelined memdb is enabled"), err.Error(), stmt) } + + // pessimistic-auto-commit is on + origPessimisticAutoCommit := config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() + config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(true) + defer func() { + config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(origPessimisticAutoCommit) + }() + err := panicToErr( + func() error { + _, err := tk.Exec("insert into t values(3, 3)") + return err + }, + ) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "pipelined memdb is enabled"), err.Error()) + tk.MustQuery("show warnings").CheckContain("pessimistic-auto-commit config is ignored in favor of Pipelined DML") + config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(false) + + // enable by hint + // Hint works for DELETE and UPDATE, but not for INSERT if the hint is in its select clause. + tk.MustExec("set @@tidb_dml_type = standard") + err = panicToErr( + func() error { + _, err := tk.Exec("delete /*+ SET_VAR(tidb_dml_type=bulk) */ from t") + // "insert into t select /*+ SET_VAR(tidb_dml_type=bulk) */ * from t" won't work + return err + }, + ) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "pipelined memdb is enabled"), err.Error()) } func TestPipelinedDMLNegative(t *testing.T) { @@ -127,27 +158,21 @@ func TestPipelinedDMLNegative(t *testing.T) { // not in auto-commit txn tk.MustExec("set session tidb_dml_type = bulk") tk.MustExec("begin") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can only be used for auto-commit INSERT, REPLACE, UPDATE or DELETE. Fallback to standard mode") tk.MustExec("insert into t values(2, 2)") tk.MustExec("commit") - // pessimistic-auto-commit is on - origPessimisticAutoCommit := config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Load() - config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(true) - defer func() { - config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(origPessimisticAutoCommit) - }() - tk.MustExec("insert into t values(3, 3)") - config.GetGlobalConfig().PessimisticTxn.PessimisticAutoCommit.Store(false) - // binlog is enabled tk.Session().GetSessionVars().BinlogClient = binloginfo.MockPumpsClient(&testkit.MockPumpClient{}) tk.MustExec("insert into t values(4, 4)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used with Binlog: BinlogClient != nil. Fallback to standard mode") tk.Session().GetSessionVars().BinlogClient = nil // in a running txn tk.MustExec("set session tidb_dml_type = standard") tk.MustExec("begin") - tk.MustExec("set session tidb_dml_type = bulk") // turn on bulk dml in a txn doesn't effect the current txn. + // turn on bulk dml in a txn doesn't affect the current txn. + tk.MustExec("set session tidb_dml_type = bulk") tk.MustExec("insert into t values(5, 5)") tk.MustExec("commit") @@ -155,9 +180,34 @@ func TestPipelinedDMLNegative(t *testing.T) { tk.Session().GetSessionVars().InRestrictedSQL = true tk.MustExec("insert into t values(6, 6)") tk.Session().GetSessionVars().InRestrictedSQL = false + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used for internal SQL. Fallback to standard mode") // it's a read statement - tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1 1", "2 2", "3 3", "4 4", "5 5", "6 6")) + tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1 1", "2 2", "4 4", "5 5", "6 6")) + + // for deprecated batch-dml + tk.Session().GetSessionVars().BatchDelete = true + tk.Session().GetSessionVars().DMLBatchSize = 1 + variable.EnableBatchDML.Store(true) + tk.MustExec("insert into t values(7, 7)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used with the deprecated Batch DML. Fallback to standard mode") + tk.Session().GetSessionVars().BatchDelete = false + tk.Session().GetSessionVars().DMLBatchSize = 0 + variable.EnableBatchDML.Store(false) + + // for explain and explain analyze + tk.Session().GetSessionVars().BinlogClient = binloginfo.MockPumpsClient(&testkit.MockPumpClient{}) + tk.MustExec("explain insert into t values(8, 8)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used with Binlog: BinlogClient != nil. Fallback to standard mode") + tk.MustExec("explain analyze insert into t values(9, 9)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used with Binlog: BinlogClient != nil. Fallback to standard mode") + tk.Session().GetSessionVars().BinlogClient = nil + + // disable MDL + tk.MustExec("set global tidb_enable_metadata_lock = off") + tk.MustExec("insert into t values(10, 10)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used without Metadata Lock. Fallback to standard mode") + tk.MustExec("set global tidb_enable_metadata_lock = on") } func compareTables(t *testing.T, tk *testkit.TestKit, t1, t2 string) { @@ -254,27 +304,104 @@ func TestPipelinedDMLInsertRPC(t *testing.T) { store := realtikvtest.CreateMockStoreAndSetup(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec("drop table if exists t1") - tk.MustExec("create table t1 (a int, b int, unique index idx(b))") - res := tk.MustQuery("explain analyze insert ignore into t1 values (1,1), (2,2), (3,3), (4,4), (5,5)") - explain := getExplainResult(res) - require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) - // Test with bulk dml. - tk.MustExec("set session tidb_dml_type = bulk") - // Test normal insert. - tk.MustExec("truncate table t1") - res = tk.MustQuery("explain analyze insert into t1 values (1,1), (2,2), (3,3), (4,4), (5,5)") - explain = getExplainResult(res) - // TODO: try to optimize the rpc count, when use bulk dml, insert will send many BufferBatchGet rpc. - require.Regexp(t, "Insert.* insert:.*, rpc:{BufferBatchGet:{num_rpc:10, total_time:.*}}.*", explain) - // Test insert ignore. - tk.MustExec("truncate table t1") - res = tk.MustQuery("explain analyze insert ignore into t1 values (1,1), (2,2), (3,3), (4,4), (5,5)") - explain = getExplainResult(res) - // TODO: try to optimize the rpc count, when use bulk dml, insert ignore will send 5 BufferBatchGet and 1 BatchGet rpc. - // but without bulk dml, it will only use 1 BatchGet rpcs. - require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:5, total_time:.*}}}.*", explain) - require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + tables := []string{ + "create table _t1 (a int, b int)", // no index, auto generated handle + "create table _t1 (a int primary key, b int)", // clustered handle + "create table _t1 (a int, b int, unique index idx(b))", // unique index + "create table _t1 (a int primary key, b int, unique index idx(b))", // clustered handle + unique index + } + for _, table := range tables { + for _, tableSource := range []bool{true, false} { + hasPK := strings.Contains(table, "primary key") + hasUK := strings.Contains(table, "unique index") + tk.MustExec("drop table if exists t1, _t1") + var values string + if tableSource { + tk.MustExec("create table t1 (a int, b int)") + tk.MustExec("insert into t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)") + values = " select * from t1" + } else { + values = " values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)" + } + tk.MustExec(table) + + // Test with standard dml. + tk.MustExec("set session tidb_dml_type = standard") + res := tk.MustQuery("explain analyze insert ignore into _t1" + values) + explain := getExplainResult(res) + if hasPK || hasUK { + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + } else { + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{BatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{BatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + } + + // Test with bulk dml. + tk.MustExec("set session tidb_dml_type = bulk") + + // Test normal insert. + tk.MustExec("truncate table _t1") + res = tk.MustQuery("explain analyze insert into _t1" + values) + explain = getExplainResult(res) + // no BufferBatchGet with lazy check + require.NotRegexp(t, "Insert.* insert:.*, rpc:{BufferBatchGet:{num_rpc:.*, total_time:.*}}.*", explain) + + // Test insert ignore. + tk.MustExec("truncate table _t1") + res = tk.MustQuery("explain analyze insert ignore into _t1" + values) + explain = getExplainResult(res) + // with bulk dml, it will 1 BatchGet and 1 BufferBatchGet RPCs in prefetch phase. + // but no need to prefetch when there are no unique indexes and no primary key. + if hasPK || hasUK { + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + } else { + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + } + // The ignore takes effect now. + res = tk.MustQuery("explain analyze insert ignore into _t1" + values) + explain = getExplainResult(res) + if hasPK || hasUK { + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + } else { + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + } + + // Test insert on duplicate key update. + res = tk.MustQuery("explain analyze insert into _t1 " + values + " on duplicate key update a = values(a) + 5") + explain = getExplainResult(res) + if hasUK { + // 2 rounds checks are required: read handles by unique keys and read rows by handles + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:2, total_time:.*}}}.*", explain) + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:2, total_time:.*}}}.*", explain) + } else if hasPK { + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + require.Regexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:1, total_time:.*}}}.*", explain) + } else { + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BufferBatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + require.NotRegexp(t, "Insert.* check_insert: {total_time: .* rpc:{.*BatchGet:{num_rpc:.*, total_time:.*}}}.*", explain) + } + + // Test replace into. replace checks in the same way with insert on duplicate key update. + // However, the format of explain result is little different. + res = tk.MustQuery("explain analyze replace into _t1" + values) + explain = getExplainResult(res) + if hasUK { + require.Regexp(t, "Insert.* rpc: {.*BufferBatchGet:{num_rpc:2, total_time:.*}}.*", explain) + require.Regexp(t, "Insert.* rpc: {.*BatchGet:{num_rpc:2, total_time:.*}}.*", explain) + } else if hasPK { + require.Regexp(t, "Insert.* rpc: {.*BufferBatchGet:{num_rpc:1, total_time:.*}}.*", explain) + require.Regexp(t, "Insert.* rpc: {.*BatchGet:{num_rpc:1, total_time:.*}}.*", explain) + } else { + require.NotRegexp(t, "Insert.* rpc: {.*BufferBatchGet:{num_rpc:.*, total_time:.*}}.*", explain) + require.NotRegexp(t, "Insert.* rpc: {.*BatchGet:{num_rpc:.*, total_time:.*}}.*", explain) + } + } + } } func getExplainResult(res *testkit.Result) string { @@ -453,7 +580,6 @@ func TestPipelinedDMLInsertMemoryTest(t *testing.T) { store := realtikvtest.CreateMockStoreAndSetup(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") - tk.MustExec("drop table if exists t1, _t1") tk.MustExec("create table t1 (a int, b int, c varchar(128), unique index idx(b))") tk.MustExec("create table _t1 like t1") @@ -512,7 +638,6 @@ func TestPipelinedDMLDisableRetry(t *testing.T) { tk2 := testkit.NewTestKit(t, store) tk1.MustExec("use test") tk2.MustExec("use test") - tk1.MustExec("drop table if exists t1") tk1.MustExec("create table t1(a int primary key, b int)") tk1.MustExec("insert into t1 values(1, 1)") @@ -529,3 +654,134 @@ func TestPipelinedDMLDisableRetry(t *testing.T) { require.Error(t, err) require.True(t, kv.ErrWriteConflict.Equal(err), fmt.Sprintf("error: %s", err)) } + +func TestReplaceRowCheck(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk1 := testkit.NewTestKit(t, store) + tk1.MustExec("use test") + tk1.MustExec("drop table if exists t1, _t1") + tk1.MustExec("create table t1(a int, b int)") + tk1.MustExec("create table _t1(a int primary key, b int)") + tk1.MustExec("insert into t1 values(1, 1), (2, 2), (1, 2), (2, 1)") + tk1.MustExec("set session tidb_dml_type = bulk") + tk1.MustExec("replace into _t1 select * from t1") + tk1.MustExec("admin check table _t1") + tk1.MustQuery("select a from _t1").Sort().Check(testkit.Rows("1", "2")) + + tk1.MustExec("truncate table _t1") + tk1.MustExec("insert ignore into _t1 select * from t1") + tk1.MustExec("admin check table _t1") + tk1.MustQuery("select a from _t1").Sort().Check(testkit.Rows("1", "2")) + + tk1.MustExec("truncate table _t1") + tk1.MustExec("insert into _t1 select * from t1 on duplicate key update b = values(b)") + tk1.MustExec("admin check table _t1") + tk1.MustQuery("select a from _t1").Sort().Check(testkit.Rows("1", "2")) +} + +func TestDuplicateKeyErrorMessage(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(a int primary key, b int)") + tk.MustExec("insert into t1 values(1, 1)") + err1 := tk.ExecToErr("insert into t1 values(1, 1)") + require.Error(t, err1) + tk.MustExec("set session tidb_dml_type = bulk") + err2 := tk.ExecToErr("insert into t1 values(1, 1)") + require.Error(t, err2) + require.Equal(t, err1.Error(), err2.Error()) +} + +func TestInsertIgnoreOnDuplicateKeyUpdate(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set session tidb_dml_type = bulk") + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(a int, b int, unique index u1(a, b), unique index u2(a))") + tk.MustExec("insert into t1 values(0, 0), (1, 1)") + tk.MustExec("insert ignore into t1 values (0, 2) ,(1, 3) on duplicate key update b = 5, a = 0") + // if the statement execute successful, the following check should pass. + tk.MustQuery("select * from t1").Sort().Check(testkit.Rows("0 5", "1 1")) +} + +func TestConflictError(t *testing.T) { + require.Nil(t, failpoint.Enable("tikvclient/pipelinedMemDBMinFlushKeys", `return(10)`)) + require.Nil(t, failpoint.Enable("tikvclient/pipelinedMemDBMinFlushSize", `return(128)`)) + require.Nil(t, failpoint.Enable("tikvclient/pipelinedMemDBForceFlushSizeThreshold", `return(128)`)) + defer func() { + require.Nil(t, failpoint.Disable("tikvclient/pipelinedMemDBMinFlushKeys")) + require.Nil(t, failpoint.Disable("tikvclient/pipelinedMemDBMinFlushSize")) + require.Nil(t, failpoint.Disable("tikvclient/pipelinedMemDBForceFlushSizeThreshold")) + }() + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, _t1") + tk.MustExec("create table t1(a int primary key, b int)") + tk.MustExec("create table _t1(a int primary key, b int)") + var insert strings.Builder + insert.WriteString("insert into t1 values") + for i := 0; i < 100; i++ { + if i > 0 { + insert.WriteString(",") + } + insert.WriteString(fmt.Sprintf("(%d, %d)", i, i)) + } + tk.MustExec(insert.String()) + tk.MustExec("set session tidb_dml_type = bulk") + tk.MustExec("insert into _t1 select * from t1") + tk.MustExec("set session tidb_max_chunk_size = 32") + err := tk.ExecToErr("insert into _t1 select * from t1 order by rand()") + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "Duplicate entry"), err.Error()) +} + +func TestRejectUnsupportedTables(t *testing.T) { + store := realtikvtest.CreateMockStoreAndSetup(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set session tidb_dml_type = bulk") + + // FKs are not supported if foreign_key_checks=ON + tk.MustExec("drop table if exists parent, child") + tk.MustExec("create table parent(a int primary key)") + tk.MustExec("create table child(a int, foreign key (a) references parent(a))") + err := tk.ExecToErr("insert into parent values(1)") + require.NoError(t, err) + err = tk.ExecToErr("insert into child values(1)") + require.NoError(t, err) + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on table with foreign keys when foreign_key_checks = ON. Fallback to standard mode") + err = tk.ExecToErr("insert into child values(2)") + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "foreign key constraint fails"), err.Error()) + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on table with foreign keys when foreign_key_checks = ON. Fallback to standard mode") + + // test a delete sql that deletes two tables + tk.MustExec("insert into parent values(2)") + tk.MustExec("delete parent, child from parent left join child on parent.a = child.a") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on table with foreign keys when foreign_key_checks = ON. Fallback to standard mode") + + // swap the order of the two tables + tk.MustExec("insert into parent values(3)") + tk.MustExec("delete child, parent from child left join parent on parent.a = child.a") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on table with foreign keys when foreign_key_checks = ON. Fallback to standard mode") + + tk.MustExec("set @@foreign_key_checks=false") + tk.MustExec("insert into parent values(4)") + tk.MustExec("insert into child values(4)") + tk.MustQuery("show warnings").Check(testkit.Rows()) + + // temp tables are not supported + tk.MustExec("create temporary table temp(a int)") + tk.MustExec("insert into temp values(1)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on temporary tables. Fallback to standard mode") + + // cached tables are not supported + tk.MustExec("create table cached(a int)") + tk.MustExec("alter table cached cache") + tk.MustExec("insert into cached values(1)") + tk.MustQuery("show warnings").CheckContain("Pipelined DML can not be used on cached tables. Fallback to standard mode") +} diff --git a/tests/realtikvtest/txntest/BUILD.bazel b/tests/realtikvtest/txntest/BUILD.bazel index 05ef8ea59fd30..bd5b9d43a01f2 100644 --- a/tests/realtikvtest/txntest/BUILD.bazel +++ b/tests/realtikvtest/txntest/BUILD.bazel @@ -12,6 +12,8 @@ go_test( flaky = True, race = "on", deps = [ + "//pkg/config", + "//pkg/errno", "//pkg/expression", "//pkg/kv", "//pkg/parser", @@ -20,8 +22,10 @@ go_test( "//pkg/testkit", "//pkg/util", "//tests/realtikvtest", + "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//tikvrpc", "@io_opencensus_go//stats/view", ], ) diff --git a/tests/realtikvtest/txntest/txn_test.go b/tests/realtikvtest/txntest/txn_test.go index 33f821fc406e7..3ca07862bc420 100644 --- a/tests/realtikvtest/txntest/txn_test.go +++ b/tests/realtikvtest/txntest/txn_test.go @@ -15,16 +15,23 @@ package txntest import ( + "bytes" "context" "fmt" + "strconv" "testing" "time" + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/errno" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/tests/realtikvtest" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikvrpc" ) func TestInTxnPSProtoPointGet(t *testing.T) { @@ -342,3 +349,186 @@ func TestTxnEntrySizeLimit(t *testing.T) { tk1.MustExec("set session tidb_txn_entry_size_limit=0") tk1.MustContainErrMsg("insert into t values (1, repeat('a', 7340032))", "[kv:8025]entry too large, the max entry size is 6291456") } + +func TestCheckTxnStatusOnOptimisticTxnBreakConsistency(t *testing.T) { + // This test case overs the issue #51666 (tikv#16620). + if !*realtikvtest.WithRealTiKV { + t.Skip("skip due to not supporting mock storage") + } + + // Allow async commit + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.TiKVClient.AsyncCommit.SafeWindow = 500 * time.Millisecond + conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0 + }) + + // A helper function to determine whether a KV RPC request is handled on TiKV without RPC error or region error. + isRequestHandled := func(resp *tikvrpc.Response, err error) bool { + if err != nil || resp == nil { + return false + } + + regionErr, err := resp.GetRegionError() + if err != nil || regionErr != nil { + return false + } + + return true + } + + store := realtikvtest.CreateMockStoreAndSetup(t) + tkPrepare1 := testkit.NewTestKit(t, store) + tkPrepare2 := testkit.NewTestKit(t, store) + tk1 := testkit.NewTestKit(t, store) + tk2 := testkit.NewTestKit(t, store) + tkPrepare1.MustExec("use test") + tkPrepare2.MustExec("use test") + tk1.MustExec("use test") + tk2.MustExec("use test") + + tk1.MustExec("create table t (id int primary key, v int)") + tk1.MustExec("insert into t values (1, 10), (2, 20)") + // Table t2 for revealing the possibility that the issue causing data-index inconsistency. + tk1.MustExec("create table t2 (id int primary key, v int unique)") + tk1.MustExec("insert into t2 values (1, 10)") + + tkPrepare1.MustExec("set @@tidb_enable_async_commit = 1") + tk1.MustExec("set @@tidb_enable_async_commit = 0") + + // Prepare a ts collision (currentTxn.StartTS == lastTxn.CommitTS on the same key). + // Loop until we successfully prepare one. + var lastCommitTS uint64 + for constructionIters := 0; ; constructionIters++ { + // Reset the value which might have been updated in the previous attempt. + tkPrepare1.MustExec("update t set v = 10 where id = 1") + + // Update row 1 in async commit mode + require.NoError(t, failpoint.Enable("tikvclient/beforePrewrite", "pause")) + tkPrepapre1Ch := make(chan struct{}) + go func() { + tkPrepare1.MustExec("update t set v = v + 1 where id = 1") + tkPrepapre1Ch <- struct{}{} + }() + + // tkPrepare2 Updates TiKV's max_ts by reading. Assuming tkPrepare2's reading is just before tk1's BEGIN, + // we expect that tk1 have startTS == tkPrepare2.startTS + 1 so that the tk1.startTS == TiKV's min_commit_ts. + tkPrepare2.MustQuery("select * from t where id = 1").Check(testkit.Rows("1 10")) + tk1.MustExec("begin optimistic") + + require.NoError(t, failpoint.Disable("tikvclient/beforePrewrite")) + select { + case <-tkPrepapre1Ch: + case <-time.After(time.Second): + require.Fail(t, "tkPrepare1 not resumed after unsetting failpoint") + } + + var err error + lastCommitTS, err = strconv.ParseUint(tkPrepare1.MustQuery("select json_extract(@@tidb_last_txn_info, '$.commit_ts')").Rows()[0][0].(string), 10, 64) + require.NoError(t, err) + currentStartTS, err := strconv.ParseUint(tk1.MustQuery("select @@tidb_current_ts").Rows()[0][0].(string), 10, 64) + require.NoError(t, err) + if currentStartTS == lastCommitTS { + break + } + // Abandon and retry. + tk1.MustExec("rollback") + if constructionIters >= 1000 { + require.Fail(t, "failed to construct the ts collision situation of async commit transaction") + } + } + + // Now tk1 is in a transaction whose start ts collides with the commit ts of a previously committed transaction + // that has written row 1. The ts is in variable `lastCommitTS`. + + tk1.MustExec("update t set v = v + 100 where id = 1") + tk1.MustExec("update t set v = v + 100 where id = 2") + tk1.MustExec("update t2 set v = v + 1 where id = 1") + + // We will construct the following committing procedure for transaction in tk1: + // 1. Successfully prewrites all keys but fail to receive the response of the request that prewrites the primary + // (by simulating RPC error); + // 2. tk2 tries to access keys that were already locked by tk1, and performs resolve-locks. When the issue exists, + // the primary may be rolled back without any rollback record. + // 3. tk1 continues and retries prewriting the primary. In normal cases, it should not succeed as the transaction + // should have been rolled back by tk2's resolve-locks operation, but it succeeds in the issue. + // To simulate the procedure for tk1's commit procedure, we use the onRPCFinishedHook failpoint, and inject a hook + // when committing that makes the first prewrite on tk1's primary fail, and blocks until signaled by the channel + // `continueCommittingSignalCh`. + + require.NoError(t, failpoint.Enable("tikvclient/twoPCShortLockTTL", "return")) + require.NoError(t, failpoint.Enable("tikvclient/doNotKeepAlive", "return")) + require.NoError(t, failpoint.Enable("tikvclient/twoPCRequestBatchSizeLimit", "return")) + require.NoError(t, failpoint.Enable("tikvclient/onRPCFinishedHook", "return")) + + defer func() { + require.NoError(t, failpoint.Disable("tikvclient/twoPCShortLockTTL")) + require.NoError(t, failpoint.Disable("tikvclient/doNotKeepAlive")) + require.NoError(t, failpoint.Disable("tikvclient/twoPCRequestBatchSizeLimit")) + require.NoError(t, failpoint.Disable("tikvclient/onRPCFinishedHook")) + }() + + continueCommittingSignalCh := make(chan struct{}) + + primaryReqCount := 0 + onRPCFinishedHook := func(req *tikvrpc.Request, resp *tikvrpc.Response, err error) (*tikvrpc.Response, error) { + if req.Type == tikvrpc.CmdPrewrite { + prewriteReq := req.Prewrite() + // The failpoint "twoPCRequestBatchSizeLimit" must takes effect + require.Equal(t, 1, len(prewriteReq.GetMutations())) + if prewriteReq.GetStartVersion() == lastCommitTS && + bytes.Equal(prewriteReq.GetMutations()[0].Key, prewriteReq.PrimaryLock) && + isRequestHandled(resp, err) { + primaryReqCount++ + if primaryReqCount == 1 { + // Block until signaled + <-continueCommittingSignalCh + // Simulate RPC failure (but TiKV successfully handled the request) for the first attempt + return nil, errors.New("injected rpc error in onRPCFinishedHook") + } + } + } + return resp, err + } + + ctxWithHook := context.WithValue(context.Background(), "onRPCFinishedHook", onRPCFinishedHook) + + resCh := make(chan error) + go func() { + _, err := tk1.ExecWithContext(ctxWithHook, "commit") + resCh <- err + }() + // tk1 must be blocked by the hook function. + select { + case err := <-resCh: + require.Fail(t, "tk1 not blocked, result: "+fmt.Sprintf("%+q", err)) + case <-time.After(time.Millisecond * 50): + } + + // tk2 conflicts with tk1 and rolls back tk1 by resolving locks. + tk2.MustExec("update t set v = v + 1 where id = 2") + tk2.MustExec("insert into t2 values (2, 11)") + + // tk1 must still be blocked + select { + case err := <-resCh: + require.Fail(t, "tk1 not blocked, result: "+fmt.Sprintf("%+q", err)) + case <-time.After(time.Millisecond * 50): + } + + // Signal tk1 to continue (retry the prewrite request and continue). + close(continueCommittingSignalCh) + + var err error + select { + case err = <-resCh: + case <-time.After(time.Second): + require.Fail(t, "tk1 not resumed") + } + + require.Error(t, err) + require.Equal(t, errno.ErrWriteConflict, int(errors.Cause(err).(*errors.Error).Code())) + tk2.MustQuery("select * from t order by id").Check(testkit.Rows("1 11", "2 21")) + tk2.MustExec("admin check table t2") + tk2.MustQuery("select * from t2 order by id").Check(testkit.Rows("1 10", "2 11")) +}