From f15a02b192384899c4157c5bfd8ce17e564a5956 Mon Sep 17 00:00:00 2001 From: Grace Cai Date: Fri, 2 Jun 2023 12:16:42 +0800 Subject: [PATCH] This is an automated cherry-pick of #13692 Signed-off-by: ti-chi-bot --- TOC.md | 2 +- _docHome.md | 2 +- br/br-pitr-guide.md | 2 +- clinic/clinic-user-guide-for-tiup.md | 2 +- develop/dev-guide-aws-appflow-integration.md | 10 +- develop/dev-guide-build-cluster-in-cloud.md | 28 +- develop/dev-guide-create-database.md | 2 +- develop/dev-guide-create-secondary-indexes.md | 2 +- develop/dev-guide-create-table.md | 4 +- develop/dev-guide-delete-data.md | 2 +- develop/dev-guide-insert-data.md | 2 +- develop/dev-guide-outdated-for-django.md | 2 +- develop/dev-guide-proxysql-integration.md | 18 +- ...ev-guide-sample-application-golang-gorm.md | 326 ++++++++ .../dev-guide-sample-application-golang.md | 10 +- ...guide-sample-application-java-hibernate.md | 426 ++++++++++ .../dev-guide-sample-application-java-jdbc.md | 591 ++++++++++++++ ...v-guide-sample-application-java-mybatis.md | 741 ++++++++++++++++++ develop/dev-guide-sample-application-java.md | 16 +- ...mple-application-python-mysql-connector.md | 291 +++++++ ...e-sample-application-python-mysqlclient.md | 292 +++++++ ...-guide-sample-application-python-peewee.md | 255 ++++++ ...guide-sample-application-python-pymysql.md | 287 +++++++ ...de-sample-application-python-sqlalchemy.md | 249 ++++++ ...ev-guide-sample-application-spring-boot.md | 8 +- develop/dev-guide-tidb-crud-sql.md | 2 +- develop/dev-guide-update-data.md | 2 +- encryption-at-rest.md | 2 +- explore-htap.md | 2 +- garbage-collection-configuration.md | 4 +- .../information-schema-resource-groups.md | 2 +- .../information-schema-slow-query.md | 2 +- .../information-schema-user-privileges.md | 48 ++ quick-start-with-tidb.md | 2 +- releases/release-5.2.0.md | 6 +- releases/release-6.0.0-dmr.md | 4 +- releases/release-7.0.0.md | 510 ++++++++++++ releases/release-7.1.0.md | 550 +++++++++++++ .../sql-statement-alter-resource-group.md | 2 +- .../sql-statement-calibrate-resource.md | 123 +++ .../sql-statement-create-resource-group.md | 2 +- .../sql-statement-drop-resource-group.md | 2 +- .../sql-statement-flashback-to-timestamp.md | 2 +- sql-statements/sql-statement-load-data.md | 20 + .../sql-statement-set-resource-group.md | 96 +++ ...ql-statement-show-create-resource-group.md | 2 +- statement-summary-tables.md | 4 +- statistics.md | 2 +- system-variables.md | 4 +- tidb-distributed-execution-framework.md | 124 +++ tidb-resource-control.md | 62 +- time-to-live.md | 7 +- 52 files changed, 5077 insertions(+), 81 deletions(-) create mode 100644 develop/dev-guide-sample-application-golang-gorm.md create mode 100644 develop/dev-guide-sample-application-java-hibernate.md create mode 100644 develop/dev-guide-sample-application-java-jdbc.md create mode 100644 develop/dev-guide-sample-application-java-mybatis.md create mode 100644 develop/dev-guide-sample-application-python-mysql-connector.md create mode 100644 develop/dev-guide-sample-application-python-mysqlclient.md create mode 100644 develop/dev-guide-sample-application-python-peewee.md create mode 100644 develop/dev-guide-sample-application-python-pymysql.md create mode 100644 develop/dev-guide-sample-application-python-sqlalchemy.md create mode 100644 releases/release-7.0.0.md create mode 100644 releases/release-7.1.0.md create mode 100644 sql-statements/sql-statement-calibrate-resource.md create mode 100644 sql-statements/sql-statement-set-resource-group.md create mode 100644 tidb-distributed-execution-framework.md diff --git a/TOC.md b/TOC.md index 56390dc72cb32..81c84e30e1dd2 100644 --- a/TOC.md +++ b/TOC.md @@ -19,7 +19,7 @@ - Develop - [Overview](/develop/dev-guide-overview.md) - Quick Start - - [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md) + - [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) - Example Applications - [Golang](/develop/dev-guide-sample-application-golang.md) diff --git a/_docHome.md b/_docHome.md index 3d9a36079c623..665d6764677c5 100644 --- a/_docHome.md +++ b/_docHome.md @@ -49,7 +49,7 @@ Get the power of a cloud-native, distributed SQL database built for real-time an --> -TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB on premises or in the cloud. +TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB in a self-hosted environment or in the cloud. diff --git a/br/br-pitr-guide.md b/br/br-pitr-guide.md index 90c6fc6e46a55..903f2ff625723 100644 --- a/br/br-pitr-guide.md +++ b/br/br-pitr-guide.md @@ -123,7 +123,7 @@ Testing scenario 1 (on [TiDB Cloud](https://tidbcloud.com)): - New log data created in the cluster: 10 GB/h - Write (INSERT/UPDATE/DELETE) QPS: 10,000 -Testing scenario 2 (on-premises): +Testing scenario 2 (on TiDB Self-Hosted): - The number of TiKV nodes (8 core, 64 GB memory): 6 - The number of Regions: 50,000 diff --git a/clinic/clinic-user-guide-for-tiup.md b/clinic/clinic-user-guide-for-tiup.md index 7652a5ffffea5..7ac8b6f56205e 100644 --- a/clinic/clinic-user-guide-for-tiup.md +++ b/clinic/clinic-user-guide-for-tiup.md @@ -9,7 +9,7 @@ For TiDB clusters and DM clusters deployed using TiUP, you can use PingCAP Clini > **Note:** > -> - This document **only** applies to clusters deployed using TiUP in an on-premises environment. For clusters deployed using TiDB Operator on Kubernetes, see [PingCAP Clinic for TiDB Operator environments](https://docs.pingcap.com/tidb-in-kubernetes/stable/clinic-user-guide). +> - This document **only** applies to clusters deployed using TiUP in a self-hosted environment. For clusters deployed using TiDB Operator on Kubernetes, see [PingCAP Clinic for TiDB Operator environments](https://docs.pingcap.com/tidb-in-kubernetes/stable/clinic-user-guide). > > - PingCAP Clinic **does not support** collecting data from clusters deployed using TiDB Ansible. diff --git a/develop/dev-guide-aws-appflow-integration.md b/develop/dev-guide-aws-appflow-integration.md index 608a3c30781ba..1b2ff2a604b4c 100644 --- a/develop/dev-guide-aws-appflow-integration.md +++ b/develop/dev-guide-aws-appflow-integration.md @@ -7,9 +7,9 @@ summary: Introduce how to integrate TiDB with Amazon AppFlow step by step. [Amazon AppFlow](https://aws.amazon.com/appflow/) is a fully managed API integration service that you use to connect your software as a service (SaaS) applications to AWS services, and securely transfer data. With Amazon AppFlow, you can import and export data from and to TiDB into many types of data providers, such as Salesforce, Amazon S3, LinkedIn, and GitHub. For more information, see [Supported source and destination applications](https://docs.aws.amazon.com/appflow/latest/userguide/app-specific.html) in AWS documentation. -This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a TiDB Cloud Serverless Tier cluster as an example. +This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a TiDB Serverless cluster as an example. -If you do not have a TiDB cluster, you can create a [Serverless Tier](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. +If you do not have a TiDB cluster, you can create a [TiDB Serverless](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. ## Prerequisites @@ -66,7 +66,7 @@ git clone https://github.com/pingcap-inc/tidb-appflow-integration > > - The `--guided` option uses prompts to guide you through the deployment. Your input will be stored in a configuration file, which is `samconfig.toml` by default. > - `stack_name` specifies the name of AWS Lambda that you are deploying. - > - This prompted guide uses AWS as the cloud provider of TiDB Cloud Serverless Tier. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. + > - This prompted guide uses AWS as the cloud provider of TiDB Serverless. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. > - If you have already run `sam deploy --guided` before, you can just run `sam deploy` instead, and SAM CLI will use the configuration file `samconfig.toml` to simplify the interaction. If you see a similar output as follows, this Lambda is successfully deployed. @@ -148,7 +148,7 @@ Choose the **Source details** and **Destination details**. TiDB connector can be ``` 5. After the `sf_account` table is created, click **Connect**. A connection dialog is displayed. -6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a TiDB Cloud Serverless Tier cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. +6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a TiDB Serverless cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. ![tidb connection message](/media/develop/aws-appflow-step-tidb-connection-message.png) @@ -244,5 +244,5 @@ test> SELECT * FROM sf_account; - If anything goes wrong, you can navigate to the [CloudWatch](https://console.aws.amazon.com/cloudwatch/home) page on the AWS Management Console to get logs. - The steps in this document are based on [Building custom connectors using the Amazon AppFlow Custom Connector SDK](https://aws.amazon.com/blogs/compute/building-custom-connectors-using-the-amazon-appflow-custom-connector-sdk/). -- [TiDB Cloud Serverless Tier](https://docs.pingcap.com/tidbcloud/select-cluster-tier#serverless-tier-beta) is **NOT** a production environment. +- [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta) is **NOT** a production environment. - To prevent excessive length, the examples in this document only show the `Insert` strategy, but `Update` and `Upsert` strategies are also tested and can be used. \ No newline at end of file diff --git a/develop/dev-guide-build-cluster-in-cloud.md b/develop/dev-guide-build-cluster-in-cloud.md index e71393be66a30..3c5d7425c52dd 100644 --- a/develop/dev-guide-build-cluster-in-cloud.md +++ b/develop/dev-guide-build-cluster-in-cloud.md @@ -1,15 +1,15 @@ --- -title: Build a TiDB Cluster in TiDB Cloud (Serverless Tier) -summary: Learn how to build a TiDB cluster in TiDB Cloud (Serverless Tier) and connect to a TiDB Cloud cluster. +title: Build a TiDB Serverless Cluster +summary: Learn how to build a TiDB Serverless cluster in TiDB Cloud and connect to it. --- -# Build a TiDB Cluster in TiDB Cloud (Serverless Tier) +# Build a TiDB Serverless Cluster -This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://en.pingcap.com/tidb-cloud) to create a Serverless Tier cluster, connect to it, and run a sample application on it. +This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://en.pingcap.com/tidb-cloud) to create a TiDB Serverless cluster, connect to it, and run a sample application on it. If you need to run TiDB on your local machine, see [Starting TiDB Locally](/quick-start-with-tidb.md). @@ -21,7 +21,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. -## Step 1. Create a Serverless Tier cluster +## Step 1. Create a TiDB Serverless cluster 1. If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/free-trial) to sign up for an account. @@ -29,9 +29,9 @@ This document walks you through the quickest way to get started with TiDB Cloud. 3. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **Create Cluster**. -4. On the **Create Cluster** page, **Serverless Tier** is selected by default. Update the default cluster name if necessary, and then select the region where you want to create your cluster. +4. On the **Create Cluster** page, **Serverless** is selected by default. Update the default cluster name if necessary, and then select the region where you want to create your cluster. -5. Click **Create** to create a Serverless Tier cluster. +5. Click **Create** to create a TiDB Serverless cluster. Your TiDB Cloud cluster will be created in approximately 30 seconds. @@ -45,7 +45,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. > **Note:** > -> For [Serverless Tier clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#serverless-tier), when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). +> For [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta), when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). @@ -53,7 +53,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. > **Note:** > -> For [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta), when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). +> For [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta), when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). @@ -130,7 +130,7 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 -2. Run the connection string obtained in [Step 1](#step-1-create-a-serverless-tier-cluster). +2. Run the connection string obtained in [Step 1](#step-1-create-a-tidb-serverless-cluster). {{< copyable "shell-regular" >}} @@ -142,8 +142,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a Serverless Tier cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters). -> - If you encounter problems when connecting to a Serverless Tier cluster, you can read [Secure Connections to Serverless Tier Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters) for more information. +> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters). +> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters) for more information. @@ -151,8 +151,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a Serverless Tier cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-tier-clusters.md). -> - If you encounter problems when connecting to a Serverless Tier cluster, you can read [Secure Connections to Serverless Tier Clusters](/tidb-cloud/secure-connections-to-serverless-tier-clusters.md) for more information. +> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-tier-clusters.md). +> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](/tidb-cloud/secure-connections-to-serverless-tier-clusters.md) for more information. diff --git a/develop/dev-guide-create-database.md b/develop/dev-guide-create-database.md index d79fbbb44462f..c9ef93d081af3 100644 --- a/develop/dev-guide-create-database.md +++ b/develop/dev-guide-create-database.md @@ -11,7 +11,7 @@ This document describes how to create a database using SQL and various programmi Before creating a database, do the following: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). ## What is database diff --git a/develop/dev-guide-create-secondary-indexes.md b/develop/dev-guide-create-secondary-indexes.md index 486cc168af422..0e57fd3c6f4a9 100644 --- a/develop/dev-guide-create-secondary-indexes.md +++ b/develop/dev-guide-create-secondary-indexes.md @@ -11,7 +11,7 @@ This document describes how to create a secondary index using SQL and various pr Before creating a secondary index, do the following: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). - [Create a Table](/develop/dev-guide-create-table.md). diff --git a/develop/dev-guide-create-table.md b/develop/dev-guide-create-table.md index 4fc1bc58ba275..1ad1b6dbf7ff4 100644 --- a/develop/dev-guide-create-table.md +++ b/develop/dev-guide-create-table.md @@ -11,7 +11,7 @@ This document introduces how to create tables using the SQL statement and the re Before reading this document, make sure that the following tasks are completed: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). @@ -290,7 +290,7 @@ ALTER TABLE `bookshop`.`ratings` SET TIFLASH REPLICA 1; > **Note:** > -> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster) to create a Serverless Tier cluster that includes **TiFlash**. +> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster that includes **TiFlash**. Then you can go on to perform the following query: diff --git a/develop/dev-guide-delete-data.md b/develop/dev-guide-delete-data.md index 5dfb874bc7566..96278a73313a5 100644 --- a/develop/dev-guide-delete-data.md +++ b/develop/dev-guide-delete-data.md @@ -11,7 +11,7 @@ This document describes how to use the [DELETE](/sql-statements/sql-statement-de Before reading this document, you need to prepare the following: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md) +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) - [Insert Data](/develop/dev-guide-insert-data.md) diff --git a/develop/dev-guide-insert-data.md b/develop/dev-guide-insert-data.md index b95183c995610..94694f824bba5 100644 --- a/develop/dev-guide-insert-data.md +++ b/develop/dev-guide-insert-data.md @@ -13,7 +13,7 @@ This document describes how to insert data into TiDB by using the SQL language w Before reading this document, you need to prepare the following: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) ## Insert rows diff --git a/develop/dev-guide-outdated-for-django.md b/develop/dev-guide-outdated-for-django.md index 7a0f8240cb80e..632acb1ce1759 100644 --- a/develop/dev-guide-outdated-for-django.md +++ b/develop/dev-guide-outdated-for-django.md @@ -25,7 +25,7 @@ The above command starts a temporary and single-node cluster with mock TiKV. The > > To deploy a "real" TiDB cluster for production, see the following guides: > -> + [Deploy TiDB using TiUP for On-Premises](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) +> + [Deploy TiDB using TiUP for Self-Hosted Environment](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) > + [Deploy TiDB on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable) > > You can also [use TiDB Cloud](https://pingcap.com/products/tidbcloud/), a fully-managed Database-as-a-Service (DBaaS) of TiDB. diff --git a/develop/dev-guide-proxysql-integration.md b/develop/dev-guide-proxysql-integration.md index b89fd24ce7d84..e0465d329a0b4 100644 --- a/develop/dev-guide-proxysql-integration.md +++ b/develop/dev-guide-proxysql-integration.md @@ -119,11 +119,11 @@ systemctl start docker ### Option 1: Integrate TiDB Cloud with ProxySQL -For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a TiDB Serverless Tier cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. +For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a TiDB Serverless cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. -#### Step 1. Create a TiDB Cloud Serverless Tier cluster +#### Step 1. Create a TiDB Serverless cluster -1. [Create a free TiDB Serverless Tier cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. +1. [Create a free TiDB Serverless cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. 2. Get your cluster hostname, port, and username for later use. 1. On the [Clusters](https://tidbcloud.com/console/clusters) page, click your cluster name to go to the cluster overview page. @@ -327,12 +327,12 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. > > 1. Adds a user using the username and password of your cluster. > 2. Assigns the user to the monitoring account. - > 3. Adds your TiDB Serverless Tier cluster to the list of hosts. - > 4. Enables a secure connection between ProxySQL and the TiDB Serverless Tier cluster. + > 3. Adds your TiDB Serverless cluster to the list of hosts. + > 4. Enables a secure connection between ProxySQL and the TiDB Serverless cluster. > > To have a better understanding, it is strongly recommended that you check the `proxysql-prepare.sql` file. To learn more about ProxySQL configuration, see [ProxySQL documentation](https://proxysql.com/documentation/proxysql-configuration/). - The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the TiDB Serverless Tier cluster is established. + The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the TiDB Serverless cluster is established. ``` *************************** 1. row *************************** @@ -388,7 +388,7 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. SELECT VERSION(); ``` - If the TiDB version is displayed, you are successfully connected to your TiDB Serverless Tier cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. + If the TiDB version is displayed, you are successfully connected to your TiDB Serverless cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. > **Note:** > @@ -636,7 +636,7 @@ ProxySQL can be installed on many different platforms. The following takes CentO For a full list of supported platforms and the corresponding version requirements, see [ProxySQL documentation](https://proxysql.com/documentation/installing-proxysql/). -#### Step 1. Create a TiDB Cloud Dedicated Tier cluster +#### Step 1. Create a TiDB Dedicated cluster For detailed steps, see [Create a TiDB Cluster](https://docs.pingcap.com/tidbcloud/create-tidb-cluster). @@ -687,7 +687,7 @@ To use ProxySQL as a proxy for TiDB, you need to configure ProxySQL. To do so, y The above step will take you to the ProxySQL admin prompt. -2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Cloud Dedicated Tier cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). +2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Dedicated cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). ```sql INSERT INTO mysql_servers(hostgroup_id, hostname, port) diff --git a/develop/dev-guide-sample-application-golang-gorm.md b/develop/dev-guide-sample-application-golang-gorm.md new file mode 100644 index 0000000000000..95bb8e9d0dfda --- /dev/null +++ b/develop/dev-guide-sample-application-golang-gorm.md @@ -0,0 +1,326 @@ +--- +title: Build a Simple CRUD App with TiDB and GORM +summary: Learn how to build a simple CRUD application with TiDB and GORM. +--- + + + + +# Build a Simple CRUD App with TiDB and GORM + +[GORM](https://gorm.io/) is a popular open-source ORM library for Golang. + +This document describes how to use TiDB and GORM to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Golang 1.16 or a later version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-golang.git +``` + +Compared with GORM, the go-sql-driver/mysql implementation might be not a best practice, because you need to write error handling logic, close `*sql.Rows` manually and cannot reuse code easily, which makes your code slightly redundant. + +The following instructions take `v1.23.5` as an example. + +To adapt TiDB transactions, write a toolkit [util](https://github.com/pingcap-inc/tidb-example-golang/tree/main/util) according to the following code: + +```go +package util + +import ( + "context" + "database/sql" +) + +type TiDBSqlTx struct { + *sql.Tx + conn *sql.Conn + pessimistic bool +} + +func TiDBSqlBegin(db *sql.DB, pessimistic bool) (*TiDBSqlTx, error) { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return nil, err + } + if pessimistic { + _, err = conn.ExecContext(ctx, "set @@tidb_txn_mode=?", "pessimistic") + } else { + _, err = conn.ExecContext(ctx, "set @@tidb_txn_mode=?", "optimistic") + } + if err != nil { + return nil, err + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &TiDBSqlTx{ + conn: conn, + Tx: tx, + pessimistic: pessimistic, + }, nil +} + +func (tx *TiDBSqlTx) Commit() error { + defer tx.conn.Close() + return tx.Tx.Commit() +} + +func (tx *TiDBSqlTx) Rollback() error { + defer tx.conn.Close() + return tx.Tx.Rollback() +} +``` + +Change to the `gorm` directory: + +```shell +cd gorm +``` + +The structure of this directory is as follows: + +``` +. +├── Makefile +├── go.mod +├── go.sum +└── gorm.go +``` + +`gorm.go` is the main body of the `gorm`. Compared with go-sql-driver/mysql, GORM avoids differences in database creation between different databases. It also implements a lot of operations, such as AutoMigrate and CRUD of objects, which greatly simplifies the code. + +`Player` is a data entity struct that is a mapping for tables. Each property of a `Player` corresponds to a field in the `player` table. Compared with go-sql-driver/mysql, `Player` in GORM adds struct tags to indicate mapping relationships for more information, such as `gorm:"primaryKey;type:VARCHAR(36);column:id"`. + +```go + +package main + +import ( + "fmt" + "math/rand" + + "github.com/google/uuid" + "github.com/pingcap-inc/tidb-example-golang/util" + + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +type Player struct { + ID string `gorm:"primaryKey;type:VARCHAR(36);column:id"` + Coins int `gorm:"column:coins"` + Goods int `gorm:"column:goods"` +} + +func (*Player) TableName() string { + return "player" +} + +func main() { + // 1. Configure the example database connection. + db := createDB() + + // AutoMigrate for player table + db.AutoMigrate(&Player{}) + + // 2. Run some simple examples. + simpleExample(db) + + // 3. Explore more. + tradeExample(db) +} + +func tradeExample(db *gorm.DB) { + // Player 1: id is "1", has only 100 coins. + // Player 2: id is "2", has 114514 coins, and 20 goods. + player1 := &Player{ID: "1", Coins: 100} + player2 := &Player{ID: "2", Coins: 114514, Goods: 20} + + // Create two players "by hand", using the INSERT statement on the backend. + db.Clauses(clause.OnConflict{UpdateAll: true}).Create(player1) + db.Clauses(clause.OnConflict{UpdateAll: true}).Create(player2) + + // Player 1 wants to buy 10 goods from player 2. + // It will cost 500 coins, but player 1 cannot afford it. + fmt.Println("\nbuyGoods:\n => this trade will fail") + if err := buyGoods(db, player2.ID, player1.ID, 10, 500); err == nil { + panic("there shouldn't be success") + } + + // So player 1 has to reduce the incoming quantity to two. + fmt.Println("\nbuyGoods:\n => this trade will success") + if err := buyGoods(db, player2.ID, player1.ID, 2, 100); err != nil { + panic(err) + } +} + +func simpleExample(db *gorm.DB) { + // Create a player, who has a coin and a goods. + if err := db.Clauses(clause.OnConflict{UpdateAll: true}). + Create(&Player{ID: "test", Coins: 1, Goods: 1}).Error; err != nil { + panic(err) + } + + // Get a player. + var testPlayer Player + db.Find(&testPlayer, "id = ?", "test") + fmt.Printf("getPlayer: %+v\n", testPlayer) + + // Create players with bulk inserts. Insert 1919 players totally, with 114 players per batch. + bulkInsertPlayers := make([]Player, 1919, 1919) + total, batch := 1919, 114 + for i := 0; i < total; i++ { + bulkInsertPlayers[i] = Player{ + ID: uuid.New().String(), + Coins: rand.Intn(10000), + Goods: rand.Intn(10000), + } + } + + if err := db.Session(&gorm.Session{Logger: db.Logger.LogMode(logger.Error)}). + CreateInBatches(bulkInsertPlayers, batch).Error; err != nil { + panic(err) + } + + // Count players amount. + playersCount := int64(0) + db.Model(&Player{}).Count(&playersCount) + fmt.Printf("countPlayers: %d\n", playersCount) + + // Print 3 players. + threePlayers := make([]Player, 3, 3) + db.Limit(3).Find(&threePlayers) + for index, player := range threePlayers { + fmt.Printf("print %d player: %+v\n", index+1, player) + } +} + +func createDB() *gorm.DB { + dsn := "root:@tcp(127.0.0.1:4000)/test?charset=utf8mb4" + db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Info), + }) + if err != nil { + panic(err) + } + + return db +} + +func buyGoods(db *gorm.DB, sellID, buyID string, amount, price int) error { + return util.TiDBGormBegin(db, true, func(tx *gorm.DB) error { + var sellPlayer, buyPlayer Player + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&sellPlayer, "id = ?", sellID).Error; err != nil { + return err + } + + if sellPlayer.ID != sellID || sellPlayer.Goods < amount { + return fmt.Errorf("sell player %s goods not enough", sellID) + } + + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&buyPlayer, "id = ?", buyID).Error; err != nil { + return err + } + + if buyPlayer.ID != buyID || buyPlayer.Coins < price { + return fmt.Errorf("buy player %s coins not enough", buyID) + } + + updateSQL := "UPDATE player set goods = goods + ?, coins = coins + ? WHERE id = ?" + if err := tx.Exec(updateSQL, -amount, price, sellID).Error; err != nil { + return err + } + + if err := tx.Exec(updateSQL, amount, -price, buyID).Error; err != nil { + return err + } + + fmt.Println("\n[buyGoods]:\n 'trade success'") + return nil + }) +} +``` + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, modify the value of the `dsn` in `gorm.go`: + +```go +dsn := "root:@tcp(127.0.0.1:4000)/test?charset=utf8mb4" +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `mysql.RegisterTLSConfig` and `dsn` as follows: + +```go +mysql.RegisterTLSConfig("register-tidb-tls", &tls.Config { + MinVersion: tls.VersionTLS12, + ServerName: "xxx.tidbcloud.com", +}) + +dsn := "2aEp24QWEDLqRFs.root:123456@tcp(xxx.tidbcloud.com:4000)/test?charset=utf8mb4&tls=register-tidb-tls" +``` + +### Step 3.2 Run the code + +To run the code, you can run `make build` and `make run` respectively: + +```shell +make build # this command executes `go build -o bin/gorm-example` +make run # this command executes `./bin/gorm-example` +``` + +Or you can use the native commands: + +```shell +go build -o bin/gorm-example +./bin/gorm-example +``` + +Or run the `make` command directly, which is a combination of `make build` and `make run`. + +## Step 4. Expected output + +[GORM Expected Output](https://github.com/pingcap-inc/tidb-example-golang/blob/main/Expected-Output.md#gorm) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-golang.md b/develop/dev-guide-sample-application-golang.md index 94d6409a4ffc1..e28abbc7f137e 100644 --- a/develop/dev-guide-sample-application-golang.md +++ b/develop/dev-guide-sample-application-golang.md @@ -20,9 +20,9 @@ This document describes how to use TiDB and Golang to build a simple CRUD applic The following introduces how to start a TiDB cluster. -**Use a TiDB Cloud Serverless Tier cluster** +**Use a TiDB Serverless cluster** -For detailed steps, see [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). **Use a local cluster** @@ -32,7 +32,7 @@ For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md# -See [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). @@ -764,6 +764,7 @@ When using go-sql-driver/mysql, you need to connect to your cluster and run the ### Step 3.2 Modify parameters for TiDB Cloud +<<<<<<< HEAD:develop/dev-guide-sample-application-golang.md
@@ -796,6 +797,9 @@ dsn := "2aEp24QWEDLqRFs.root:123456@tcp(xxx.tidbcloud.com:4000)/test?charset=utf
If you are using a TiDB Cloud Serverless Tier cluster, modify the value of the `dsn` in `sqldriver.go`: +======= +If you are using a TiDB Serverless cluster, modify the value of the `dsn` in `sqldriver.go`: +>>>>>>> 76416ca7e (tidb: rename products (#13692)):develop/dev-guide-sample-application-golang-sql-driver.md ```go dsn := "root:@tcp(127.0.0.1:4000)/test?charset=utf8mb4" diff --git a/develop/dev-guide-sample-application-java-hibernate.md b/develop/dev-guide-sample-application-java-hibernate.md new file mode 100644 index 0000000000000..ef346f9c3150d --- /dev/null +++ b/develop/dev-guide-sample-application-java-hibernate.md @@ -0,0 +1,426 @@ +--- +title: Build a Simple CRUD App with TiDB and Hibernate +summary: Learn how to build a simple CRUD application with TiDB and Hibernate. +--- + + + + +# Build a Simple CRUD App with TiDB and Hibernate + +This document describes how to use TiDB and Hibernate to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Java 8 or a later Java version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-java.git +``` + +Compared with Hibernate, the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. + +Hibernate is a popular open-source Java ORM, and it supports TiDB dialect starting from `v6.0.0.Beta2`, which fits TiDB features well. The following instructions take `v6.0.0.Beta2` as an example. + +Change to the `plain-java-hibernate` directory: + +```shell +cd plain-java-hibernate +``` + +The structure of this directory is as follows: + +``` +. +├── Makefile +├── plain-java-hibernate.iml +├── pom.xml +└── src + └── main + ├── java + │ └── com + │ └── pingcap + │ └── HibernateExample.java + └── resources + └── hibernate.cfg.xml +``` + +`hibernate.cfg.xml` is the Hibernate configuration file: + +```xml + + + + + + + com.mysql.cj.jdbc.Driver + org.hibernate.dialect.TiDBDialect + jdbc:mysql://localhost:4000/test + root + + false + + + create-drop + + + true + true + + +``` + +`HibernateExample.java` is the main body of the `plain-java-hibernate`. Compared with JDBC, when using Hibernate, you only need to write the path of the configuration file, because Hibernate avoids differences in database creation between different databases. + +`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. Compared with JDBC, Hibernate encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. + +`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. Compared with JDBC, `PlayerBean` in Hibernate adds annotations to indicate mapping relationships for more information. + +```java +package com.pingcap; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.hibernate.JDBCException; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.cfg.Configuration; +import org.hibernate.query.NativeQuery; +import org.hibernate.query.Query; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +@Entity +@Table(name = "player_hibernate") +class PlayerBean { + @Id + private String id; + @Column(name = "coins") + private Integer coins; + @Column(name = "goods") + private Integer goods; + + public PlayerBean() { + } + + public PlayerBean(String id, Integer coins, Integer goods) { + this.id = id; + this.coins = coins; + this.goods = goods; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Integer getCoins() { + return coins; + } + + public void setCoins(Integer coins) { + this.coins = coins; + } + + public Integer getGoods() { + return goods; + } + + public void setGoods(Integer goods) { + this.goods = goods; + } + + @Override + public String toString() { + return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", + "id", this.id, "coins", this.coins, "goods", this.goods); + } +} + +/** + * Main class for the basic Hibernate example. + **/ +public class HibernateExample +{ + public static class PlayerDAO { + public static class NotEnoughException extends RuntimeException { + public NotEnoughException(String message) { + super(message); + } + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic so we don't have to duplicate it in + // various places. + public Object runTransaction(Session session, Function fn) { + Object resultObject = null; + + Transaction txn = session.beginTransaction(); + try { + resultObject = fn.apply(session); + txn.commit(); + System.out.println("APP: COMMIT;"); + } catch (JDBCException e) { + System.out.println("APP: ROLLBACK BY JDBC ERROR;"); + txn.rollback(); + } catch (NotEnoughException e) { + System.out.printf("APP: ROLLBACK BY LOGIC; %s", e.getMessage()); + txn.rollback(); + } + return resultObject; + } + + public Function createPlayers(List players) throws JDBCException { + return session -> { + Integer addedPlayerAmount = 0; + for (PlayerBean player: players) { + session.persist(player); + addedPlayerAmount ++; + } + System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); + return addedPlayerAmount; + }; + } + + public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) throws JDBCException { + return session -> { + PlayerBean sellPlayer = session.get(PlayerBean.class, sellId); + PlayerBean buyPlayer = session.get(PlayerBean.class, buyId); + + if (buyPlayer == null || sellPlayer == null) { + throw new NotEnoughException("sell or buy player not exist"); + } + + if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { + throw new NotEnoughException("coins or goods not enough, rollback"); + } + + buyPlayer.setGoods(buyPlayer.getGoods() + amount); + buyPlayer.setCoins(buyPlayer.getCoins() - price); + session.persist(buyPlayer); + + sellPlayer.setGoods(sellPlayer.getGoods() - amount); + sellPlayer.setCoins(sellPlayer.getCoins() + price); + session.persist(sellPlayer); + + System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); + return 0; + }; + } + + public Function getPlayerByID(String id) throws JDBCException { + return session -> session.get(PlayerBean.class, id); + } + + public Function printPlayers(Integer limit) throws JDBCException { + return session -> { + NativeQuery limitQuery = session.createNativeQuery("SELECT * FROM player_hibernate LIMIT :limit", PlayerBean.class); + limitQuery.setParameter("limit", limit); + List players = limitQuery.getResultList(); + + for (PlayerBean player: players) { + System.out.println("\n[printPlayers]:\n" + player); + } + return 0; + }; + } + + public Function countPlayers() throws JDBCException { + return session -> { + Query countQuery = session.createQuery("SELECT count(player_hibernate) FROM PlayerBean player_hibernate", Long.class); + return countQuery.getSingleResult(); + }; + } + } + + public static void main(String[] args) { + // 1. Create a SessionFactory based on our hibernate.cfg.xml configuration + // file, which defines how to connect to the database. + SessionFactory sessionFactory + = new Configuration() + .configure("hibernate.cfg.xml") + .addAnnotatedClass(PlayerBean.class) + .buildSessionFactory(); + + try (Session session = sessionFactory.openSession()) { + // 2. And then, create DAO to manager your data. + PlayerDAO playerDAO = new PlayerDAO(); + + // 3. Run some simple example. + + // Create a player who has 1 coin and 1 goods. + playerDAO.runTransaction(session, playerDAO.createPlayers(Collections.singletonList( + new PlayerBean("test", 1, 1)))); + + // Get a player. + PlayerBean testPlayer = (PlayerBean)playerDAO.runTransaction(session, playerDAO.getPlayerByID("test")); + System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", + testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); + + // Count players amount. + Long count = (Long)playerDAO.runTransaction(session, playerDAO.countPlayers()); + System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); + + // Print 3 players. + playerDAO.runTransaction(session, playerDAO.printPlayers(3)); + + // 4. Getting further. + + // Player 1: id is "1", has only 100 coins. + // Player 2: id is "2", has 114514 coins, and 20 goods. + PlayerBean player1 = new PlayerBean("1", 100, 0); + PlayerBean player2 = new PlayerBean("2", 114514, 20); + + // Create two players "by hand", using the INSERT statement on the backend. + int addedCount = (Integer)playerDAO.runTransaction(session, + playerDAO.createPlayers(Arrays.asList(player1, player2))); + System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); + + // Player 1 wants to buy 10 goods from player 2. + // It will cost 500 coins, but player 1 can't afford it. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); + Integer updatedCount = (Integer)playerDAO.runTransaction(session, + playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + + // So player 1 have to reduce his incoming quantity to two. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); + updatedCount = (Integer)playerDAO.runTransaction(session, + playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + } finally { + sessionFactory.close(); + } + } +} +``` + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Table initialization + +No need to initialize tables manually. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, modify the `hibernate.connection.url`, `hibernate.connection.username`, `hibernate.connection.password` in `hibernate.cfg.xml`. + +```xml + + + + + + + com.mysql.cj.jdbc.Driver + org.hibernate.dialect.TiDBDialect + jdbc:mysql://localhost:4000/test + root + + false + + + create-drop + + + true + true + + +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the parameters as follows: + +```xml + + + + + + + com.mysql.cj.jdbc.Driver + org.hibernate.dialect.TiDBDialect + jdbc:mysql://xxx.tidbcloud.com:4000/test?sslMode=VERIFY_IDENTITY&enabledTLSProtocols=TLSv1.2,TLSv1.3 + 2aEp24QWEDLqRFs.root + 123456 + false + + + create-drop + + + true + true + + +``` + +### Step 3.3 Run + +To run the code, you can run `make build` and `make run` respectively: + +```shell +make build # this command executes `mvn clean package` +make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` +``` + +Or you can use the native commands: + +```shell +mvn clean package +java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar +``` + +Or run the `make` command directly, which is a combination of `make build` and `make run`. + +## Step 4. Expected output + +[Hibernate Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-hibernate) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-java-jdbc.md b/develop/dev-guide-sample-application-java-jdbc.md new file mode 100644 index 0000000000000..02ba214c71e0c --- /dev/null +++ b/develop/dev-guide-sample-application-java-jdbc.md @@ -0,0 +1,591 @@ +--- +title: Build a Simple CRUD App with TiDB and JDBC +summary: Learn how to build a simple CRUD application with TiDB and JDBC. +--- + + + + +# Build a Simple CRUD App with TiDB and JDBC + +This document describes how to use TiDB and JDBC to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Java 8 or a later Java version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-java.git +``` + +Change to the `plain-java-jdbc` directory: + +```shell +cd plain-java-jdbc +``` + +The structure of this directory is as follows: + +``` +. +├── Makefile +├── plain-java-jdbc.iml +├── pom.xml +└── src + └── main + ├── java + │ └── com + │ └── pingcap + │ └── JDBCExample.java + └── resources + └── dbinit.sql +``` + +You can find initialization statements for the table creation in `dbinit.sql`: + +```sql +USE test; +DROP TABLE IF EXISTS player; + +CREATE TABLE player ( + `id` VARCHAR(36), + `coins` INTEGER, + `goods` INTEGER, + PRIMARY KEY (`id`) +); +``` + +`JDBCExample.java` is the main body of the `plain-java-jdbc`. TiDB is highly compatible with the MySQL protocol, so you need to initialize a MySQL source instance `MysqlDataSource` to connect to TiDB. Then, you can initialize `PlayerDAO` for object management and use it to read, edit, add, and delete data. + +`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods to provide the ability to write data. + +`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. + +```java +package com.pingcap; + +import com.mysql.cj.jdbc.MysqlDataSource; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +/** + * Main class for the basic JDBC example. + **/ +public class JDBCExample +{ + public static class PlayerBean { + private String id; + private Integer coins; + private Integer goods; + + public PlayerBean() { + } + + public PlayerBean(String id, Integer coins, Integer goods) { + this.id = id; + this.coins = coins; + this.goods = goods; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Integer getCoins() { + return coins; + } + + public void setCoins(Integer coins) { + this.coins = coins; + } + + public Integer getGoods() { + return goods; + } + + public void setGoods(Integer goods) { + this.goods = goods; + } + + @Override + public String toString() { + return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", + "id", this.id, "coins", this.coins, "goods", this.goods); + } + } + + /** + * Data access object used by 'ExampleDataSource'. + * Example for CURD and bulk insert. + */ + public static class PlayerDAO { + private final MysqlDataSource ds; + private final Random rand = new Random(); + + PlayerDAO(MysqlDataSource ds) { + this.ds = ds; + } + + /** + * Create players by passing in a List of PlayerBean. + * + * @param players Will create players list + * @return The number of create accounts + */ + public int createPlayers(List players){ + int rows = 0; + + Connection connection = null; + PreparedStatement preparedStatement = null; + try { + connection = ds.getConnection(); + preparedStatement = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)"); + } catch (SQLException e) { + System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + e.printStackTrace(); + + return -1; + } + + try { + for (PlayerBean player : players) { + preparedStatement.setString(1, player.getId()); + preparedStatement.setInt(2, player.getCoins()); + preparedStatement.setInt(3, player.getGoods()); + + preparedStatement.execute(); + rows += preparedStatement.getUpdateCount(); + } + } catch (SQLException e) { + System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + e.printStackTrace(); + } finally { + try { + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + System.out.printf("\n[createPlayers]:\n '%s'\n", preparedStatement); + return rows; + } + + /** + * Buy goods and transfer funds between one player and another in one transaction. + * @param sellId Sell player id. + * @param buyId Buy player id. + * @param amount Goods amount, if sell player has not enough goods, the trade will break. + * @param price Price should pay, if buy player has not enough coins, the trade will break. + * + * @return The number of effected players. + */ + public int buyGoods(String sellId, String buyId, Integer amount, Integer price) { + int effectPlayers = 0; + + Connection connection = null; + try { + connection = ds.getConnection(); + } catch (SQLException e) { + System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + e.printStackTrace(); + return effectPlayers; + } + + try { + connection.setAutoCommit(false); + + PreparedStatement playerQuery = connection.prepareStatement("SELECT * FROM player WHERE id=? OR id=? FOR UPDATE"); + playerQuery.setString(1, sellId); + playerQuery.setString(2, buyId); + playerQuery.execute(); + + PlayerBean sellPlayer = null; + PlayerBean buyPlayer = null; + + ResultSet playerQueryResultSet = playerQuery.getResultSet(); + while (playerQueryResultSet.next()) { + PlayerBean player = new PlayerBean( + playerQueryResultSet.getString("id"), + playerQueryResultSet.getInt("coins"), + playerQueryResultSet.getInt("goods") + ); + + System.out.println("\n[buyGoods]:\n 'check goods and coins enough'"); + System.out.println(player); + + if (sellId.equals(player.getId())) { + sellPlayer = player; + } else { + buyPlayer = player; + } + } + + if (sellPlayer == null || buyPlayer == null) { + throw new SQLException("player not exist."); + } + + if (sellPlayer.getGoods().compareTo(amount) < 0) { + throw new SQLException(String.format("sell player %s goods not enough.", sellId)); + } + + if (buyPlayer.getCoins().compareTo(price) < 0) { + throw new SQLException(String.format("buy player %s coins not enough.", buyId)); + } + + PreparedStatement transfer = connection.prepareStatement("UPDATE player set goods = goods + ?, coins = coins + ? WHERE id=?"); + transfer.setInt(1, -amount); + transfer.setInt(2, price); + transfer.setString(3, sellId); + transfer.execute(); + effectPlayers += transfer.getUpdateCount(); + + transfer.setInt(1, amount); + transfer.setInt(2, -price); + transfer.setString(3, buyId); + transfer.execute(); + effectPlayers += transfer.getUpdateCount(); + + connection.commit(); + + System.out.println("\n[buyGoods]:\n 'trade success'"); + } catch (SQLException e) { + System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + + try { + System.out.println("[buyGoods] Rollback"); + + connection.rollback(); + } catch (SQLException ex) { + // do nothing + } + } finally { + try { + connection.close(); + } catch (SQLException e) { + // do nothing + } + } + + return effectPlayers; + } + + /** + * Get the player info by id. + * + * @param id Player id. + * @return The player of this id. + */ + public PlayerBean getPlayer(String id) { + PlayerBean player = null; + + try (Connection connection = ds.getConnection()) { + PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player WHERE id = ?"); + preparedStatement.setString(1, id); + preparedStatement.execute(); + + ResultSet res = preparedStatement.executeQuery(); + if(!res.next()) { + System.out.printf("No players in the table with id %s", id); + } else { + player = new PlayerBean(res.getString("id"), res.getInt("coins"), res.getInt("goods")); + } + } catch (SQLException e) { + System.out.printf("PlayerDAO.getPlayer ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return player; + } + + /** + * Insert randomized account data (id, coins, goods) using the JDBC fast path for + * bulk inserts. The fastest way to get data into TiDB is using the + * TiDB Lightning(https://docs.pingcap.com/tidb/stable/tidb-lightning-overview). + * However, if you must bulk insert from the application using INSERT SQL, the best + * option is the method shown here. It will require the following: + * + * Add `rewriteBatchedStatements=true` to your JDBC connection settings. + * Setting rewriteBatchedStatements to true now causes CallableStatements + * with batched arguments to be re-written in the form "CALL (...); CALL (...); ..." + * to send the batch in as few client/server round trips as possible. + * https://dev.mysql.com/doc/relnotes/connector-j/5.1/en/news-5-1-3.html + * + * You can see the `rewriteBatchedStatements` param effect logic at + * implement function: `com.mysql.cj.jdbc.StatementImpl.executeBatchUsingMultiQueries` + * + * @param total Add players amount. + * @param batchSize Bulk insert size for per batch. + * + * @return The number of new accounts inserted. + */ + public int bulkInsertRandomPlayers(Integer total, Integer batchSize) { + int totalNewPlayers = 0; + + try (Connection connection = ds.getConnection()) { + // We're managing the commit lifecycle ourselves, so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)")) { + for (int i=0; i<=(total/batchSize);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewPlayers; + } + + + /** + * Print a subset of players from the data store by limit. + * + * @param limit Print max size. + */ + public void printPlayers(Integer limit) { + try (Connection connection = ds.getConnection()) { + PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player LIMIT ?"); + preparedStatement.setInt(1, limit); + preparedStatement.execute(); + + ResultSet res = preparedStatement.executeQuery(); + while (!res.next()) { + PlayerBean player = new PlayerBean(res.getString("id"), + res.getInt("coins"), res.getInt("goods")); + System.out.println("\n[printPlayers]:\n" + player); + } + } catch (SQLException e) { + System.out.printf("PlayerDAO.printPlayers ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } + + + /** + * Count players from the data store. + * + * @return All players count + */ + public int countPlayers() { + int count = 0; + + try (Connection connection = ds.getConnection()) { + PreparedStatement preparedStatement = connection.prepareStatement("SELECT count(*) FROM player"); + preparedStatement.execute(); + + ResultSet res = preparedStatement.executeQuery(); + if(res.next()) { + count = res.getInt(1); + } + } catch (SQLException e) { + System.out.printf("PlayerDAO.countPlayers ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return count; + } + } + + public static void main(String[] args) { + // 1. Configure the example database connection. + + // 1.1 Create a mysql data source instance. + MysqlDataSource mysqlDataSource = new MysqlDataSource(); + + // 1.2 Set server name, port, database name, username and password. + mysqlDataSource.setServerName("localhost"); + mysqlDataSource.setPortNumber(4000); + mysqlDataSource.setDatabaseName("test"); + mysqlDataSource.setUser("root"); + mysqlDataSource.setPassword(""); + + // Or you can use jdbc string instead. + // mysqlDataSource.setURL("jdbc:mysql://{host}:{port}/test?user={user}&password={password}"); + + // 2. And then, create DAO to manager your data. + PlayerDAO dao = new PlayerDAO(mysqlDataSource); + + // 3. Run some simple example. + + // Create a player, has a coin and a goods. + dao.createPlayers(Collections.singletonList(new PlayerBean("test", 1, 1))); + + // Get a player. + PlayerBean testPlayer = dao.getPlayer("test"); + System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", + testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); + + // Create players with bulk inserts, insert 1919 players totally, and per batch for 114 players. + int addedCount = dao.bulkInsertRandomPlayers(1919, 114); + System.out.printf("PlayerDAO.bulkInsertRandomPlayers:\n => %d total inserted players\n", addedCount); + + // Count players amount. + int count = dao.countPlayers(); + System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); + + // Print 3 players. + dao.printPlayers(3); + + // 4. Getting further. + + // Player 1: id is "1", has only 100 coins. + // Player 2: id is "2", has 114514 coins, and 20 goods. + PlayerBean player1 = new PlayerBean("1", 100, 0); + PlayerBean player2 = new PlayerBean("2", 114514, 20); + + // Create two players "by hand", using the INSERT statement on the backend. + addedCount = dao.createPlayers(Arrays.asList(player1, player2)); + System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); + + // Player 1 wants to buy 10 goods from player 2. + // It will cost 500 coins, but player 1 can't afford it. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); + int updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 10, 500); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + + // So player 1 have to reduce his incoming quantity to two. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); + updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 2, 100); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + } +} +``` + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Table initialization + + + +When using JDBC, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-jdbc` directory: + +```shell +make mysql +``` + +Or you can execute the following command: + +```shell +mysql --host 127.0.0.1 --port 4000 -u root + + + +When using JDBC, you need to connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file to initialize the database tables manually. + + + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, modify the parameters of the host, port, user, and password in `JDBCExample.java`: + +```java +mysqlDataSource.setServerName("localhost"); +mysqlDataSource.setPortNumber(4000); +mysqlDataSource.setDatabaseName("test"); +mysqlDataSource.setUser("root"); +mysqlDataSource.setPassword(""); +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the parameters as follows: + +```java +mysqlDataSource.setServerName("xxx.tidbcloud.com"); +mysqlDataSource.setPortNumber(4000); +mysqlDataSource.setDatabaseName("test"); +mysqlDataSource.setUser("2aEp24QWEDLqRFs.root"); +mysqlDataSource.setPassword("123456"); +mysqlDataSource.setSslMode(PropertyDefinitions.SslMode.VERIFY_IDENTITY.name()); +mysqlDataSource.setEnabledTLSProtocols("TLSv1.2,TLSv1.3"); +``` + +### Step 3.3 Run + +To run the code, you can run `make build` and `make run` respectively: + +```shell +make build # this command executes `mvn clean package` +make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` +``` + +Or you can use the native commands: + +```shell +mvn clean package +java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar +``` + +Or run the `make` command directly, which is a combination of `make build` and `make run`. + +## Step 4. Expected output + +[JDBC Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-jdbc) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-java-mybatis.md b/develop/dev-guide-sample-application-java-mybatis.md new file mode 100644 index 0000000000000..36290ccb29789 --- /dev/null +++ b/develop/dev-guide-sample-application-java-mybatis.md @@ -0,0 +1,741 @@ +--- +title: Build a Simple CRUD App with TiDB and Mybatis +summary: Learn how to build a simple CRUD application with TiDB and Mybatis. +--- + + + + +# Build a Simple CRUD App with TiDB and Mybatis + +This document describes how to use TiDB and Mybatis to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Java 8 or a later Java version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-java.git +``` + +Compared with [Mybatis](https://mybatis.org/mybatis-3/index.html), the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. + +Mybatis is a popular open-source Java class persistence framework. The following uses [MyBatis Generator](https://mybatis.org/generator/quickstart.html) as a Maven plugin to generate the persistence layer code. + +Change to the `plain-java-mybatis` directory: + +```shell +cd plain-java-mybatis +``` + +The structure of this directory is as follows: + +``` +. +├── Makefile +├── pom.xml +└── src + └── main + ├── java + │   └── com + │   └── pingcap + │   ├── MybatisExample.java + │   ├── dao + │   │   └── PlayerDAO.java + │   └── model + │   ├── Player.java + │   ├── PlayerMapper.java + │   └── PlayerMapperEx.java + └── resources + ├── dbinit.sql + ├── log4j.properties + ├── mapper + │   ├── PlayerMapper.xml + │   └── PlayerMapperEx.xml + ├── mybatis-config.xml + └── mybatis-generator.xml +``` + +The automatically generated files are: + +- `src/main/java/com/pingcap/model/Player.java`: The `Player` entity class. +- `src/main/java/com/pingcap/model/PlayerMapper.java`: The interface of `PlayerMapper`. +- `src/main/resources/mapper/PlayerMapper.xml`: The XML mapping of `Player`. Mybatis uses this configuration to automatically generate the implementation class of the `PlayerMapper` interface. + +The strategy for generating these files is written in `mybatis-generator.xml`, which is the configuration file for [Mybatis Generator](https://mybatis.org/generator/quickstart.html). There are comments in the following configuration file to describe how to use it. + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +`mybatis-generator.xml` is included in `pom.xml` as the configuration of `mybatis-generator-maven-plugin`. + +```xml + + org.mybatis.generator + mybatis-generator-maven-plugin + 1.4.1 + + src/main/resources/mybatis-generator.xml + true + true + + + + + + mysql + mysql-connector-java + 5.1.49 + + + +``` + +Once included in the Maven plugin, you can delete the old generated files and make new ones using `mvn mybatis-generate`. Or you can use `make gen` to delete the old file and generate a new one at the same time. + +> **Note:** +> +> The property `configuration.overwrite` in `mybatis-generator.xml` only ensures that the generated Java code files are overwritten. But the XML mapping files are still written as appended. Therefore, it is recommended to delete the old file before Mybaits Generator generating a new one. + +`Player.java` is a data entity class file generated using Mybatis Generator, which is a mapping of database tables in the application. Each property of the `Player` class corresponds to a field in the `player` table. + +```java +package com.pingcap.model; + +public class Player { + private String id; + + private Integer coins; + + private Integer goods; + + public Player(String id, Integer coins, Integer goods) { + this.id = id; + this.coins = coins; + this.goods = goods; + } + + public Player() { + super(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Integer getCoins() { + return coins; + } + + public void setCoins(Integer coins) { + this.coins = coins; + } + + public Integer getGoods() { + return goods; + } + + public void setGoods(Integer goods) { + this.goods = goods; + } +} +``` + +`PlayerMapper.java` is a mapping interface file generated using Mybatis Generator. This file only defines the interface, and the implementation classes of interface are automatically generated using XML or annotations. + +```java +package com.pingcap.model; + +import com.pingcap.model.Player; + +public interface PlayerMapper { + int deleteByPrimaryKey(String id); + + int insert(Player row); + + int insertSelective(Player row); + + Player selectByPrimaryKey(String id); + + int updateByPrimaryKeySelective(Player row); + + int updateByPrimaryKey(Player row); +} +``` + +`PlayerMapper.xml` is a mapping XML file generated using Mybatis Generator. Mybatis uses this to automatically generate the implementation class of the `PlayerMapper` interface. + +```xml + + + + + + + + + + + + id, coins, goods + + + + delete from player + where id = #{id,jdbcType=VARCHAR} + + + insert into player (id, coins, goods + ) + values (#{id,jdbcType=VARCHAR}, #{coins,jdbcType=INTEGER}, #{goods,jdbcType=INTEGER} + ) + + + insert into player + + + id, + + + coins, + + + goods, + + + + + #{id,jdbcType=VARCHAR}, + + + #{coins,jdbcType=INTEGER}, + + + #{goods,jdbcType=INTEGER}, + + + + + update player + + + coins = #{coins,jdbcType=INTEGER}, + + + goods = #{goods,jdbcType=INTEGER}, + + + where id = #{id,jdbcType=VARCHAR} + + + update player + set coins = #{coins,jdbcType=INTEGER}, + goods = #{goods,jdbcType=INTEGER} + where id = #{id,jdbcType=VARCHAR} + + +``` + +Since Mybatis Generator needs to generate the source code from the table definition, the table needs to be created first. To create the table, you can use `dbinit.sql`. + +```sql +USE test; +DROP TABLE IF EXISTS player; + +CREATE TABLE player ( + `id` VARCHAR(36), + `coins` INTEGER, + `goods` INTEGER, + PRIMARY KEY (`id`) +); +``` + +Split the interface `PlayerMapperEx` additionally to extend from `PlayerMapper` and write a matching `PlayerMapperEx.xml` file. Avoid changing `PlayerMapper.java` and `PlayerMapper.xml` directly. This is to avoid overwrite by Mybatis Generator. + +Define the added interface in `PlayerMapperEx.java`: + +```java +package com.pingcap.model; + +import java.util.List; + +public interface PlayerMapperEx extends PlayerMapper { + Player selectByPrimaryKeyWithLock(String id); + + List selectByLimit(Integer limit); + + Integer count(); +} +``` + +Define the mapping rules in `PlayerMapperEx.xml`: + +```xml + + + + + + + + + + + + id, coins, goods + + + + + + + + + +``` + +`PlayerDAO.java` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. In it, Mybatis encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. + +```java +package com.pingcap.dao; + +import com.pingcap.model.Player; +import com.pingcap.model.PlayerMapperEx; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; + +import java.util.List; +import java.util.function.Function; + +public class PlayerDAO { + public static class NotEnoughException extends RuntimeException { + public NotEnoughException(String message) { + super(message); + } + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic, so we don't have to duplicate it in + // various places. + public Object runTransaction(SqlSessionFactory sessionFactory, Function fn) { + Object resultObject = null; + SqlSession session = null; + + try { + // open a session with autoCommit is false + session = sessionFactory.openSession(false); + + // get player mapper + PlayerMapperEx playerMapperEx = session.getMapper(PlayerMapperEx.class); + + resultObject = fn.apply(playerMapperEx); + session.commit(); + System.out.println("APP: COMMIT;"); + } catch (Exception e) { + if (e instanceof NotEnoughException) { + System.out.printf("APP: ROLLBACK BY LOGIC; \n%s\n", e.getMessage()); + } else { + System.out.printf("APP: ROLLBACK BY ERROR; \n%s\n", e.getMessage()); + } + + if (session != null) { + session.rollback(); + } + } finally { + if (session != null) { + session.close(); + } + } + + return resultObject; + } + + public Function createPlayers(List players) { + return playerMapperEx -> { + Integer addedPlayerAmount = 0; + for (Player player: players) { + playerMapperEx.insert(player); + addedPlayerAmount ++; + } + System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); + return addedPlayerAmount; + }; + } + + public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) { + return playerMapperEx -> { + Player sellPlayer = playerMapperEx.selectByPrimaryKeyWithLock(sellId); + Player buyPlayer = playerMapperEx.selectByPrimaryKeyWithLock(buyId); + + if (buyPlayer == null || sellPlayer == null) { + throw new NotEnoughException("sell or buy player not exist"); + } + + if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { + throw new NotEnoughException("coins or goods not enough, rollback"); + } + + int affectRows = 0; + buyPlayer.setGoods(buyPlayer.getGoods() + amount); + buyPlayer.setCoins(buyPlayer.getCoins() - price); + affectRows += playerMapperEx.updateByPrimaryKey(buyPlayer); + + sellPlayer.setGoods(sellPlayer.getGoods() - amount); + sellPlayer.setCoins(sellPlayer.getCoins() + price); + affectRows += playerMapperEx.updateByPrimaryKey(sellPlayer); + + System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); + return affectRows; + }; + } + + public Function getPlayerByID(String id) { + return playerMapperEx -> playerMapperEx.selectByPrimaryKey(id); + } + + public Function printPlayers(Integer limit) { + return playerMapperEx -> { + List players = playerMapperEx.selectByLimit(limit); + + for (Player player: players) { + System.out.println("\n[printPlayers]:\n" + player); + } + return 0; + }; + } + + public Function countPlayers() { + return PlayerMapperEx::count; + } +} +``` + +`MybatisExample` is the main class of the `plain-java-mybatis` sample application. It defines the entry functions: + +```java +package com.pingcap; + +import com.pingcap.dao.PlayerDAO; +import com.pingcap.model.Player; +import org.apache.ibatis.io.Resources; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; + +public class MybatisExample { + public static void main( String[] args ) throws IOException { + // 1. Create a SqlSessionFactory based on our mybatis-config.xml configuration + // file, which defines how to connect to the database. + InputStream inputStream = Resources.getResourceAsStream("mybatis-config.xml"); + SqlSessionFactory sessionFactory = new SqlSessionFactoryBuilder().build(inputStream); + + // 2. And then, create DAO to manager your data + PlayerDAO playerDAO = new PlayerDAO(); + + // 3. Run some simple examples. + + // Create a player who has 1 coin and 1 goods. + playerDAO.runTransaction(sessionFactory, playerDAO.createPlayers( + Collections.singletonList(new Player("test", 1, 1)))); + + // Get a player. + Player testPlayer = (Player)playerDAO.runTransaction(sessionFactory, playerDAO.getPlayerByID("test")); + System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", + testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); + + // Count players amount. + Integer count = (Integer)playerDAO.runTransaction(sessionFactory, playerDAO.countPlayers()); + System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); + + // Print 3 players. + playerDAO.runTransaction(sessionFactory, playerDAO.printPlayers(3)); + + // 4. Getting further. + + // Player 1: id is "1", has only 100 coins. + // Player 2: id is "2", has 114514 coins, and 20 goods. + Player player1 = new Player("1", 100, 0); + Player player2 = new Player("2", 114514, 20); + + // Create two players "by hand", using the INSERT statement on the backend. + int addedCount = (Integer)playerDAO.runTransaction(sessionFactory, + playerDAO.createPlayers(Arrays.asList(player1, player2))); + System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); + + // Player 1 wants to buy 10 goods from player 2. + // It will cost 500 coins, but player 1 cannot afford it. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); + Integer updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, + playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + + // So player 1 has to reduce the incoming quantity to two. + System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); + updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, + playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); + System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); + } +} +``` + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Table initialization + +When using Mybatis, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-mybatis` directory: + +```shell +make prepare +``` + +Or you can execute the following command: + +```shell +mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql +``` + +If you are using a non-local cluster or MySQL client has not been installed, connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, modify the `dataSource.url`, `dataSource.username`, `dataSource.password` in `mybatis-config.xml`. + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the parameters in `dataSource` node as follows: + +```xml + + + + + ... + + + + + + + + ... + + +``` + +### Step 3.3 Run + +To run the code, you can run `make prepare`, `make gen`, `make build` and `make run` respectively: + +```shell +make prepare +# this command executes : +# - `mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql` +# - `mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player"` + +make gen +# this command executes : +# - `rm -f src/main/java/com/pingcap/model/Player.java` +# - `rm -f src/main/java/com/pingcap/model/PlayerMapper.java` +# - `rm -f src/main/resources/mapper/PlayerMapper.xml` +# - `mvn mybatis-generator:generate` + +make build # this command executes `mvn clean package` +make run # this command executes `java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar` +``` + +Or you can use the native commands: + +```shell +mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql +mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player" +rm -f src/main/java/com/pingcap/model/Player.java +rm -f src/main/java/com/pingcap/model/PlayerMapper.java +rm -f src/main/resources/mapper/PlayerMapper.xml +mvn mybatis-generator:generate +mvn clean package +java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar +``` + +Or run the `make` command directly, which is a combination of `make prepare`, `make gen`, `make build` and `make run`. + +## Step 4. Expected output + +[Mybatis Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-mybatis) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-java.md b/develop/dev-guide-sample-application-java.md index fbe93c2c644b8..fe482bd741c2c 100644 --- a/develop/dev-guide-sample-application-java.md +++ b/develop/dev-guide-sample-application-java.md @@ -22,9 +22,9 @@ This document describes how to use TiDB and Java to build a simple CRUD applicat The following introduces how to start a TiDB cluster. -**Use a TiDB Cloud Serverless Tier cluster** +**Use a TiDB Serverless cluster** -For detailed steps, see [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). **Use a local cluster** @@ -34,7 +34,7 @@ For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md# -See [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). @@ -293,7 +293,7 @@ public interface PlayerMapper { id, coins, goods - select + select from player where `id` = #{id,jdbcType=VARCHAR} @@ -1448,7 +1448,7 @@ When using JDBC, you need to connect to your cluster and run the statement in th
-If you are using a TiDB Cloud Serverless Tier cluster, modify the `dataSource.url`, `dataSource.username`, `dataSource.password` in `mybatis-config.xml`. +If you are using a TiDB Serverless cluster, modify the `dataSource.url`, `dataSource.username`, `dataSource.password` in `mybatis-config.xml`. ```xml @@ -1523,7 +1523,7 @@ In this case, you can modify the parameters in `dataSource` node as follows:
-If you are using a TiDB Cloud Serverless Tier cluster, modify the `hibernate.connection.url`, `hibernate.connection.username`, `hibernate.connection.password` in `hibernate.cfg.xml`. +If you are using a TiDB Serverless cluster, modify the `hibernate.connection.url`, `hibernate.connection.username`, `hibernate.connection.password` in `hibernate.cfg.xml`. ```xml @@ -1589,7 +1589,7 @@ In this case, you can modify the parameters as follows:
-If you are using a TiDB Cloud Serverless Tier cluster, modify the parameters of the host, port, user, and password in `JDBCExample.java`: +If you are using a TiDB Serverless cluster, modify the parameters of the host, port, user, and password in `JDBCExample.java`: ```java mysqlDataSource.setServerName("localhost"); diff --git a/develop/dev-guide-sample-application-python-mysql-connector.md b/develop/dev-guide-sample-application-python-mysql-connector.md new file mode 100644 index 0000000000000..e9809293c1951 --- /dev/null +++ b/develop/dev-guide-sample-application-python-mysql-connector.md @@ -0,0 +1,291 @@ +--- +title: Build a Simple CRUD App with TiDB and MySQL Connector/Python +summary: Learn how to build a simple CRUD application with TiDB and MySQL Connector/Python. +aliases: ['/tidb/dev/dev-guide-sample-application-python','/tidb/dev/dev-guide-outdated-for-python-mysql-connector'] +--- + + + + +# Build a Simple CRUD App with TiDB and MySQL Connector/Python + +[MySQL Connector/Python](https://dev.mysql.com/doc/connector-python/en/) is a popular open-source driver for Python. + +This document describes how to use TiDB and MySQL Connector/Python to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Python 3.10 or a later Python version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-python.git +``` + +The following uses MySQL Connector/Python 8.0.31 as an example. Drivers for Python are more convenient to use than other languages, but they do not shield the underlying implementation and require manual management of transactions. If there are not a lot of scenarios where SQL is required, it is recommended to use ORM, which can help reduce the coupling of your program. + +```python +import uuid +from typing import List + +from mysql.connector import connect, MySQLConnection +from mysql.connector.cursor import MySQLCursor + + +def get_connection(autocommit: bool = True) -> MySQLConnection: + connection = connect(host='127.0.0.1', + port=4000, + user='root', + password='', + database='test') + connection.autocommit = autocommit + return connection + + +def create_player(cursor: MySQLCursor, player: tuple) -> None: + cursor.execute("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", player) + + +def get_player(cursor: MySQLCursor, player_id: str) -> tuple: + cursor.execute("SELECT id, coins, goods FROM player WHERE id = %s", (player_id,)) + return cursor.fetchone() + + +def get_players_with_limit(cursor: MySQLCursor, limit: int) -> List[tuple]: + cursor.execute("SELECT id, coins, goods FROM player LIMIT %s", (limit,)) + return cursor.fetchall() + + +def random_player(amount: int) -> List[tuple]: + players = [] + for _ in range(amount): + players.append((str(uuid.uuid4()), 10000, 10000)) + + return players + + +def bulk_create_player(cursor: MySQLCursor, players: List[tuple]) -> None: + cursor.executemany("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", players) + + +def get_count(cursor: MySQLCursor) -> int: + cursor.execute("SELECT count(*) FROM player") + return cursor.fetchone()[0] + + +def trade_check(cursor: MySQLCursor, sell_id: str, buy_id: str, amount: int, price: int) -> bool: + get_player_with_lock_sql = "SELECT coins, goods FROM player WHERE id = %s FOR UPDATE" + + # sell player goods check + cursor.execute(get_player_with_lock_sql, (sell_id,)) + _, sell_goods = cursor.fetchone() + if sell_goods < amount: + print(f'sell player {sell_id} goods not enough') + return False + + # buy player coins check + cursor.execute(get_player_with_lock_sql, (buy_id,)) + buy_coins, _ = cursor.fetchone() + if buy_coins < price: + print(f'buy player {buy_id} coins not enough') + return False + + +def trade_update(cursor: MySQLCursor, sell_id: str, buy_id: str, amount: int, price: int) -> None: + update_player_sql = "UPDATE player set goods = goods + %s, coins = coins + %s WHERE id = %s" + + # deduct the goods of seller, and raise his/her the coins + cursor.execute(update_player_sql, (-amount, price, sell_id)) + # deduct the coins of buyer, and raise his/her the goods + cursor.execute(update_player_sql, (amount, -price, buy_id)) + + +def trade(connection: MySQLConnection, sell_id: str, buy_id: str, amount: int, price: int) -> None: + with connection.cursor() as cursor: + if trade_check(cursor, sell_id, buy_id, amount, price) is False: + connection.rollback() + return + + try: + trade_update(cursor, sell_id, buy_id, amount, price) + except Exception as err: + connection.rollback() + print(f'something went wrong: {err}') + else: + connection.commit() + print("trade success") + + +def simple_example() -> None: + with get_connection(autocommit=True) as connection: + with connection.cursor() as cur: + # create a player, who has a coin and a goods. + create_player(cur, ("test", 1, 1)) + + # get this player, and print it. + test_player = get_player(cur, "test") + print(f'id:{test_player[0]}, coins:{test_player[1]}, goods:{test_player[2]}') + + # create players with bulk inserts. + # insert 1919 players totally, with 114 players per batch. + # each player has a random UUID + player_list = random_player(1919) + for idx in range(0, len(player_list), 114): + bulk_create_player(cur, player_list[idx:idx + 114]) + + # print the number of players + count = get_count(cur) + print(f'number of players: {count}') + + # print 3 players. + three_players = get_players_with_limit(cur, 3) + for player in three_players: + print(f'id:{player[0]}, coins:{player[1]}, goods:{player[2]}') + + +def trade_example() -> None: + with get_connection(autocommit=False) as conn: + with conn.cursor() as cur: + # create two players + # player 1: id is "1", has only 100 coins. + # player 2: id is "2", has 114514 coins, and 20 goods. + create_player(cur, ("1", 100, 0)) + create_player(cur, ("2", 114514, 20)) + conn.commit() + + # player 1 wants to buy 10 goods from player 2. + # it will cost 500 coins, but player 1 cannot afford it. + # so this trade will fail, and nobody will lose their coins or goods + trade(conn, sell_id="2", buy_id="1", amount=10, price=500) + + # then player 1 has to reduce the incoming quantity to 2. + # this trade will be successful + trade(conn, sell_id="2", buy_id="1", amount=2, price=100) + + # let's take a look for player 1 and player 2 currently + with conn.cursor() as cur: + _, player1_coin, player1_goods = get_player(cur, "1") + print(f'id:1, coins:{player1_coin}, goods:{player1_goods}') + _, player2_coin, player2_goods = get_player(cur, "2") + print(f'id:2, coins:{player2_coin}, goods:{player2_goods}') + + +simple_example() +trade_example() +``` + +The driver has a lower level of encapsulation than ORM, so there are a lot of SQL statements in the program. Unlike ORM, there is no data object in drivers, so the `Player` queried by the driver is represented as a tuple. + +For more information about how to use MySQL Connector/Python, refer to [MySQL Connector/Python documentation](https://dev.mysql.com/doc/connector-python/en/). + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Initialize table + +Before running the code, you need to initialize the table manually. If you are using a local TiDB cluster, you can run the following command: + + + +
+ +```shell +mysql --host 127.0.0.1 --port 4000 -u root < player_init.sql +``` + +
+ +
+ +```shell +mycli --host 127.0.0.1 --port 4000 -u root --no-warn < player_init.sql +``` + +
+ + + +If you are not using a local cluster, or have not installed a MySQL client, connect to your cluster using your preferred method (such as Navicat, DBeaver, or other GUI tools) and run the SQL statements in the `player_init.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, you need to provide your CA root path and replace `` in the following examples with your CA path. To get the CA root path on your system, refer to [Where is the CA root path on my system?](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters#where-is-the-ca-root-path-on-my-system).> + +If you are using a TiDB Serverless cluster, change the `get_connection` function in `mysql_connector_python_example.py`: + +```python +def get_connection(autocommit: bool = True) -> MySQLConnection: + connection = connect(host='127.0.0.1', + port=4000, + user='root', + password='', + database='test') + connection.autocommit = autocommit + return connection +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `get_connection` as follows: + +```python +def get_connection(autocommit: bool = True) -> MySQLConnection: + connection = connect( + host="xxx.tidbcloud.com", + port=4000, + user="2aEp24QWEDLqRFs.root", + password="123456", + database="test", + autocommit=autocommit, + ssl_ca='', + ssl_verify_identity=True + ) + connection.autocommit = autocommit + return connection +``` + +### Step 3.3 Run the code + +Before running the code, use the following command to install dependencies: + +```bash +pip3 install -r requirement.txt +``` + +If you need to run the script multiple times, follow the [Table initialization](#step-31-initialize-table) section to initialize the table again before each run. + +```bash +python3 mysql_connector_python_example.py +``` + +## Step 4. Expected output + +[MySQL Connector/Python Expected Output](https://github.com/pingcap-inc/tidb-example-python/blob/main/Expected-Output.md#mysql-connector-python) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-python-mysqlclient.md b/develop/dev-guide-sample-application-python-mysqlclient.md new file mode 100644 index 0000000000000..2e993a66b891e --- /dev/null +++ b/develop/dev-guide-sample-application-python-mysqlclient.md @@ -0,0 +1,292 @@ +--- +title: Build a Simple CRUD App with TiDB and mysqlclient +summary: Learn how to build a simple CRUD application with TiDB and mysqlclient. +--- + + + + +# Build a Simple CRUD App with TiDB and mysqlclient + +[mysqlclient](https://pypi.org/project/mysqlclient/) is a popular open-source driver for Python. + +This document describes how to use TiDB and mysqlclient to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Python 3.10 or a later Python version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-python.git +``` + +The following uses mysqlclient 2.1.1 as an example. Drivers for Python are more convenient to use than other languages, but they do not shield the underlying implementation and require manual management of transactions. If there are not a lot of scenarios where SQL is required, it is recommended to use ORM, which can help reduce the coupling of your program. + +```python +import uuid +from typing import List + +import MySQLdb +from MySQLdb import Connection +from MySQLdb.cursors import Cursor + +def get_connection(autocommit: bool = True) -> MySQLdb.Connection: + return MySQLdb.connect( + host="127.0.0.1", + port=4000, + user="root", + password="", + database="test", + autocommit=autocommit + ) + + +def create_player(cursor: Cursor, player: tuple) -> None: + cursor.execute("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", player) + + +def get_player(cursor: Cursor, player_id: str) -> tuple: + cursor.execute("SELECT id, coins, goods FROM player WHERE id = %s", (player_id,)) + return cursor.fetchone() + + +def get_players_with_limit(cursor: Cursor, limit: int) -> List[tuple]: + cursor.execute("SELECT id, coins, goods FROM player LIMIT %s", (limit,)) + return cursor.fetchall() + + +def random_player(amount: int) -> List[tuple]: + players = [] + for _ in range(amount): + players.append((uuid.uuid4(), 10000, 10000)) + + return players + + +def bulk_create_player(cursor: Cursor, players: List[tuple]) -> None: + cursor.executemany("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", players) + + +def get_count(cursor: Cursor) -> None: + cursor.execute("SELECT count(*) FROM player") + return cursor.fetchone()[0] + + +def trade_check(cursor: Cursor, sell_id: str, buy_id: str, amount: int, price: int) -> bool: + get_player_with_lock_sql = "SELECT coins, goods FROM player WHERE id = %s FOR UPDATE" + + # sell player goods check + cursor.execute(get_player_with_lock_sql, (sell_id,)) + _, sell_goods = cursor.fetchone() + if sell_goods < amount: + print(f'sell player {sell_id} goods not enough') + return False + + # buy player coins check + cursor.execute(get_player_with_lock_sql, (buy_id,)) + buy_coins, _ = cursor.fetchone() + if buy_coins < price: + print(f'buy player {buy_id} coins not enough') + return False + + +def trade_update(cursor: Cursor, sell_id: str, buy_id: str, amount: int, price: int) -> None: + update_player_sql = "UPDATE player set goods = goods + %s, coins = coins + %s WHERE id = %s" + + # deduct the goods of seller, and raise his/her the coins + cursor.execute(update_player_sql, (-amount, price, sell_id)) + # deduct the coins of buyer, and raise his/her the goods + cursor.execute(update_player_sql, (amount, -price, buy_id)) + + +def trade(connection: Connection, sell_id: str, buy_id: str, amount: int, price: int) -> None: + with connection.cursor() as cursor: + if trade_check(cursor, sell_id, buy_id, amount, price) is False: + connection.rollback() + return + + try: + trade_update(cursor, sell_id, buy_id, amount, price) + except Exception as err: + connection.rollback() + print(f'something went wrong: {err}') + else: + connection.commit() + print("trade success") + + +def simple_example() -> None: + with get_connection(autocommit=True) as conn: + with conn.cursor() as cur: + # create a player, who has a coin and a goods. + create_player(cur, ("test", 1, 1)) + + # get this player, and print it. + test_player = get_player(cur, "test") + print(f'id:{test_player[0]}, coins:{test_player[1]}, goods:{test_player[2]}') + + # create players with bulk inserts. + # insert 1919 players totally, with 114 players per batch. + # each player has a random UUID + player_list = random_player(1919) + for idx in range(0, len(player_list), 114): + bulk_create_player(cur, player_list[idx:idx + 114]) + + # print the number of players + count = get_count(cur) + print(f'number of players: {count}') + + # print 3 players. + three_players = get_players_with_limit(cur, 3) + for player in three_players: + print(f'id:{player[0]}, coins:{player[1]}, goods:{player[2]}') + + +def trade_example() -> None: + with get_connection(autocommit=False) as conn: + with conn.cursor() as cur: + # create two players + # player 1: id is "1", has only 100 coins. + # player 2: id is "2", has 114514 coins, and 20 goods. + create_player(cur, ("1", 100, 0)) + create_player(cur, ("2", 114514, 20)) + conn.commit() + + # player 1 wants to buy 10 goods from player 2. + # it will cost 500 coins, but player 1 cannot afford it. + # so this trade will fail, and nobody will lose their coins or goods + trade(conn, sell_id="2", buy_id="1", amount=10, price=500) + + # then player 1 has to reduce the incoming quantity to 2. + # this trade will be successful + trade(conn, sell_id="2", buy_id="1", amount=2, price=100) + + # let's take a look for player 1 and player 2 currently + with conn.cursor() as cur: + _, player1_coin, player1_goods = get_player(cur, "1") + print(f'id:1, coins:{player1_coin}, goods:{player1_goods}') + _, player2_coin, player2_goods = get_player(cur, "2") + print(f'id:2, coins:{player2_coin}, goods:{player2_goods}') + + +simple_example() +trade_example() +``` + +The driver has a lower level of encapsulation than ORM, so there are a lot of SQL statements in the program. Unlike ORM, there is no data object in drivers, so the `Player` queried by the driver is represented as a tuple. + +For more information about how to use mysqlclient, refer to [mysqlclient documentation](https://mysqlclient.readthedocs.io/). + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Initialize table + +Before running the code, you need to initialize the table manually. If you are using a local TiDB cluster, you can run the following command: + + + +
+ +```shell +mysql --host 127.0.0.1 --port 4000 -u root < player_init.sql +``` + +
+ +
+ +```shell +mycli --host 127.0.0.1 --port 4000 -u root --no-warn < player_init.sql +``` + +
+ + + +If you are not using a local cluster, or have not installed a MySQL client, connect to your cluster using your preferred method (such as Navicat, DBeaver, or other GUI tools) and run the SQL statements in the `player_init.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, you need to provide your CA root path and replace `` in the following examples with your CA path. To get the CA root path on your system, refer to [Where is the CA root path on my system?](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters#where-is-the-ca-root-path-on-my-system). + +If you are using a TiDB Serverless cluster, change the `get_connection` function in `mysqlclient_example.py`: + +```python +def get_connection(autocommit: bool = True) -> MySQLdb.Connection: + return MySQLdb.connect( + host="127.0.0.1", + port=4000, + user="root", + password="", + database="test", + autocommit=autocommit + ) +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `get_connection` as follows: + +```python +def get_connection(autocommit: bool = True) -> MySQLdb.Connection: + return MySQLdb.connect( + host="xxx.tidbcloud.com", + port=4000, + user="2aEp24QWEDLqRFs.root", + password="123456", + database="test", + autocommit=autocommit, + ssl_mode="VERIFY_IDENTITY", + ssl={ + "ca": "" + } + ) +``` + +### Step 3.3 Run the code + +Before running the code, use the following command to install dependencies: + +```bash +pip3 install -r requirement.txt +``` + +If you need to run the script multiple times, follow the [Table initialization](#step-31-initialize-table) section to initialize the table again before each run. + +```bash +python3 mysqlclient_example.py +``` + +## Step 4. Expected output + +[mysqlclient Expected Output](https://github.com/pingcap-inc/tidb-example-python/blob/main/Expected-Output.md#mysqlclient) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-python-peewee.md b/develop/dev-guide-sample-application-python-peewee.md new file mode 100644 index 0000000000000..96f116188d14f --- /dev/null +++ b/develop/dev-guide-sample-application-python-peewee.md @@ -0,0 +1,255 @@ +--- +title: Build a Simple CRUD App with TiDB and peewee +summary: Learn how to build a simple CRUD application with TiDB and peewee. +--- + + + + +# Build a Simple CRUD App with TiDB and peewee + +[peewee](http://docs.peewee-orm.com/en/latest/) is a popular open-source ORM library for Python. + +This document describes how to use TiDB and peewee to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Python 3.10 or a later Python version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-python.git +``` + +The following uses peewee 3.15.4 as an example. + +```python +import os +import uuid +from typing import List + +from peewee import * + +from playhouse.db_url import connect + +db = connect('mysql://root:@127.0.0.1:4000/test') + + +class Player(Model): + id = CharField(max_length=36, primary_key=True) + coins = IntegerField() + goods = IntegerField() + + class Meta: + database = db + table_name = "player" + + +def random_player(amount: int) -> List[Player]: + players = [] + for _ in range(amount): + players.append(Player(id=uuid.uuid4(), coins=10000, goods=10000)) + + return players + + +def simple_example() -> None: + # create a player, who has a coin and a goods. + Player.create(id="test", coins=1, goods=1) + + # get this player, and print it. + test_player = Player.select().where(Player.id == "test").get() + print(f'id:{test_player.id}, coins:{test_player.coins}, goods:{test_player.goods}') + + # create players with bulk inserts. + # insert 1919 players totally, with 114 players per batch. + # each player has a random UUID + player_list = random_player(1919) + Player.bulk_create(player_list, 114) + + # print the number of players + count = Player.select().count() + print(f'number of players: {count}') + + # print 3 players. + three_players = Player.select().limit(3) + for player in three_players: + print(f'id:{player.id}, coins:{player.coins}, goods:{player.goods}') + + +def trade_check(sell_id: str, buy_id: str, amount: int, price: int) -> bool: + sell_goods = Player.select(Player.goods).where(Player.id == sell_id).get().goods + if sell_goods < amount: + print(f'sell player {sell_id} goods not enough') + return False + + buy_coins = Player.select(Player.coins).where(Player.id == buy_id).get().coins + if buy_coins < price: + print(f'buy player {buy_id} coins not enough') + return False + + return True + + +def trade(sell_id: str, buy_id: str, amount: int, price: int) -> None: + with db.atomic() as txn: + try: + if trade_check(sell_id, buy_id, amount, price) is False: + txn.rollback() + return + + # deduct the goods of seller, and raise his/her the coins + Player.update(goods=Player.goods - amount, coins=Player.coins + price).where(Player.id == sell_id).execute() + # deduct the coins of buyer, and raise his/her the goods + Player.update(goods=Player.goods + amount, coins=Player.coins - price).where(Player.id == buy_id).execute() + + except Exception as err: + txn.rollback() + print(f'something went wrong: {err}') + else: + txn.commit() + print("trade success") + + +def trade_example() -> None: + # create two players + # player 1: id is "1", has only 100 coins. + # player 2: id is "2", has 114514 coins, and 20 goods. + Player.create(id="1", coins=100, goods=0) + Player.create(id="2", coins=114514, goods=20) + + # player 1 wants to buy 10 goods from player 2. + # it will cost 500 coins, but player 1 cannot afford it. + # so this trade will fail, and nobody will lose their coins or goods + trade(sell_id="2", buy_id="1", amount=10, price=500) + + # then player 1 has to reduce the incoming quantity to 2. + # this trade will be successful + trade(sell_id="2", buy_id="1", amount=2, price=100) + + # let's take a look for player 1 and player 2 currently + after_trade_players = Player.select().where(Player.id.in_(["1", "2"])) + for player in after_trade_players: + print(f'id:{player.id}, coins:{player.coins}, goods:{player.goods}') + + +db.connect() + +# recreate the player table +db.drop_tables([Player]) +db.create_tables([Player]) + +simple_example() +trade_example() +``` + +Compared with using drivers directly, peewee provides an abstraction for the specific details of different databases when you create a database connection. In addition, peewee encapsulates some operations such as session management and CRUD of basic objects, which greatly simplifies the code. + +The `Player` class is a mapping of a table to attributes in the application. Each attribute of `Player` corresponds to a field in the `player` table. To provide SQLAlchemy with more information, the attribute is defined as `id = Column(String(36), primary_key=True)` to indicate the field type and its additional attributes. For example, `id = Column(String(36), primary_key=True)` indicates that the `id` attribute is `String` type, the corresponding field in database is `VARCHAR` type, the length is `36`, and it is a primary key. + +For more information about how to use peewee, refer to [peewee documentation](http://docs.peewee-orm.com/en/latest/). + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Initialize table + +Before running the code, you need to initialize the table manually. If you are using a local TiDB cluster, you can run the following command: + + + +
+ +```shell +mysql --host 127.0.0.1 --port 4000 -u root < player_init.sql +``` + +
+ +
+ +```shell +mycli --host 127.0.0.1 --port 4000 -u root --no-warn < player_init.sql +``` + +
+ + + +If you are not using a local cluster, or have not installed a MySQL client, connect to your cluster using your preferred method (such as Navicat, DBeaver, or other GUI tools) and run the SQL statements in the `player_init.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, you need to provide your CA root path and replace `` in the following examples with your CA path. To get the CA root path on your system, refer to [Where is the CA root path on my system?](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters#where-is-the-ca-root-path-on-my-system). + +If you are using a TiDB Serverless cluster, modify the parameters of the `connect` function in `peewee_example.py`: + +```python +db = connect('mysql://root:@127.0.0.1:4000/test') +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `connect` as follows: + +- When peewee uses PyMySQL as the driver: + + ```python + db = connect('mysql://2aEp24QWEDLqRFs.root:123456@xxx.tidbcloud.com:4000/test', + ssl_verify_cert=True, ssl_ca="") + ``` + +- When peewee uses mysqlclient as the driver: + + ```python + db = connect('mysql://2aEp24QWEDLqRFs.root:123456@xxx.tidbcloud.com:4000/test', + ssl_mode="VERIFY_IDENTITY", ssl={"ca": ""}) + ``` + +Because peewee will pass parameters to the driver, you need to pay attention to the usage type of the driver when using peewee. + +### Step 3.3 Run the code + +Before running the code, use the following command to install dependencies: + +```bash +pip3 install -r requirement.txt +``` + +If you need to run the script multiple times, follow the [Table initialization](#step-31-initialize-table) section to initialize the table again before each run. + +```bash +python3 peewee_example.py +``` + +## Step 4. Expected output + +[peewee Expected Output](https://github.com/pingcap-inc/tidb-example-python/blob/main/Expected-Output.md#peewee) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-python-pymysql.md b/develop/dev-guide-sample-application-python-pymysql.md new file mode 100644 index 0000000000000..bac670caf7fe4 --- /dev/null +++ b/develop/dev-guide-sample-application-python-pymysql.md @@ -0,0 +1,287 @@ +--- +title: Build a Simple CRUD App with TiDB and PyMySQL +summary: Learn how to build a simple CRUD application with TiDB and PyMySQL. +--- + + + + +# Build a Simple CRUD App with TiDB and PyMySQL + +[PyMySQL](https://pypi.org/project/PyMySQL/) is a popular open-source driver for Python. + +This document describes how to use TiDB and PyMySQL to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Python 3.10 or a later Python version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-python.git +``` + +The following uses PyMySQL 1.0.2 as an example. Drivers for Python are more convenient to use than other languages, but they do not shield the underlying implementation and require manual management of transactions. If there are not a lot of scenarios where SQL is required, it is recommended to use ORM, which can help reduce the coupling of your program. + +```python +import uuid +from typing import List + +import pymysql.cursors +from pymysql import Connection +from pymysql.cursors import DictCursor + + +def get_connection(autocommit: bool = False) -> Connection: + return pymysql.connect(host='127.0.0.1', + port=4000, + user='root', + password='', + database='test', + cursorclass=DictCursor, + autocommit=autocommit) + + +def create_player(cursor: DictCursor, player: tuple) -> None: + cursor.execute("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", player) + + +def get_player(cursor: DictCursor, player_id: str) -> dict: + cursor.execute("SELECT id, coins, goods FROM player WHERE id = %s", (player_id,)) + return cursor.fetchone() + + +def get_players_with_limit(cursor: DictCursor, limit: int) -> tuple: + cursor.execute("SELECT id, coins, goods FROM player LIMIT %s", (limit,)) + return cursor.fetchall() + + +def random_player(amount: int) -> List[tuple]: + players = [] + for _ in range(amount): + players.append((uuid.uuid4(), 10000, 10000)) + + return players + + +def bulk_create_player(cursor: DictCursor, players: List[tuple]) -> None: + cursor.executemany("INSERT INTO player (id, coins, goods) VALUES (%s, %s, %s)", players) + + +def get_count(cursor: DictCursor) -> int: + cursor.execute("SELECT count(*) as count FROM player") + return cursor.fetchone()['count'] + + +def trade_check(cursor: DictCursor, sell_id: str, buy_id: str, amount: int, price: int) -> bool: + get_player_with_lock_sql = "SELECT coins, goods FROM player WHERE id = %s FOR UPDATE" + + # sell player goods check + cursor.execute(get_player_with_lock_sql, (sell_id,)) + seller = cursor.fetchone() + if seller['goods'] < amount: + print(f'sell player {sell_id} goods not enough') + return False + + # buy player coins check + cursor.execute(get_player_with_lock_sql, (buy_id,)) + buyer = cursor.fetchone() + if buyer['coins'] < price: + print(f'buy player {buy_id} coins not enough') + return False + + +def trade_update(cursor: DictCursor, sell_id: str, buy_id: str, amount: int, price: int) -> None: + update_player_sql = "UPDATE player set goods = goods + %s, coins = coins + %s WHERE id = %s" + + # deduct the goods of seller, and raise his/her the coins + cursor.execute(update_player_sql, (-amount, price, sell_id)) + # deduct the coins of buyer, and raise his/her the goods + cursor.execute(update_player_sql, (amount, -price, buy_id)) + + +def trade(connection: Connection, sell_id: str, buy_id: str, amount: int, price: int) -> None: + with connection.cursor() as cursor: + if trade_check(cursor, sell_id, buy_id, amount, price) is False: + connection.rollback() + return + + try: + trade_update(cursor, sell_id, buy_id, amount, price) + except Exception as err: + connection.rollback() + print(f'something went wrong: {err}') + else: + connection.commit() + print("trade success") + + +def simple_example() -> None: + with get_connection(autocommit=True) as connection: + with connection.cursor() as cur: + # create a player, who has a coin and a goods. + create_player(cur, ("test", 1, 1)) + + # get this player, and print it. + test_player = get_player(cur, "test") + print(test_player) + + # create players with bulk inserts. + # insert 1919 players totally, with 114 players per batch. + # each player has a random UUID + player_list = random_player(1919) + for idx in range(0, len(player_list), 114): + bulk_create_player(cur, player_list[idx:idx + 114]) + + # print the number of players + count = get_count(cur) + print(f'number of players: {count}') + + # print 3 players. + three_players = get_players_with_limit(cur, 3) + for player in three_players: + print(player) + + +def trade_example() -> None: + with get_connection(autocommit=False) as connection: + with connection.cursor() as cur: + # create two players + # player 1: id is "1", has only 100 coins. + # player 2: id is "2", has 114514 coins, and 20 goods. + create_player(cur, ("1", 100, 0)) + create_player(cur, ("2", 114514, 20)) + connection.commit() + + # player 1 wants to buy 10 goods from player 2. + # it will cost 500 coins, but player 1 cannot afford it. + # so this trade will fail, and nobody will lose their coins or goods + trade(connection, sell_id="2", buy_id="1", amount=10, price=500) + + # then player 1 has to reduce the incoming quantity to 2. + # this trade will be successful + trade(connection, sell_id="2", buy_id="1", amount=2, price=100) + + # let's take a look for player 1 and player 2 currently + with connection.cursor() as cur: + print(get_player(cur, "1")) + print(get_player(cur, "2")) + + +simple_example() +trade_example() +``` + +The driver has a lower level of encapsulation than ORM, so there are a lot of SQL statements in the program. Unlike ORM, there is no data object in drivers, so the `Player` queried by the driver is represented as a dictionary. + +For more information about how to use PyMySQL, refer to [PyMySQL documentation](https://pymysql.readthedocs.io/en/latest/). + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Initialize table + +Before running the code, you need to initialize the table manually. If you are using a local TiDB cluster, you can run the following command: + + + +
+ +```shell +mysql --host 127.0.0.1 --port 4000 -u root < player_init.sql +``` + +
+ +
+ +```shell +mycli --host 127.0.0.1 --port 4000 -u root --no-warn < player_init.sql +``` + +
+ + + +If you are not using a local cluster, or have not installed a MySQL client, connect to your cluster using your preferred method (such as Navicat, DBeaver, or other GUI tools) and run the SQL statements in the `player_init.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, you need to provide your CA root path and replace `` in the following examples with your CA path. To get the CA root path on your system, refer to [Where is the CA root path on my system?](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters#where-is-the-ca-root-path-on-my-system). + +If you are using a TiDB Serverless cluster, change the `get_connection` function in `pymysql_example.py`: + +```python +def get_connection(autocommit: bool = False) -> Connection: + return pymysql.connect(host='127.0.0.1', + port=4000, + user='root', + password='', + database='test', + cursorclass=DictCursor, + autocommit=autocommit) +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `get_connection` as follows: + +```python +def get_connection(autocommit: bool = False) -> Connection: + return pymysql.connect(host='xxx.tidbcloud.com', + port=4000, + user='2aEp24QWEDLqRFs.root', + password='123546', + database='test', + cursorclass=DictCursor, + autocommit=autocommit, + ssl_ca='', + ssl_verify_cert=True, + ssl_verify_identity=True) +``` + +### Step 3.3 Run the code + +Before running the code, use the following command to install dependencies: + +```bash +pip3 install -r requirement.txt +``` + +If you need to run the script multiple times, follow the [Table initialization](#step-31-initialize-table) section to initialize the table again before each run. + +```bash +python3 pymysql_example.py +``` + +## Step 4. Expected output + +[PyMySQL Expected Output](https://github.com/pingcap-inc/tidb-example-python/blob/main/Expected-Output.md#PyMySQL) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-python-sqlalchemy.md b/develop/dev-guide-sample-application-python-sqlalchemy.md new file mode 100644 index 0000000000000..7e003b803f6fc --- /dev/null +++ b/develop/dev-guide-sample-application-python-sqlalchemy.md @@ -0,0 +1,249 @@ +--- +title: Build a Simple CRUD App with TiDB and SQLAlchemy +summary: Learn how to build a simple CRUD application with TiDB and SQLAlchemy. +aliases: ['/tidb/dev/dev-guide-outdated-for-sqlalchemy'] +--- + + + + +# Build a Simple CRUD App with TiDB and SQLAlchemy + +[SQLAlchemy](https://www.sqlalchemy.org/) is a popular open-source ORM library for Python. + +This document describes how to use TiDB and SQLAlchemy to build a simple CRUD application. + +> **Note:** +> +> It is recommended to use Python 3.10 or a later Python version. + +## Step 1. Launch your TiDB cluster + + + +The following introduces how to start a TiDB cluster. + +**Use a TiDB Serverless cluster** + +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + +**Use a local cluster** + +For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB cluster using TiUP](/production-deployment-using-tiup.md). + + + + + +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). + + + +## Step 2. Get the code + +```shell +git clone https://github.com/pingcap-inc/tidb-example-python.git +``` + +The following uses SQLAlchemy 1.44 as an example. + +```python +import uuid +from typing import List + +from sqlalchemy import create_engine, String, Column, Integer, select, func +from sqlalchemy.orm import declarative_base, sessionmaker + +engine = create_engine('mysql://root:@127.0.0.1:4000/test') +Base = declarative_base() +Base.metadata.create_all(engine) +Session = sessionmaker(bind=engine) + + +class Player(Base): + __tablename__ = "player" + + id = Column(String(36), primary_key=True) + coins = Column(Integer) + goods = Column(Integer) + + def __repr__(self): + return f'Player(id={self.id!r}, coins={self.coins!r}, goods={self.goods!r})' + + +def random_player(amount: int) -> List[Player]: + players = [] + for _ in range(amount): + players.append(Player(id=uuid.uuid4(), coins=10000, goods=10000)) + + return players + + +def simple_example() -> None: + with Session() as session: + # create a player, who has a coin and a goods. + session.add(Player(id="test", coins=1, goods=1)) + + # get this player, and print it. + get_test_stmt = select(Player).where(Player.id == "test") + for player in session.scalars(get_test_stmt): + print(player) + + # create players with bulk inserts. + # insert 1919 players totally, with 114 players per batch. + # each player has a random UUID + player_list = random_player(1919) + for idx in range(0, len(player_list), 114): + session.bulk_save_objects(player_list[idx:idx + 114]) + + # print the number of players + count = session.query(func.count(Player.id)).scalar() + print(f'number of players: {count}') + + # print 3 players. + three_players = session.query(Player).limit(3).all() + for player in three_players: + print(player) + + session.commit() + + +def trade_check(session: Session, sell_id: str, buy_id: str, amount: int, price: int) -> bool: + # sell player goods check + sell_player = session.query(Player.goods).filter(Player.id == sell_id).with_for_update().one() + if sell_player.goods < amount: + print(f'sell player {sell_id} goods not enough') + return False + + # buy player coins check + buy_player = session.query(Player.coins).filter(Player.id == buy_id).with_for_update().one() + if buy_player.coins < price: + print(f'buy player {buy_id} coins not enough') + return False + + +def trade(sell_id: str, buy_id: str, amount: int, price: int) -> None: + with Session() as session: + if trade_check(session, sell_id, buy_id, amount, price) is False: + return + + # deduct the goods of seller, and raise his/her the coins + session.query(Player).filter(Player.id == sell_id). \ + update({'goods': Player.goods - amount, 'coins': Player.coins + price}) + # deduct the coins of buyer, and raise his/her the goods + session.query(Player).filter(Player.id == buy_id). \ + update({'goods': Player.goods + amount, 'coins': Player.coins - price}) + + session.commit() + print("trade success") + + +def trade_example() -> None: + with Session() as session: + # create two players + # player 1: id is "1", has only 100 coins. + # player 2: id is "2", has 114514 coins, and 20 goods. + session.add(Player(id="1", coins=100, goods=0)) + session.add(Player(id="2", coins=114514, goods=20)) + session.commit() + + # player 1 wants to buy 10 goods from player 2. + # it will cost 500 coins, but player 1 cannot afford it. + # so this trade will fail, and nobody will lose their coins or goods + trade(sell_id="2", buy_id="1", amount=10, price=500) + + # then player 1 has to reduce the incoming quantity to 2. + # this trade will be successful + trade(sell_id="2", buy_id="1", amount=2, price=100) + + with Session() as session: + traders = session.query(Player).filter(Player.id.in_(("1", "2"))).all() + for player in traders: + print(player) + session.commit() + + +simple_example() +trade_example() +``` + +Compared with using drivers directly, SQLAlchemy provides an abstraction for the specific details of different databases when you create a database connection. In addition, SQLAlchemy encapsulates some operations such as session management and CRUD of basic objects, which greatly simplifies the code. + +The `Player` class is a mapping of a table to attributes in the application. Each attribute of `Player` corresponds to a field in the `player` table. To provide SQLAlchemy with more information, the attribute is defined as `id = Column(String(36), primary_key=True)` to indicate the field type and its additional attributes. For example, `id = Column(String(36), primary_key=True)` indicates that the `id` attribute is `String` type, the corresponding field in database is `VARCHAR` type, the length is `36`, and it is a primary key. + +For more information about how to use SQLAlchemy, refer to [SQLAlchemy documentation](https://www.sqlalchemy.org/). + +## Step 3. Run the code + +The following content introduces how to run the code step by step. + +### Step 3.1 Initialize table + +Before running the code, you need to initialize the table manually. If you are using a local TiDB cluster, you can run the following command: + + + +
+ +```shell +mysql --host 127.0.0.1 --port 4000 -u root < player_init.sql +``` + +
+ +
+ +```shell +mycli --host 127.0.0.1 --port 4000 -u root --no-warn < player_init.sql +``` + +
+ + + +If you are not using a local cluster, or have not installed a MySQL client, connect to your cluster using your preferred method (such as Navicat, DBeaver, or other GUI tools) and run the SQL statements in the `player_init.sql` file. + +### Step 3.2 Modify parameters for TiDB Cloud + +If you are using a TiDB Serverless cluster, you need to provide your CA root path and replace `` in the following examples with your CA path. To get the CA root path on your system, refer to [Where is the CA root path on my system?](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-tier-clusters#where-is-the-ca-root-path-on-my-system). + +If you are using a TiDB Serverless cluster, modify the parameters of the `create_engine` function in `sqlalchemy_example.py`: + +```python +engine = create_engine('mysql://root:@127.0.0.1:4000/test') +``` + +Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: + +- Endpoint: `xxx.tidbcloud.com` +- Port: `4000` +- User: `2aEp24QWEDLqRFs.root` + +In this case, you can modify the `create_engine` as follows: + +```python +engine = create_engine('mysql://2aEp24QWEDLqRFs.root:123456@xxx.tidbcloud.com:4000/test', connect_args={ + "ssl_mode": "VERIFY_IDENTITY", + "ssl": { + "ca": "" + } +}) +``` + +### Step 3.3 Run the code + +Before running the code, use the following command to install dependencies: + +```bash +pip3 install -r requirement.txt +``` + +If you need to run the script multiple times, follow the [Table initialization](#step-31-initialize-table) section to initialize the table again before each run. + +```bash +python3 sqlalchemy_example.py +``` + +## Step 4. Expected output + +[SQLAlchemy Expected Output](https://github.com/pingcap-inc/tidb-example-python/blob/main/Expected-Output.md#SQLAlchemy) \ No newline at end of file diff --git a/develop/dev-guide-sample-application-spring-boot.md b/develop/dev-guide-sample-application-spring-boot.md index e6c5b95f82faa..543842a1e6e4e 100644 --- a/develop/dev-guide-sample-application-spring-boot.md +++ b/develop/dev-guide-sample-application-spring-boot.md @@ -21,9 +21,9 @@ You can build your own application based on this example. The following introduces how to start a TiDB cluster. -**Use a TiDB Cloud Serverless Tier cluster** +**Use a TiDB Serverless cluster** -For detailed steps, see [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). **Use a local cluster** @@ -33,7 +33,7 @@ For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md# -See [Create a Serverless Tier cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster). +See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). @@ -97,7 +97,7 @@ If you want to learn more about the code of this application, refer to [Implemen ### Step 5.1 Change parameters -If you are using a TiDB Cloud Serverless Tier cluster, change the `spring.datasource.url`, `spring.datasource.username`, `spring.datasource.password` parameters in the `application.yml` (located in `src/main/resources`). +If you are using a TiDB Serverless cluster, change the `spring.datasource.url`, `spring.datasource.username`, `spring.datasource.password` parameters in the `application.yml` (located in `src/main/resources`). ```yaml spring: diff --git a/develop/dev-guide-tidb-crud-sql.md b/develop/dev-guide-tidb-crud-sql.md index cdf1495f10d06..9e0ca6c900e99 100644 --- a/develop/dev-guide-tidb-crud-sql.md +++ b/develop/dev-guide-tidb-crud-sql.md @@ -9,7 +9,7 @@ This document briefly introduces how to use TiDB's CURD SQL. ## Before you start -Please make sure you are connected to a TiDB cluster. If not, refer to [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-serverless-tier-cluster) to create a Serverless Tier cluster. +Please make sure you are connected to a TiDB cluster. If not, refer to [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster. ## Explore SQL with TiDB diff --git a/develop/dev-guide-update-data.md b/develop/dev-guide-update-data.md index b0a96d7f84c5d..2dcb69a80ccbb 100644 --- a/develop/dev-guide-update-data.md +++ b/develop/dev-guide-update-data.md @@ -14,7 +14,7 @@ This document describes how to use the following SQL statements to update the da Before reading this document, you need to prepare the following: -- [Build a TiDB Cluster in TiDB Cloud (Serverless Tier)](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md). - If you want to `UPDATE` data, you need to [insert data](/develop/dev-guide-insert-data.md) first. diff --git a/encryption-at-rest.md b/encryption-at-rest.md index 38b96b9276529..4438b8afeebc7 100644 --- a/encryption-at-rest.md +++ b/encryption-at-rest.md @@ -21,7 +21,7 @@ When a TiDB cluster is deployed, the majority of user data is stored on TiKV and TiKV supports encryption at rest. This feature allows TiKV to transparently encrypt data files using [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) or [SM4](https://en.wikipedia.org/wiki/SM4_(cipher)) in [CTR](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation) mode. To enable encryption at rest, an encryption key must be provided by the user and this key is called master key. TiKV automatically rotates data keys that it used to encrypt actual data files. Manually rotating the master key can be done occasionally. Note that encryption at rest only encrypts data at rest (namely, on disk) and not while data is transferred over network. It is advised to use TLS together with encryption at rest. -Optionally, you can use AWS KMS for both cloud and on-premises deployments. You can also supply the plaintext master key in a file. +Optionally, you can use AWS KMS for both cloud and self-hosted deployments. You can also supply the plaintext master key in a file. TiKV currently does not exclude encryption keys and user data from core dumps. It is advised to disable core dumps for the TiKV process when using encryption at rest. This is not currently handled by TiKV itself. diff --git a/explore-htap.md b/explore-htap.md index 5d32d5cb3d9a2..ec62e5e4be6f4 100644 --- a/explore-htap.md +++ b/explore-htap.md @@ -13,7 +13,7 @@ This guide describes how to explore and use the features of TiDB Hybrid Transact ## Use cases -TiDB HTAP can handle the massive data that increases rapidly, reduce the cost of DevOps, and be deployed in either on-premises or cloud environments easily, which brings the value of data assets in real time. +TiDB HTAP can handle the massive data that increases rapidly, reduce the cost of DevOps, and be deployed in either self-hosted or cloud environments easily, which brings the value of data assets in real time. The following are the typical use cases of HTAP: diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index f2857d6898d57..89e8beaacfcaf 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -20,7 +20,7 @@ Garbage collection is configured via the following system variables: > **Note:** > -> This section is only applicable to on-premises TiDB. TiDB Cloud does not have a GC I/O limit by default. +> This section is only applicable to TiDB Self-Hosted. TiDB Cloud does not have a GC I/O limit by default. @@ -58,7 +58,7 @@ Based on the `DISTRIBUTED` GC mode, the mechanism of GC in Compaction Filter use > **Note:** > -> The following examples of modifying TiKV configurations are only applicable to on-premises TiDB. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. +> The following examples of modifying TiKV configurations are only applicable to TiDB Self-Hosted. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. diff --git a/information-schema/information-schema-resource-groups.md b/information-schema/information-schema-resource-groups.md index 8b094de03661e..20618008182b5 100644 --- a/information-schema/information-schema-resource-groups.md +++ b/information-schema/information-schema-resource-groups.md @@ -13,7 +13,7 @@ summary: Learn the `RESOURCE_GROUPS` information_schema table. > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/information-schema/information-schema-slow-query.md b/information-schema/information-schema-slow-query.md index 8b72f9716316f..49f4ec8607c0a 100644 --- a/information-schema/information-schema-slow-query.md +++ b/information-schema/information-schema-slow-query.md @@ -11,7 +11,7 @@ The `SLOW_QUERY` table provides the slow query information of the current node, > **Note:** > -> The `SLOW_QUERY` table is unavailable for [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> The `SLOW_QUERY` table is unavailable for [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/information-schema/information-schema-user-privileges.md b/information-schema/information-schema-user-privileges.md index e8d1cd6e44f09..e22245751e075 100644 --- a/information-schema/information-schema-user-privileges.md +++ b/information-schema/information-schema-user-privileges.md @@ -73,6 +73,54 @@ The output is as follows: 31 rows in set (0.00 sec) ``` +<<<<<<< HEAD +======= + + + + + + +```sql ++------------+---------------+-------------------------+--------------+ +| GRANTEE | TABLE_CATALOG | PRIVILEGE_TYPE | IS_GRANTABLE | ++------------+---------------+-------------------------+--------------+ +| 'root'@'%' | def | SELECT | YES | +| 'root'@'%' | def | INSERT | YES | +| 'root'@'%' | def | UPDATE | YES | +| 'root'@'%' | def | DELETE | YES | +| 'root'@'%' | def | CREATE | YES | +| 'root'@'%' | def | DROP | YES | +| 'root'@'%' | def | PROCESS | YES | +| 'root'@'%' | def | REFERENCES | YES | +| 'root'@'%' | def | ALTER | YES | +| 'root'@'%' | def | SHOW DATABASES | YES | +| 'root'@'%' | def | SUPER | YES | +| 'root'@'%' | def | EXECUTE | YES | +| 'root'@'%' | def | INDEX | YES | +| 'root'@'%' | def | CREATE USER | YES | +| 'root'@'%' | def | CREATE TABLESPACE | YES | +| 'root'@'%' | def | TRIGGER | YES | +| 'root'@'%' | def | CREATE VIEW | YES | +| 'root'@'%' | def | SHOW VIEW | YES | +| 'root'@'%' | def | CREATE ROLE | YES | +| 'root'@'%' | def | DROP ROLE | YES | +| 'root'@'%' | def | CREATE TEMPORARY TABLES | YES | +| 'root'@'%' | def | LOCK TABLES | YES | +| 'root'@'%' | def | CREATE ROUTINE | YES | +| 'root'@'%' | def | ALTER ROUTINE | YES | +| 'root'@'%' | def | EVENT | YES | +| 'root'@'%' | def | RELOAD | YES | +| 'root'@'%' | def | FILE | YES | +| 'root'@'%' | def | REPLICATION CLIENT | YES | +| 'root'@'%' | def | REPLICATION SLAVE | YES | ++------------+---------------+-------------------------+--------------+ +29 rows in set (0.00 sec) +``` + + + +>>>>>>> 76416ca7e (tidb: rename products (#13692)) Fields in the `USER_PRIVILEGES` table are described as follows: * `GRANTEE`: The name of the granted user, which is in the format of `'user_name'@'host_name'`. diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index b4e60bb25a095..b55fd1095c826 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -14,7 +14,7 @@ This guide walks you through the quickest way to get started with TiDB. For non- > > The deployment method provided in this guide is **ONLY FOR** quick start, **NOT FOR** production. > -> - To deploy an on-premises production cluster, see [production installation guide](/production-deployment-using-tiup.md). +> - To deploy a self-hosted production cluster, see [production installation guide](/production-deployment-using-tiup.md). > - To deploy TiDB on Kubernetes, see [Get Started with TiDB on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/get-started). > - To manage TiDB in the cloud, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart). diff --git a/releases/release-5.2.0.md b/releases/release-5.2.0.md index 91d1c90240dba..ed6f6e0ebe14f 100644 --- a/releases/release-5.2.0.md +++ b/releases/release-5.2.0.md @@ -20,7 +20,7 @@ In v5.2, the key new features and improvements are as follows: - Add the TiFlash I/O traffic limit feature to improve the stability of read and write for TiFlash - TiKV introduces a new flow control mechanism to replace the previous RocksDB write stall mechanism to improve the stability of TiKV flow control - Simplify the operation and maintenance of Data Migration (DM) to reduce the management cost. -- TiCDC supports HTTP protocol OpenAPI to manage TiCDC tasks. It provides a more user-friendly operation method for both Kubernetes and on-premises environments. (Experimental feature) +- TiCDC supports HTTP protocol OpenAPI to manage TiCDC tasks. It provides a more user-friendly operation method for both Kubernetes and self-hosted environments. (Experimental feature) ## Compatibility changes @@ -165,7 +165,7 @@ In v5.2, the key new features and improvements are as follows: ### TiDB data share subscription -TiCDC supports using the HTTP protocol (OpenAPI) to manage TiCDC tasks, which is a more user-friendly operation method for both Kubernetes and on-premises environments. (Experimental feature) +TiCDC supports using the HTTP protocol (OpenAPI) to manage TiCDC tasks, which is a more user-friendly operation method for both Kubernetes and self-hosted environments. (Experimental feature) [#2411](https://github.com/pingcap/tiflow/issues/2411) @@ -210,7 +210,7 @@ Support running the `tiup playground` command on Mac computers with Apple M1 chi - Support completing the garbage collection automatically for the bindings in the "deleted" status [#26206](https://github.com/pingcap/tidb/pull/26206) - Support showing whether a binding is used for query optimization in the result of `EXPLAIN VERBOSE` [#26930](https://github.com/pingcap/tidb/pull/26930) - Add a new status variation `last_plan_binding_update_time` to view the timestamp corresponding to the binding cache in the current TiDB instance [#26340](https://github.com/pingcap/tidb/pull/26340) - - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the on-premises TiDB version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the TiDB Self-Hosted version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + PD diff --git a/releases/release-6.0.0-dmr.md b/releases/release-6.0.0-dmr.md index 565324f36029e..9b8c8211fc528 100644 --- a/releases/release-6.0.0-dmr.md +++ b/releases/release-6.0.0-dmr.md @@ -41,7 +41,7 @@ Starting from TiDB v6.0.0, TiDB provides two types of releases: - Development Milestone Releases - Development Milestone Releases (DMR) are released approximately every two months. A DMR introduces new features and improvements, but does not accept patch releases. It is not recommended for on-premises users to use DMR in production environments. For example, v6.0.0-DMR is a DMR. + Development Milestone Releases (DMR) are released approximately every two months. A DMR introduces new features and improvements, but does not accept patch releases. It is not recommended for users to use DMR in production environments. For example, v6.0.0-DMR is a DMR. TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. @@ -266,7 +266,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. - An enterprise-level database management platform, TiDB Enterprise Manager - TiDB Enterprise Manager (TiEM) is an enterprise-level database management platform based on the TiDB database, which aims to help users manage TiDB clusters in on-premises or public cloud environments. + TiDB Enterprise Manager (TiEM) is an enterprise-level database management platform based on the TiDB database, which aims to help users manage TiDB clusters in self-hosted or public cloud environments. TiEM not only provides full lifecycle visual management for TiDB clusters, but also provides one-stop services: parameter management, version upgrades, cluster clone, active-standby cluster switching, data import and export, data replication, and data backup and restore services. TiEM can improve the efficiency of DevOps on TiDB and reduce the DevOps cost for enterprises. diff --git a/releases/release-7.0.0.md b/releases/release-7.0.0.md new file mode 100644 index 0000000000000..633de81032515 --- /dev/null +++ b/releases/release-7.0.0.md @@ -0,0 +1,510 @@ +--- +title: TiDB 7.0.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 7.0.0. +--- + +# TiDB 7.0.0 Release Notes + +Release date: March 30, 2023 + +TiDB version: 7.0.0-[DMR](/releases/versioning.md#development-milestone-releases) + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.0/quick-start-with-tidb) | [Installation package](https://www.pingcap.com/download/?version=v7.0.0#version-list) + +In v7.0.0-DMR, the key new features and improvements are as follows: + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryFeatureDescription
Scalability and Performance
Session level non-prepared SQL plan cache (experimental)Support automatically reusing plan cache at the session level to reduce compilation and shorten the query time for the same SQL patterns without manually setting prepare statements in advance.
TiFlash supports the disaggregated storage and compute architecture and S3 shared storage (experimental)TiFlash introduces a cloud-native architecture as an option: +
    +
  • Disaggregates TiFlash's compute and storage, which is a milestone for elastic HTAP resource utilization.
    • +
  • Introduces S3-based storage engine, which can provide shared storage at a lower cost.
    • +
+
Reliability and Availability
Resource control enhancement (experimental) Support using resource groups to allocate and isolate resources for various applications or workloads within one cluster. In this release, TiDB adds support for different resource binding modes (user, session, and statement levels) and user-defined priorities. Additionally, you can also use commands to perform resource calibration (estimation for the whole resource amount).
TiFlash supports spill to diskTiFlash supports intermediate result spill to disk to mitigate OOMs in data-intensive operations such as aggregations, sorts, and hash joins.
SQLRow-level TTL (GA)Support managing database size and improve performance by automatically expiring data of a certain age.
Reorganize LIST/RANGE partitionThe REORGANIZE PARTITION statement can be used for merging adjacent partitions or splitting one partition into many, which provides better usability of partitioned tables.
DB Operations and Observability
TiDB enhances the functionalities of LOAD DATA statements (experimental)TiDB enhances the functionalities of LOAD DATA SQL statements, such as supporting data import from S3/GCS.
TiCDC supports object storage sink (GA)TiCDC supports replicating row change events to object storage services, including Amazon S3, GCS, Azure Blob Storage, and NFS.
+ +## Feature details + +### Scalability + +* TiFlash supports the disaggregated storage and compute architecture and supports object storage in this architecture (experimental) [#6882](https://github.com/pingcap/tiflash/issues/6882) @[flowbehappy](https://github.com/flowbehappy) + + Before v7.0.0, TiFlash only supports the coupled storage and compute architecture. In this architecture, each TiFlash node acts as both storage and compute node, and its computing and storage capabilities cannot be independently expanded. In addition, TiFlash nodes can only use local storage. + + Starting from v7.0.0, TiFlash also supports the disaggregated storage and compute architecture. In this architecture, TiFlash nodes are divided into two types (Compute Nodes and Write Nodes) and support object storage that is compatible with S3 API. Both types of nodes can be independently scaled for computing or storage capacities. The **disaggregated storage and compute architecture** and **coupled storage and compute architecture** cannot be used in the same cluster or converted to each other. You can configure which architecture to use when you deploy TiFlash. + + For more information, see [documentation](/tiflash/tiflash-disaggregated-and-s3.md). + +### Performance + +* Achieve compatibility between Fast Online DDL and PITR [#38045](https://github.com/pingcap/tidb/issues/38045) @[Leavrth](https://github.com/Leavrth) + + In TiDB v6.5.0, [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) is not fully compatible with [PITR](/br/backup-and-restore-overview.md). To ensure a full data backup, it is recommended to first stop the PITR background backup task, quickly add indexes using Fast Online DDL, and then resume the PITR backup task. + + Starting from TiDB v7.0.0, Fast Online DDL and PITR are fully compatible. When restoring cluster data through PITR, the index operations added via Fast Online DDL during log backup will be automatically replayed to achieve compatibility. + + For more information, see [documentation](/ddl-introduction.md). + +* TiFlash supports null-aware semi join and null-aware anti semi join operators [#6674](https://github.com/pingcap/tiflash/issues/6674) @[gengliqi](https://github.com/gengliqi) + + When using `IN`, `NOT IN`, `= ANY`, or `!= ALL` operators in correlated subqueries, TiDB optimizes the computing performance by converting them to semi join or anti semi join. If the join key column might be `NULL`, a null-aware join algorithm is required, such as [Null-aware semi join](/explain-subqueries.md#null-aware-semi-join-in-and--any-subqueries) and [Null-aware anti semi join](/explain-subqueries.md#null-aware-anti-semi-join-not-in-and--all-subqueries). + + Before v7.0.0, TiFlash does not support null-aware semi join and null-aware anti semi join operators, preventing these subqueries from being directly pushed down to TiFlash. Starting from v7.0.0, TiFlash supports null-aware semi join and null-aware anti semi join operators. If a SQL statement contains these correlated subqueries, the tables in the query have TiFlash replicas, and [MPP mode](/tiflash/use-tiflash-mpp-mode.md) is enabled, the optimizer automatically determines whether to push down null-aware semi join and null-aware anti semi join operators to TiFlash to improve overall performance. + + For more information, see [documentation](/tiflash/tiflash-supported-pushdown-calculations.md). + +* TiFlash supports using FastScan (GA) [#5252](https://github.com/pingcap/tiflash/issues/5252) @[hongyunyan](https://github.com/hongyunyan) + + Starting from v6.3.0, TiFlash introduces FastScan as an experimental feature. In v7.0.0, this feature becomes generally available. You can enable FastScan using the system variable [`tiflash_fastscan`](/system-variables.md#tiflash_fastscan-new-in-v630). By sacrificing strong consistency, this feature significantly improves table scan performance. If the corresponding table only involves `INSERT` operations without any `UPDATE`/`DELETE` operations, FastScan can keep strong consistency and improve the scan performance. + + For more information, see [documentation](/tiflash/use-fastscan.md). + +* TiFlash supports late materialization (experimental) [#5829](https://github.com/pingcap/tiflash/issues/5829) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + + When processing a `SELECT` statement with filter conditions (`WHERE` clause), TiFlash reads all the data from the columns required by the query by default, and then filters and aggregates the data based on the query conditions. Late materialization is an optimization method that supports pushing down part of the filter conditions to the TableScan operator. That is, TiFlash first scans the column data related to the filter conditions that are pushed down, filters the rows that meet the condition, and then scans the other column data of these rows for further calculation, thereby reducing IO scans and computations of data processing. + + The TiFlash late materialization feature is not enabled by default. You can enable it by setting the `tidb_opt_enable_late_materialization` system variable to `OFF`. When the feature is enabled, the TiDB optimizer will determine which filter conditions to be pushed down based on statistics and filter conditions. + + For more information, see [documentation](/tiflash/tiflash-late-materialization.md). + +* Support caching execution plans for non-prepared statements (experimental) [#36598](https://github.com/pingcap/tidb/issues/36598) @[qw4990](https://github.com/qw4990) + + The execution plan cache is important for improving the load capacity of concurrent OLTP and TiDB already supports [Prepared execution plan cache](/sql-prepared-plan-cache.md). In v7.0.0, TiDB can also cache execution plans for non-Prepare statements, expanding the scope of execution plan cache and improving the concurrent processing capacity of TiDB. + + This feature is disabled by default. You can enable it by setting the system variable [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache) to `ON`. For stability reasons, TiDB v7.0.0 allocates a new area for caching non-prepared execution plans and you can set the cache size using the system variable [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size). Additionally, this feature has certain restrictions on SQL statements. For more information, see [Restrictions](/sql-non-prepared-plan-cache.md#restrictions). + + For more information, see [documentation](/sql-non-prepared-plan-cache.md). + +* TiDB removes the execution plan cache constraint for subqueries [#40219](https://github.com/pingcap/tidb/issues/40219) @[fzzf678](https://github.com/fzzf678) + + TiDB v7.0.0 removes the execution plan cache constraint for subqueries. This means that the execution plan of SQL statements with subqueries can now be cached, such as `SELECT * FROM t WHERE a > (SELECT ...)`. This feature further expands the application scope of execution plan cache and improves the execution efficiency of SQL queries. + + For more information, see [documentation](/sql-prepared-plan-cache.md). + +* TiKV supports automatically generating empty log files for log recycling [#14371](https://github.com/tikv/tikv/issues/14371) @[LykxSassinator](https://github.com/LykxSassinator) + + In v6.3.0, TiKV introduced the [Raft log recycling](/tikv-configuration-file.md#enable-log-recycle-new-in-v630) feature to reduce long-tail latency caused by write load. However, log recycling can only take effect when the number of Raft log files reaches a certain threshold, making it difficult for users to directly experience the throughput improvement brought by this feature. + + In v7.0.0, a new configuration item called `raft-engine.prefill-for-recycle` was introduced to improve user experience. This item controls whether empty log files are generated for recycling when the process starts. When this configuration is enabled, TiKV automatically fills a batch of empty log files during initialization, ensuring that log recycling takes effect immediately after initialization. + + For more information, see [documentation](/tikv-configuration-file.md#prefill-for-recycle-new-in-v700). + +* Support deriving the TopN or Limit operator from [window functions](/functions-and-operators/expressions-pushed-down.md) to improve window function performance [#13936](https://github.com/tikv/tikv/issues/13936) @[windtalker](https://github.com/windtalker) + + This feature is disabled by default. To enable it, you can set the session variable [tidb_opt_derive_topn](/system-variables.md#tidb_opt_derive_topn-new-in-v700) to `ON`. + + For more information, see [documentation](/derive-topn-from-window.md). + +* Support creating unique indexes through Fast Online DDL [#40730](https://github.com/pingcap/tidb/issues/40730) @[tangenta](https://github.com/tangenta) + + TiDB v6.5.0 supports creating ordinary secondary indexes via Fast Online DDL. TiDB v7.0.0 supports creating unique indexes via Fast Online DDL. Compared to v6.1.0, adding unique indexes to large tables is expected to be several times faster with improved performance. + + For more information, see [documentation](/ddl-introduction.md). + +### Reliability + +* Enhance the resource control feature (experimental) [#38825](https://github.com/pingcap/tidb/issues/38825) @[nolouch](https://github.com/nolouch) @[BornChanger](https://github.com/BornChanger) @[glorv](https://github.com/glorv) @[tiancaiamao](https://github.com/tiancaiamao) @[Connor1996](https://github.com/Connor1996) @[JmPotato](https://github.com/JmPotato) @[hnes](https://github.com/hnes) @[CabinfeverB](https://github.com/CabinfeverB) @[HuSharp](https://github.com/HuSharp) + + TiDB enhances the resource control feature based on resource groups. This feature significantly improves the resource utilization efficiency and performance of TiDB clusters. The introduction of the resource control feature is a milestone for TiDB. You can divide a distributed database cluster into multiple logical units, map different database users to corresponding resource groups, and set the quota for each resource group as needed. When the cluster resources are limited, all resources used by sessions in the same resource group are limited to the quota. In this way, even if a resource group is over-consumed, the sessions in other resource groups are not affected. + + With this feature, you can combine multiple small and medium-sized applications from different systems into a single TiDB cluster. When the workload of an application grows larger, it does not affect the normal operation of other applications. When the system workload is low, busy applications can still be allocated the required system resources even if they exceed the set quotas, so as to achieve the maximum utilization of resources. In addition, the rational use of the resource control feature can reduce the number of clusters, ease the difficulty of operation and maintenance, and save management costs. + + This feature provides a built-in Resource Control Dashboard for the actual usage of resources in Grafana, assisting you to allocate resources more rationally. It also supports dynamic resource management capabilities based on both session and statement levels (Hint). The introduction of this feature will help you gain more precise control over the resource usage of your TiDB cluster, and dynamically adjust quotas based on actual needs. + + In TiDB v7.0.0, you can set the absolute scheduling priority (`PRIORITY`) for resource groups to guarantee that important services can get resources. It also extends the way to set resource groups. + + You can use resource groups in the following ways: + + - User level. Bind a user using the [`CREATE USER`](/sql-statements/sql-statement-create-user.md) or [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) statements to a specific resource group. After binding a resource group to a user, sessions newly created by the user are automatically bound to the corresponding resource group. + - Session level. Set the resource group used by the current session via [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md). + - Statement level. Set the resource group used by the current statement via [`RESOURCE_GROUP()`](/optimizer-hints.md#resource_groupresource_group_name). + + For more information, see [documentation](/tidb-resource-control.md). + +* Support a checkpoint mechanism for Fast Online DDL, improving fault tolerance and automatic recovery capability [#42164](https://github.com/pingcap/tidb/issues/42164) @[tangenta](https://github.com/tangenta) + + TiDB v7.0.0 introduces a checkpoint mechanism for [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), which significantly improves its fault tolerance and automatic recovery capabilities. By periodically recording and synchronizing the DDL progress, ongoing DDL operations can continue to be executed in Fast Online DDL mode even if there is a TiDB DDL Owner failure or switch. This makes the execution of DDL more stable and efficient. + + For more information, see [documentation](/ddl-introduction.md). + +* TiFlash supports spilling to disk [#6528](https://github.com/pingcap/tiflash/issues/6528) @[windtalker](https://github.com/windtalker) + + To improve execution performance, TiFlash runs data entirely in memory as much as possible. When the amount of data exceeds the total size of memory, TiFlash terminates the query to avoid system crashes caused by running out of memory. Therefore, the amount of data that TiFlash can handle is limited by the available memory. + + Starting from v7.0.0, TiFlash supports spilling to disk. By adjusting the threshold of memory usage for operators ([`tidb_max_bytes_before_tiflash_external_group_by`](/system-variables.md#tidb_max_bytes_before_tiflash_external_group_by-new-in-v700), [`tidb_max_bytes_before_tiflash_external_sort`](/system-variables.md#tidb_max_bytes_before_tiflash_external_sort-new-in-v700), and [`tidb_max_bytes_before_tiflash_external_join`](/system-variables.md#tidb_max_bytes_before_tiflash_external_join-new-in-v700)), you can control the maximum amount of memory that an operator can use. When the memory used by the operator exceeds the threshold, it automatically writes data to disk. This sacrifices some performance but allows for processing of more data. + + For more information, see [documentation](/tiflash/tiflash-spill-disk.md). + +* Improve the efficiency of collecting statistics [#41930](https://github.com/pingcap/tidb/issues/41930) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + + In v7.0.0, TiDB further optimizes the logic of collecting statistics, reducing the collection time by about 25%. This optimization improves the operational efficiency and stability of large database clusters, reducing the impact of statistics collection on cluster performance. + +* Add new optimizer hints for MPP optimization [#39710](https://github.com/pingcap/tidb/issues/39710) @[Reminiscent](https://github.com/Reminiscent) + + In v7.0.0, TiDB adds a series of optimizer hints to influence the generation of MPP execution plans. + + - [`SHUFFLE_JOIN()`](/optimizer-hints.md#shuffle_joint1_name--tl_name-): takes effect on MPP. It hints the optimizer to use the Shuffle Join algorithm for the specified table. + - [`BROADCAST_JOIN()`](/optimizer-hints.md#broadcast_joint1_name--tl_name-): takes effect on MPP. It hints the optimizer to use the Broadcast Join algorithm for the specified table. + - [`MPP_1PHASE_AGG()`](/optimizer-hints.md#mpp_1phase_agg): takes effect on MPP. It hints the optimizer to use the one-phase aggregation algorithm for all aggregate functions in the specified query block. + - [`MPP_2PHASE_AGG()`](/optimizer-hints.md#mpp_2phase_agg): takes effect on MPP. It hints the optimizer to use the two-phase aggregation algorithm for all aggregate functions in the specified query block. + + MPP optimizer hints can help you intervene in HTAP queries, improving performance and stability for HTAP workloads. + + For more information, see [documentation](/optimizer-hints.md). + +* Optimizer hints support specifying join methods and join orders [#36600](https://github.com/pingcap/tidb/issues/36600) @[Reminiscent](https://github.com/Reminiscent) + + In v7.0.0, the optimizer hint [`LEADING()`](/optimizer-hints.md#leadingt1_name--tl_name-) can be used in conjunction with hints that affect the join method, and their behaviors are compatible. In the case of multi-table joins, you can effectively specify the optimal join method and join order, thereby enhancing the control of optimizer hints over execution plans. + + The new hint behavior has minor changes. To ensure forward compatibility, TiDB introduces the system variable [`tidb_opt_advanced_join_hint`](/system-variables.md#tidb_opt_advanced_join_hint-new-in-v700). When this variable is set to `OFF`, the optimizer hint behavior is compatible with earlier versions. When you upgrade your cluster from earlier versions to v7.0.0 or later versions, this variable will be set to `OFF`. To obtain more flexible hint behavior, after you confirm that the behavior does not cause a performance regression, it is strongly recommended to set this variable to `ON`. + + For more information, see [documentation](/optimizer-hints.md). + +### Availability + +* Support the `prefer-leader` option, which provides higher availability for read operations and reduces response latency in unstable network conditions [#40905](https://github.com/pingcap/tidb/issues/40905) @[LykxSassinator](https://github.com/LykxSassinator) + + You can control TiDB's data reading behavior through the system variable [`tidb_replica_read`](/system-variables.md#tidb_replica_read-new-in-v40). In v7.0.0, this variable adds the `prefer-leader` option. When the variable is set to `prefer-leader`, TiDB prioritizes selecting the leader replica to perform read operations. When the processing speed of the leader replica slows down significantly, such as due to disk or network performance fluctuations, TiDB selects other available follower replicas to perform read operations, providing higher availability and reducing response latency. + + For more information, see [documentation](/develop/dev-guide-use-follower-read.md). + +### SQL + +* Time to live (TTL) is generally available [#39262](https://github.com/pingcap/tidb/issues/39262) @[lcwangchao](https://github.com/lcwangchao) @[YangKeao](https://github.com/YangKeao) + + TTL provides row-level lifecycle control policies. In TiDB, tables with TTL attributes set automatically check and delete expired row data based on the configuration. The goal of TTL is to help users periodically clean up unnecessary data in time while minimizing the impact on cluster workloads. + + For more information, see [documentation](/time-to-live.md). + +* Support `ALTER TABLE…REORGANIZE PARTITION` [#15000](https://github.com/pingcap/tidb/issues/15000) @[mjonss](https://github.com/mjonss) + + TiDB supports the `ALTER TABLE...REORGANIZE PARTITION` syntax. Using this syntax, you can reorganize some or all of the partitions of a table, including merging, splitting, or other modifications, without losing data. + + For more information, see [documentation](/partitioned-table.md#reorganize-partitions). + +* Support Key partitioning [#41364](https://github.com/pingcap/tidb/issues/41364) @[TonsnakeLin](https://github.com/TonsnakeLin) + + Now TiDB supports Key partitioning. Both Key partitioning and Hash partitioning can evenly distribute data into a certain number of partitions. The difference is that Hash partitioning only supports distributing data based on a specified integer expression or an integer column, while Key partitioning supports distributing data based on a column list, and partitioning columns of Key partitioning are not limited to the integer type. + + For more information, see [documentation](/partitioned-table.md#key-partitioning). + +### DB operations + +* TiCDC supports replicating change data to storage services (GA) [#6797](https://github.com/pingcap/tiflow/issues/6797) @[zhaoxinyu](https://github.com/zhaoxinyu) + + TiCDC supports replicating changed data to Amazon S3, GCS, Azure Blob Storage, NFS, and other S3-compatible storage services. Storage services are reasonably priced and easy to use. If you are not using Kafka, you can use storage services. TiCDC saves the changed logs to a file and then sends it to the storage services instead. From the storage services, your own consumer program can read the newly generated changed log files periodically. Currently, TiCDC supports replicating changed logs in canal-json and CSV formats to the storage service. + + For more information, see [documentation](/ticdc/ticdc-sink-to-cloud-storage.md). + +* TiCDC OpenAPI v2 [#8019](https://github.com/pingcap/tiflow/issues/8019) @[sdojjy](https://github.com/sdojjy) + + TiCDC provides OpenAPI v2. Compared with OpenAPI v1, OpenAPI v2 provides more comprehensive support for replication tasks. The features provided by TiCDC OpenAPI are a subset of the [`cdc cli` tool](/ticdc/ticdc-manage-changefeed.md). You can query and operate TiCDC clusters via OpenAPI v2, such as getting TiCDC node status, checking cluster health status, and managing replication tasks. + + For more information, see [documentation](/ticdc/ticdc-open-api-v2.md). + +* [DBeaver](https://dbeaver.io/) v23.0.1 supports TiDB by default [#17396](https://github.com/dbeaver/dbeaver/issues/17396) @[Icemap](https://github.com/Icemap) + + - Provides an independent TiDB module, icon, and logo. + - The default configuration supports [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta), making it easier to connect to TiDB Serverless. + - Supports identifying TiDB versions to display or hide foreign key tabs. + - Supports visualizing SQL execution plans in `EXPLAIN` results. + - Supports highlighting TiDB keywords such as `PESSIMISTIC`, `OPTIMISTIC`, `AUTO_RANDOM`, `PLACEMENT`, `POLICY`, `REORGANIZE`, `EXCHANGE`, `CACHE`, `NONCLUSTERED`, and `CLUSTERED`. + - Supports highlighting TiDB functions such as  `TIDB_BOUNDED_STALENESS`, `TIDB_DECODE_KEY`, `TIDB_DECODE_PLAN`, `TIDB_IS_DDL_OWNER`, `TIDB_PARSE_TSO`, `TIDB_VERSION`, `TIDB_DECODE_SQL_DIGESTS`, and `TIDB_SHARD`. + + For more information, see [DBeaver documentation](https://github.com/dbeaver/dbeaver/wiki). + +### Data migration + +* Enhance the functionalities of `LOAD DATA` statements and support importing data from cloud storage (experimental) [#40499](https://github.com/pingcap/tidb/issues/40499) @[lance6716](https://github.com/lance6716) + + Before TiDB v7.0.0, the `LOAD DATA` statement could only import data files from the client side. If you wanted to import data from cloud storage, you had to rely on TiDB Lightning. However, deploying TiDB Lightning separately would bring additional deployment and management costs. In v7.0.0, you can directly import data from cloud storage using the `LOAD DATA` statement. Some examples of the feature are as follows: + + - Supports importing data from Amazon S3 and Google Cloud Storage to TiDB. Supports importing multiple source files to TiDB in one go with wildcards. + - Support using `DEFINED NULL BY` to define null. + - Support source files in CSV and TSV formats. + + For more information, see [documentation](/sql-statements/sql-statement-load-data.md). + +* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[gozssky](https://github.com/gozssky) + + Starting from v6.6.0, TiDB Lightning supports compressing locally encoded and sorted key-value pairs for network transfer when sending them to TiKV, thus reducing the amount of data transferred over the network and lowering the network bandwidth overhead. In the earlier TiDB versions before this feature is supported, TiDB Lightning requires relatively high network bandwidth and incurs high traffic charges in case of large data volumes. + + In v7.0.0, this feature becomes GA and is disabled by default. To enable it, you can set the `compress-kv-pairs` configuration item of TiDB Lightning to `"gzip"` or `"gz"`. + + For more information, see [documentation](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). + +## Compatibility changes + +> **Note:** +> +> This section provides compatibility changes you need to know when you upgrade from v6.6.0 to the current version (v7.0.0). If you are upgrading from v6.5.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. + +### MySQL compatibility + +* TiDB removes the constraint that the auto-increment column must be an index [#40580](https://github.com/pingcap/tidb/issues/40580) @[tiancaiamao](https://github.com/tiancaiamao) + + Before v7.0.0, TiDB's behavior is consistent with MySQL, requiring the auto-increment column to be an index or index prefix. Starting from v7.0.0, TiDB removes the constraint that the auto-increment column must be an index or index prefix. Now you can define the primary key of a table more flexibly and use the auto-increment column to implement sorting and pagination more conveniently. This also avoids the write hotspot problem caused by the auto-increment column and improves query performance by using the table with clustered indexes. With the new release, you can create a table using the following syntax: + + ```sql + CREATE TABLE test1 ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `k` int(11) NOT NULL DEFAULT '0', + `c` char(120) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + PRIMARY KEY(`k`, `id`) + ); + ``` + + This feature does not affect TiCDC data replication. + + For more information, see [documentation](/mysql-compatibility.md#auto-increment-id). + +* TiDB supports Key partitions, as shown in the following example: + + ```sql + CREATE TABLE employees ( + id INT NOT NULL, + fname VARCHAR(30), + lname VARCHAR(30), + hired DATE NOT NULL DEFAULT '1970-01-01', + separated DATE DEFAULT '9999-12-31', + job_code INT, + store_id INT) PARTITION BY KEY(store_id) PARTITIONS 4; + ``` + + Starting from v7.0.0, TiDB supports Key partitions and can parse the MySQL `PARTITION BY LINEAR KEY` syntax. However, TiDB ignores the `LINEAR` keyword and uses a non-linear hash algorithm instead. Currently, the `KEY` partition type does not support partition statements with an empty partition column list. + + For more information, see [documentation](/partitioned-table.md#key-partitioning). + +### Behavior changes + +* TiCDC fixes the issue of incorrect encoding of `FLOAT` data in Avro [#8490](https://github.com/pingcap/tiflow/issues/8490) @[3AceShowHand](https://github.com/3AceShowHand) + + When upgrading the TiCDC cluster to v7.0.0, if a table replicated using Avro contains the `FLOAT` data type, you need to manually adjust the compatibility policy of Confluent Schema Registry to `None` before upgrading so that the changefeed can successfully update the schema. Otherwise, after upgrading, the changefeed will be unable to update the schema and enter an error state. + +* Starting from v7.0.0, [`tidb_dml_batch_size`](/system-variables.md#tidb_dml_batch_size) system variable no longer takes effect on the [`LOAD DATA` statement](/sql-statements/sql-statement-load-data.md). + +### System variables + +| Variable name | Change type | Description | +|--------|------------------------------|------| +| `tidb_pessimistic_txn_aggressive_locking` | Deleted | This variable is renamed to [`tidb_pessimistic_txn_fair_locking`](/system-variables.md#tidb_pessimistic_txn_fair_locking-new-in-v700). | +| [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache) | Modified | Takes effect starting from v7.0.0 and controls whether to enable the [Non-prepared plan cache](/sql-non-prepared-plan-cache.md) feature. | +| [`tidb_enable_null_aware_anti_join`](/system-variables.md#tidb_enable_null_aware_anti_join-new-in-v630) | Modified | Changes the default value from `OFF` to `ON` after further tests, meaning that TiDB applies Null-Aware Hash Join when Anti Join is generated by subqueries led by special set operators `NOT IN` and `!= ALL` by default. | +| [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660) | Modified | Changes the default value from `OFF` to `ON`, meaning that the cluster isolates resources by resource group by default. Resource Control is enabled by default in v7.0.0, so that you can use this feature whenever you want. | +| [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size) | Modified | Takes effect starting from v7.0.0 and controls the maximum number of execution plans that can be cached by [Non-prepared plan cache](/sql-non-prepared-plan-cache.md). | +| [`tidb_rc_read_check_ts`](/system-variables.md#tidb_rc_read_check_ts-new-in-v600) | Modified | Starting from v7.0.0, this variable is no longer effective for cursor fetch read in the prepared statement protocol. | +| [`tidb_enable_inl_join_inner_multi_pattern`](/system-variables.md#tidb_enable_inl_join_inner_multi_pattern-new-in-v700) | Newly added | This variable controls whether Index Join is supported when the inner table has `Selection` or `Projection` operators on it. | +| [`tidb_enable_plan_cache_for_subquery`](/system-variables.md#tidb_enable_plan_cache_for_subquery-new-in-v700) | Newly added | This variable controls whether Prepared Plan Cache caches queries that contain subqueries. | +| [`tidb_enable_plan_replayer_continuous_capture`](/system-variables.md#tidb_enable_plan_replayer_continuous_capture-new-in-v700) | Newly added | This variable controls whether to enable the [`PLAN REPLAYER CONTINUOUS CAPTURE`](/sql-plan-replayer.md#use-plan-replayer-continuous-capture) feature. The default value `OFF` means to disable the feature. | +| [`tidb_load_based_replica_read_threshold`](/system-variables.md#tidb_load_based_replica_read_threshold-new-in-v700) | Newly added | This variable sets the threshold for triggering load-based replica read. The feature controlled by this variable is not fully functional in TiDB v7.0.0. Do not change the default value. | +| [`tidb_opt_advanced_join_hint`](/system-variables.md#tidb_opt_advanced_join_hint-new-in-v700) | Newly added | This variable controls whether the join method hint influences the optimization of join reorder. The default value is `ON`, which means the new compatible control mode is used. The value `OFF` means the behavior before v7.0.0 is used. For forward compatibility, the value of this variable is set to `OFF` when the cluster is upgraded from an earlier version to v7.0.0 or later. | +| [`tidb_opt_derive_topn`](/system-variables.md#tidb_opt_derive_topn-new-in-v700) | Newly added | This variable controls whether to enable the [Derive TopN or Limit from Window Functions](/derive-topn-from-window.md) optimization rule. The default value is `OFF`, which means the optimization rule is not enabled. | +| [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) | Newly added | This variable controls whether to enable the [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) feature. The default value is `OFF`, which means the feature is not enabled. | +| [`tidb_opt_ordering_index_selectivity_threshold`](/system-variables.md#tidb_opt_ordering_index_selectivity_threshold-new-in-v700) | Newly added | This variable controls how the optimizer selects indexes when the SQL statement contains `ORDER BY` and `LIMIT` clauses and has filtering conditions. | +| [`tidb_pessimistic_txn_fair_locking`](/system-variables.md#tidb_pessimistic_txn_fair_locking-new-in-v700) | Newly added | Controls whether to enable the enhanced pessimistic lock-waking model to reduce the tail latency of transactions under single-row conflict scenarios. The default value is `ON`. When the cluster is upgraded from an earlier version to v7.0.0 or later, the value of this variable is set to `OFF`. | +| [`tidb_ttl_running_tasks`](/system-variables.md#tidb_ttl_running_tasks-new-in-v700) | Newly added | This variable is used to limit the concurrency of TTL tasks across the entire cluster. The default value `-1` means that the TTL tasks are the same as the number of TiKV nodes. | + +### Configuration file parameters + +| Configuration file | Configuration parameter | Change type | Description | +| -------- | -------- | -------- | -------- | +| TiKV | `server.snap-max-write-bytes-per-sec` | Deleted | This parameter is renamed to [`server.snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec). | +| TiKV | [`raft-engine.enable-log-recycle`](/tikv-configuration-file.md#enable-log-recycle-new-in-v630) | Modified | The default value changes from `false` to `true`. | +| TiKV | [`resolved-ts.advance-ts-interval`](/tikv-configuration-file.md#advance-ts-interval) | Modified | The default value changes from `"1s"` to `"20s"`. This modification can increase the interval of the regular advancement of Resolved TS and reduce the traffic consumption between TiKV nodes. | +| TiKV | [`resource-control.enabled`](/tikv-configuration-file.md#resource-control) | Modified | The default value changes from `false` to `true`. | +| TiKV | [`raft-engine.prefill-for-recycle`](/tikv-configuration-file.md#prefill-for-recycle-new-in-v700) | Newly added | Controls whether to generate empty log files for log recycling in Raft Engine. The default value is `false`. | +| PD | [`degraded-mode-wait-duration`](/pd-configuration-file.md#degraded-mode-wait-duration) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the waiting time for triggering the degraded mode. The default value is `0s`. | +| PD | [`read-base-cost`](/pd-configuration-file.md#read-base-cost) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the basis factor for conversion from a read request to RU. The default value is `0.25`. | +| PD | [`read-cost-per-byte`](/pd-configuration-file.md#read-cost-per-byte) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the basis factor for conversion from read flow to RU. The default value is `1/ (64 * 1024)`. | +| PD | [`read-cpu-ms-cost`](/pd-configuration-file.md#read-cpu-ms-cost) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the basis factor for conversion from CPU to RU. The default value is `1/3`. | +| PD | [`write-base-cost`](/pd-configuration-file.md#write-base-cost) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the basis factor for conversion from a write request to RU. The default value is `1`. | +| PD | [`write-cost-per-byte`](/pd-configuration-file.md#write-cost-per-byte) | Newly added | A [Resource Control](/tidb-resource-control.md)-related configuration item. It controls the basis factor for conversion from write flow to RU. The default value is `1/1024`. | +| TiFlash | [`flash.disaggregated_mode`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | In the disaggregated architecture of TiFlash, it indicates whether this TiFlash node is a write node or a compute node. The value can be `tiflash_write` or `tiflash_compute`. | +| TiFlash | [`storage.s3.endpoint`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | The endpoint to connect to S3. | +| TiFlash | [`storage.s3.bucket`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | The bucket where TiFlash stores all data. | +| TiFlash | [`storage.s3.root`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | The root directory of data storage in S3 bucket. | +| TiFlash | [`storage.s3.access_key_id`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | `ACCESS_KEY_ID` for accessing S3. | +| TiFlash | [`storage.s3.secret_access_key`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | `SECRET_ACCESS_KEY` for accessing S3. | +| TiFlash | [`storage.remote.cache.dir`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | The local data cache directory of TiFlash compute node. | +| TiFlash | [`storage.remote.cache.capacity`](/tiflash/tiflash-disaggregated-and-s3.md) | Newly added | The size of the local data cache directory of TiFlash compute node. | +| TiDB Lightning | [`add-index-by-sql`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Newly added | Controls whether to use SQL to add indexes in physical import mode. The default value is `false`, which means that TiDB Lightning will encode both row data and index data into KV pairs and import them into TiKV together. The advantage of adding indexes using SQL is to separate the import of data and the import of indexes, which can quickly import data. Even if the index creation fails after the data is imported, the data consistency is not affected. | +| TiCDC | [`enable-table-across-nodes`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Determines whether to divide a table into multiple sync ranges according to the number of Regions. These ranges can be replicated by multiple TiCDC nodes. | +| TiCDC | [`region-threshold`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | When `enable-table-across-nodes` is enabled, this feature only takes effect on tables with more than `region-threshold` Regions. | +| DM | [`analyze`](/dm/task-configuration-file-full.md#task-configuration-file-template-advanced) | Newly added | Controls whether to execute the `ANALYZE TABLE ` operation on each table after CHECKSUM is completed. It can be configured as `"required"`/`"optional"`/`"off"`. The default value is `"optional"`. | +| DM | [`range-concurrency`](/dm/task-configuration-file-full.md#task-configuration-file-template-advanced) | Newly added | Controls the concurrency of dm-worker writing KV data to TiKV. | +| DM | [`compress-kv-pairs`](/dm/task-configuration-file-full.md#task-configuration-file-template-advanced) | Newly added | Controls whether to enable compression when dm-worker sends KV data to TiKV. Currently, only gzip is supported. The default value is empty, which means no compression. | +| DM | [`pd-addr`](/dm/task-configuration-file-full.md#task-configuration-file-template-advanced) | Newly added | Controls the address of the downstream PD server in the Physical Import mode. You can fill in either one or more PD servers. If this configuration item is empty, use the PD address information from the TiDB query by default. | + +## Improvements + ++ TiDB + + - Introduce the `EXPAND` operator to optimize the performance of SQL queries with multiple `DISTINCT` in a single `SELECT` statement [#16581](https://github.com/pingcap/tidb/issues/16581) @[AilinKid](https://github.com/AilinKid) + - Support more SQL formats for Index Join [#40505](https://github.com/pingcap/tidb/issues/40505) @[Yisaer](https://github.com/Yisaer) + - Avoid globally sorting partitioned table data in TiDB in some cases [#26166](https://github.com/pingcap/tidb/issues/26166) @[Defined2014](https://github.com/Defined2014) + - Support using `fair lock mode` and `lock only if exists` at the same time [#42068](https://github.com/pingcap/tidb/issues/42068) @[MyonKeminta](https://github.com/MyonKeminta) + - Support printing transaction slow logs and transaction internal events [#41863](https://github.com/pingcap/tidb/issues/41863) @[ekexium](https://github.com/ekexium) + - Support the `ILIKE` operator [#40943](https://github.com/pingcap/tidb/issues/40943) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ PD + + - Add a new monitoring metric for scheduling failures due to the store limit [#6043](https://github.com/tikv/pd/issues/6043) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Reduce TiFlash's memory usage on write path [#7144](https://github.com/pingcap/tiflash/issues/7144) @[hongyunyan](https://github.com/hongyunyan) + - Reduce TiFlash's restart time in scenarios with many tables [#7146](https://github.com/pingcap/tiflash/issues/7146) @[hongyunyan](https://github.com/hongyunyan) + - Support pushing down the `ILIKE` operator [#6740](https://github.com/pingcap/tiflash/issues/6740) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ Tools + + + TiCDC + + - Support distributing data changes of a single large table to multiple TiCDC nodes in scenarios where Kafka is the downstream, thus solving the scalability issue of single tables in data integration scenarios of large-scale TiDB clusters [#8247](https://github.com/pingcap/tiflow/issues/8247) @[overvenus](https://github.com/overvenus) + + You can enable this feature by setting the TiCDC configuration item `enable_table_across_nodes` to `true`. You can use `region_threshold` to specify that when the number of Regions for a table exceeds this threshold, TiCDC starts distributing data changes of the corresponding table to multiple TiCDC nodes. + + - Support splitting transactions in the redo applier to improve its throughput and reduce RTO in disaster recovery scenarios [#8318](https://github.com/pingcap/tiflow/issues/8318) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Improve the table scheduling to split a single table more evenly across various TiCDC nodes [#8247](https://github.com/pingcap/tiflow/issues/8247) @[overvenus](https://github.com/overvenus) + - Add the Large Row monitoring metrics in MQ sink [#8286](https://github.com/pingcap/tiflow/issues/8286) @[hi-rustin](https://github.com/hi-rustin) + - Reduce network traffic between TiKV and TiCDC nodes in scenarios where a Region contains data of multiple tables [#6346](https://github.com/pingcap/tiflow/issues/6346) @[overvenus](https://github.com/overvenus) + - Move the P99 metrics panel of Checkpoint TS and Resolved TS to the Lag analyze panel [#8524](https://github.com/pingcap/tiflow/issues/8524) @[hi-rustin](https://github.com/hi-rustin) + - Support applying DDL events in redo logs [#8361](https://github.com/pingcap/tiflow/issues/8361) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Support splitting and scheduling tables to TiCDC nodes based on upstream write throughput [#7720](https://github.com/pingcap/tiflow/issues/7720) @[overvenus](https://github.com/overvenus) + + + TiDB Lightning + + - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[gozssky](https://github.com/gozssky) + + Add the `add-index-by-sql` parameter. The default value is `false`, which means that TiDB Lightning encodes both row data and index data into KV pairs and import them into TiKV together. If you set it to `true`, it means that TiDB Lightning adds indexes via the `ADD INDEX` SQL statement after importing the row data to improve import speed and stability. + + - Add the `tikv-importer.keyspace-name` parameter. The default value is an empty string, which means TiDB Lightning automatically gets the key space name of the corresponding tenant to import data. If you specify a value, the specified key space name will be used to import data. This parameter provides flexibility in the configuration of TiDB Lightning when you import data to a multi-tenant TiDB cluster. [#41915](https://github.com/pingcap/tidb/issues/41915) @[lichunzhu](https://github.com/lichunzhu) + +## Bug fixes + ++ TiDB + + - Fix the issue of missing updates when upgrading TiDB from v6.5.1 to a later version [#41502](https://github.com/pingcap/tidb/issues/41502) @[chrysan](https://github.com/chrysan) + - Fix the issue that the default values of some system variables are not modified after upgrading [#41423](https://github.com/pingcap/tidb/issues/41423) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that Coprocessor request types related to adding indexes are displayed as unknown [#41400](https://github.com/pingcap/tidb/issues/41400) @[tangenta](https://github.com/tangenta) + - Fix the issue of returning "PessimisticLockNotFound" when adding an index [#41515](https://github.com/pingcap/tidb/issues/41515) @[tangenta](https://github.com/tangenta) + - Fix the issue of mistakenly returning `found duplicate key` when adding a unique index [#41630](https://github.com/pingcap/tidb/issues/41630) @[tangenta](https://github.com/tangenta) + - Fix the panic issue when adding an index [#41880](https://github.com/pingcap/tidb/issues/41880) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiFlash reports an error for generated columns during execution [#40663](https://github.com/pingcap/tidb/issues/40663) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB might not be able to obtain statistics correctly when there is a time type [#41938](https://github.com/pingcap/tidb/issues/41938) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that full index scans might cause errors when prepared plan cache is enabled [#42150](https://github.com/pingcap/tidb/issues/42150) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that `IFNULL(NOT NULL COLUMN, ...)` might return incorrect results [#41734](https://github.com/pingcap/tidb/issues/41734) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that TiDB might produce incorrect results when all data in a partitioned table is in a single Region [#41801](https://github.com/pingcap/tidb/issues/41801) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TiDB might produce incorrect results when different partitioned tables appear in a single SQL statement [#42135](https://github.com/pingcap/tidb/issues/42135) @[mjonss](https://github.com/mjonss) + - Fix the issue that statistics auto-collection might not trigger correctly on a partitioned table after adding a new index to the partitioned table [#41638](https://github.com/pingcap/tidb/issues/41638) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that TiDB might read incorrect column statistics information after collecting statistics twice in a row [#42073](https://github.com/pingcap/tidb/issues/42073) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that IndexMerge might produce incorrect results when prepare plan cache is enabled [#41828](https://github.com/pingcap/tidb/issues/41828) @[qw4990](https://github.com/qw4990) + - Fix the issue that IndexMerge might have goroutine leakage [#41605](https://github.com/pingcap/tidb/issues/41605) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that non-BIGINT unsigned integers might produce incorrect results when compared with strings/decimals [#41736](https://github.com/pingcap/tidb/issues/41736) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that killing a previous `ANALYZE` statement due to memory over-limit might cause the current `ANALYZE` statement in the same session to be killed [#41825](https://github.com/pingcap/tidb/issues/41825) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that data race might occur during the information collection process of the batch coprocessor [#41412](https://github.com/pingcap/tidb/issues/41412) @[you06](https://github.com/you06) + - Fix the issue that an assertion error prevents printing MVCC information for partitioned tables [#40629](https://github.com/pingcap/tidb/issues/40629) @[ekexium](https://github.com/ekexium) + - Fix the issue that fair lock mode adds locking to non-existent keys [#41527](https://github.com/pingcap/tidb/issues/41527) @[ekexium](https://github.com/ekexium) + - Fix the issue that `INSERT IGNORE` and `REPLACE` statements do not lock keys that do not modify values [#42121](https://github.com/pingcap/tidb/issues/42121) @[zyguan](https://github.com/zyguan) + ++ PD + + - Fix the issue that the Region Scatter operation might cause uneven distribution of leaders [#6017](https://github.com/tikv/pd/issues/6017) @[HunDunDM](https://github.com/HunDunDM) + - Fix the issue that data race might occur when getting PD members during startup [#6069](https://github.com/tikv/pd/issues/6069) @[rleungx](https://github.com/rleungx) + - Fix the issue that data race might occur when collecting hotspot statistics [#6069](https://github.com/tikv/pd/issues/6069) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that switching placement rule might cause uneven distribution of leaders [#6195](https://github.com/tikv/pd/issues/6195) @[bufferflies](https://github.com/bufferflies) + ++ TiFlash + + - Fix the issue that Decimal division does not round up the last digit in certain cases [#7022](https://github.com/pingcap/tiflash/issues/7022) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that Decimal cast rounds up incorrectly in certain cases [#6994](https://github.com/pingcap/tiflash/issues/6994) @[windtalker](https://github.com/windtalker) + - Fix the issue that TopN/Sort operators produce incorrect results after enabling the new collation [#6807](https://github.com/pingcap/tiflash/issues/6807) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiFlash reports an error when aggregating a result set larger than 12 million rows on a single TiFlash node [#6993](https://github.com/pingcap/tiflash/issues/6993) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue of insufficient wait time for splitting Region retry during the PITR recovery process [#42001](https://github.com/pingcap/tidb/issues/42001) @[joccau](https://github.com/joccau) + - Fix the issue of recovery failures due to `memory is limited` error encountered during the PITR recovery process [#41983](https://github.com/pingcap/tidb/issues/41983) @[joccau](https://github.com/joccau) + - Fix the issue that PITR log backup progress does not advance when a PD node is down [#14184](https://github.com/tikv/tikv/issues/14184) @[YuJuncen](https://github.com/YuJuncen) + - Alleviate the issue that the latency of the PITR log backup progress increases when Region leadership migration occurs [#13638](https://github.com/tikv/tikv/issues/13638) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that restarting the changefeed might cause data loss or that the checkpoint cannot advance [#8242](https://github.com/pingcap/tiflow/issues/8242) @[overvenus](https://github.com/overvenus) + - Fix the data race issue in DDL sink [#8238](https://github.com/pingcap/tiflow/issues/8238) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the changefeed in the `stopped` status might restart automatically [#8330](https://github.com/pingcap/tiflow/issues/8330) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that the TiCDC server panics when all downstream Kafka servers are unavailable [#8523](https://github.com/pingcap/tiflow/issues/8523) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that data might be lost when the downstream is MySQL and the executed statement is incompatible with TiDB [#8453](https://github.com/pingcap/tiflow/issues/8453) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that rolling upgrade might cause TiCDC OOM or that the checkpoint gets stuck [#8329](https://github.com/pingcap/tiflow/issues/8329) @[overvenus](https://github.com/overvenus) + - Fix the issue that graceful upgrade for TiCDC clusters fails on Kubernetes [#8484](https://github.com/pingcap/tiflow/issues/8484) @[overvenus](https://github.com/overvenus) + + + TiDB Data Migration (DM) + + - Fix the issue that when a DM worker node uses Google Cloud Storage, due to too frequent breakpoints, the request frequency limit of Google Cloud Storage is reached and the DM worker cannot write the data into Google Cloud Storage, thus causing the full data to fail to load [#8482](https://github.com/pingcap/tiflow/issues/8482) @[maxshuang](https://github.com/maxshuang) + - Fix the issue that when multiple DM tasks replicate the same downstream data at the same time and all use the downstream metadata table to record the breakpoint information, the breakpoint information of all tasks is written to the same metadata table and uses the same task ID [#8500](https://github.com/pingcap/tiflow/issues/8500) @[maxshuang](https://github.com/maxshuang) + + + TiDB Lightning + + - Fix the issue that when Physical Import Mode is used for importing data, if there is an `auto_random` column in the composite primary key of the target table, but the value of the column is not specified in the source data, TiDB Lightning does not generate data for the `auto_random` column automatically [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that when Logical Import Mode is used for importing data, the import fails due to lack of the `CONFIG` permission for the target cluster [#41915](https://github.com/pingcap/tidb/issues/41915) @[lichunzhu](https://github.com/lichunzhu) + +## Contributors + +We would like to thank the following contributors from the TiDB community: + +- [AntiTopQuark](https://github.com/AntiTopQuark) +- [blacktear23](https://github.com/blacktear23) +- [BornChanger](https://github.com/BornChanger) +- [Dousir9](https://github.com/Dousir9) +- [erwadba](https://github.com/erwadba) +- [HappyUncle](https://github.com/HappyUncle) +- [jiyfhust](https://github.com/jiyfhust) +- [L-maple](https://github.com/L-maple) +- [liumengya94](https://github.com/liumengya94) +- [woofyzhao](https://github.com/woofyzhao) +- [xiaguan](https://github.com/xiaguan) diff --git a/releases/release-7.1.0.md b/releases/release-7.1.0.md new file mode 100644 index 0000000000000..8d1e4a97802cb --- /dev/null +++ b/releases/release-7.1.0.md @@ -0,0 +1,550 @@ +--- +title: TiDB 7.1.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 7.1.0. +--- + +# TiDB 7.1.0 Release Notes + +Release date: May 31, 2023 + +TiDB version: 7.1.0 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.0#version-list) + +TiDB 7.1.0 is a Long-Term Support Release (LTS). + +Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, improvements, and bug fixes released in [6.6.0-DMR](/releases/release-6.6.0.md), [7.0.0-DMR](/releases/release-7.0.0.md), but also introduces the following key features and improvements: + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryFeatureDescription
Scalability and PerformanceTiFlash supports the disaggregated storage and compute architecture and S3 shared storage (experimental, introduced in v7.0.0)TiFlash introduces a cloud-native architecture as an option: +
    +
  • Disaggregates TiFlash's compute and storage, which is a milestone for elastic HTAP resource utilization.
    • +
  • Introduces S3-based storage engine, which can provide shared storage at a lower cost.
    • +
+
TiKV supports batch aggregating data requests (introduced in v6.6.0) This enhancement significantly reduces total RPCs in TiKV batch-get operations. In situations where data is highly dispersed and the gRPC thread pool has insufficient resources, batching coprocessor requests can improve performance by more than 50%.
Load-based replica readIn a read hotspot scenario, TiDB can redirect read requests for a hotspot TiKV node to its replicas. This feature efficiently scatters read hotspots and optimizes the use of cluster resources. To control the threshold for triggering load-based replica read, you can adjust the system variable tidb_load_based_replica_read_threshold.
TiKV supports partitioned Raft KV storage engine (experimental)TiKV introduces a new generation of storage engine, the partitioned Raft KV. By allowing each data Region to have a dedicated RocksDB instance, it can expand the cluster's storage capacity from TB-level to PB-level and provide more stable write latency and stronger scalability.
Reliability and availabilityResource control by resource groups (GA)Support resource management based on resource groups, which allocates and isolates resources for different workloads in the same cluster. This feature significantly enhances the stability of multi-application clusters and lays the foundation for multi-tenancy. In v7.1.0, this feature introduces the ability to estimate system capacity based on actual workload or hardware deployment.
TiFlash supports spill to disk (introduced in v7.0.0)TiFlash supports intermediate result spill to disk to mitigate OOMs in data-intensive operations such as aggregations, sorts, and hash joins.
SQLMulti-valued index (GA)Support MySQL-compatible multi-valued indexes and enhance the JSON type to improve compatibility with MySQL 8.0. This feature improves the efficiency of membership checks on multi-valued columns.
Row-level TTL (GA in v7.0.0)Support managing database size and improve performance by automatically expiring data of a certain age.
Generated columns (GA)Values in a generated column are calculated by a SQL expression in the column definition in real time. This feature pushes some application logic to the database level, thus improving query efficiency.
SecurityLDAP authenticationTiDB supports LDAP authentication, which is compatible with MySQL 8.0.
Audit log enhancement (Enterprise Edition only)TiDB Enterprise Edition enhances the database auditing feature. It significantly improves the system auditing capacity by providing more fine-grained event filtering controls, more user-friendly filter settings, a new file output format in JSON, and lifecycle management of audit logs.
+ +## Feature details + +### Performance + +* Enhance the Partitioned Raft KV storage engine (experimental) [#11515](https://github.com/tikv/tikv/issues/11515) [#12842](https://github.com/tikv/tikv/issues/12842) @[busyjay](https://github.com/busyjay) @[tonyxuqqi](https://github.com/tonyxuqqi) @[tabokie](https://github.com/tabokie) @[bufferflies](https://github.com/bufferflies) @[5kbpers](https://github.com/5kbpers) @[SpadeA-Tang](https://github.com/SpadeA-Tang) @[nolouch](https://github.com/nolouch) + + TiDB v6.6.0 introduces the Partitioned Raft KV storage engine as an experimental feature, which uses multiple RocksDB instances to store TiKV Region data, and the data of each Region is independently stored in a separate RocksDB instance. The new storage engine can better control the number and level of files in the RocksDB instance, achieve physical isolation of data operations between Regions, and support stably managing more data. Compared with the original TiKV storage engine, using the Partitioned Raft KV storage engine can achieve about twice the write throughput and reduce the elastic scaling time by about 4/5 under the same hardware conditions and mixed read and write scenarios. + + In TiDB v7.1.0, the Partitioned Raft KV storage engine is compatible with TiFlash and supports tools such as TiDB Lightning, BR, and TiCDC. + + Currently, this feature is experimental and not recommended for use in production environments. You can only use this engine in a newly created cluster and you cannot directly upgrade from the original TiKV storage engine. + + For more information, see [documentation](/partitioned-raft-kv.md). + +* TiFlash supports late materialization (GA) [#5829](https://github.com/pingcap/tiflash/issues/5829) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + + In v7.0.0, late materialization was introduced in TiFlash as an experimental feature for optimizing query performance. This feature is disabled by default (the [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) system variable defaults to `OFF`). When processing a `SELECT` statement with filter conditions (`WHERE` clause), TiFlash reads all the data from the columns required by the query, and then filters and aggregates the data based on the query conditions. When Late materialization is enabled, TiDB supports pushing down part of the filter conditions to the TableScan operator. That is, TiFlash first scans the column data related to the filter conditions that are pushed down to the TableScan operator, filters the rows that meet the condition, and then scans the other column data of these rows for further calculation, thereby reducing IO scans and computations of data processing. + + Starting from v7.1.0, the TiFlash late materialization feature is generally available and enabled by default (the [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) system variable defaults to `ON`). The TiDB optimizer decides which filters to be pushed down to the TableScan operator based on the statistics and the filter conditions of the query. + + For more information, see [documentation](/tiflash/tiflash-late-materialization.md). + +* TiFlash supports automatically choosing an MPP Join algorithm according to the overhead of network transmission [#7084](https://github.com/pingcap/tiflash/issues/7084) @[solotzg](https://github.com/solotzg) + + The TiFlash MPP mode supports multiple Join algorithms. Before v7.1.0, TiDB determines whether the MPP mode uses the Broadcast Hash Join algorithm based on the [`tidb_broadcast_join_threshold_count`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50) and [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) variables and the actual data volume. + + In v7.1.0, TiDB introduces the [`tidb_prefer_broadcast_join_by_exchange_data_size`](/system-variables.md#tidb_prefer_broadcast_join_by_exchange_data_size-new-in-v710) variable, which controls whether to choose the MPP Join algorithm based on the minimum overhead of network transmission. This variable is disabled by default, indicating that the default algorithm selection method remains the same as that before v7.1.0. You can set the variable to `ON` to enable it. When it is enabled, you no longer need to manually adjust the [`tidb_broadcast_join_threshold_count`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50) and [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) variables (both variables does not take effect at this time), TiDB automatically estimates the threshold of network transmission by different Join algorithms, and then chooses the algorithm with the smallest overhead overall, thus reducing network traffic and improving MPP query performance. + + For more information, see [documentation](/tiflash/use-tiflash-mpp-mode.md#algorithm-support-for-the-mpp-mode). + +* Support load-based replica read to mitigate read hotspots [#14151](https://github.com/tikv/tikv/issues/14151) @[sticnarf](https://github.com/sticnarf) @[you06](https://github.com/you06) + + In a read hotspot scenario, the hotspot TiKV node cannot process read requests in time, resulting in the read requests queuing. However, not all TiKV resources are exhausted at this time. To reduce latency, TiDB v7.1.0 introduces the load-based replica read feature, which allows TiDB to read data from other TiKV nodes without queuing on the hotspot TiKV node. You can control the queue length of read requests using the [`tidb_load_based_replica_read_threshold`](/system-variables.md#tidb_load_based_replica_read_threshold-new-in-v700) system variable. When the estimated queue time of the leader node exceeds this threshold, TiDB prioritizes reading data from follower nodes. This feature can improve read throughput by 70% to 200% in a read hotspot scenario compared to not scattering read hotspots. + + For more information, see [documentation](/troubleshoot-hot-spot-issues.md#scatter-read-hotspots). + +* Enhance the capability of caching execution plans for non-prepared statements (experimental) [#36598](https://github.com/pingcap/tidb/issues/36598) @[qw4990](https://github.com/qw4990) + + TiDB v7.0.0 introduces non-prepared plan cache as an experimental feature to improve the load capacity of concurrent OLTP. In v7.1.0, TiDB enhances this feature and supports caching more SQL statements. + + To improve memory utilization, TiDB v7.1.0 merges the cache pools of non-prepared and prepared plan caches. You can control the cache size using the system variable [`tidb_session_plan_cache_size`](/system-variables.md#tidb_session_plan_cache_size-new-in-v710). The [`tidb_prepared_plan_cache_size`](/system-variables.md#tidb_prepared_plan_cache_size-new-in-v610) and [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size) system variables are deprecated. + + To maintain forward compatibility, when you upgrade from an earlier version to v7.1.0 or later versions, the cache size `tidb_session_plan_cache_size` remains the same value as `tidb_prepared_plan_cache_size`, and [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache) remains the setting before the upgrade. After sufficient performance testing, you can enable non-prepared plan cache using `tidb_enable_non_prepared_plan_cache`. For a newly created cluster, non-prepared plan cache is enabled by default. + + Non-prepared plan cache does not support DML statements by default. To remove this restriction, you can set the [`tidb_enable_non_prepared_plan_cache_for_dml`](/system-variables.md#tidb_enable_non_prepared_plan_cache_for_dml-new-in-v710) system variable to `ON`. + + For more information, see [documentation](/sql-non-prepared-plan-cache.md). + +* Support the DDL distributed parallel execution framework (experimental) [#41495](https://github.com/pingcap/tidb/issues/41495) @[benjamin2037](https://github.com/benjamin2037) + + Before TiDB v7.1.0, only one TiDB node can serve as the DDL owner and execute DDL tasks at the same time. Starting from TiDB v7.1.0, in the new distributed parallel execution framework, multiple TiDB nodes can execute the same DDL task in parallel, thus better utilizing the resources of the TiDB cluster and significantly improving the performance of DDL. In addition, you can linearly improve the performance of DDL by adding more TiDB nodes. Note that this feature is currently experimental and only supports `ADD INDEX` operations. + + To use the distributed framework, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: + + ```sql + SET GLOBAL tidb_enable_dist_task = ON; + ``` + + For more information, see [documentation](/tidb-distributed-execution-framework.md). + +### Reliability + +* Resource Control becomes generally available (GA) [#38825](https://github.com/pingcap/tidb/issues/38825) @[nolouch](https://github.com/nolouch) @[BornChanger](https://github.com/BornChanger) @[glorv](https://github.com/glorv) @[tiancaiamao](https://github.com/tiancaiamao) @[Connor1996](https://github.com/Connor1996) @[JmPotato](https://github.com/JmPotato) @[hnes](https://github.com/hnes) @[CabinfeverB](https://github.com/CabinfeverB) @[HuSharp](https://github.com/HuSharp) + + TiDB enhances the resource control feature based on resource groups, which becomes GA in v7.1.0. This feature significantly improves the resource utilization efficiency and performance of TiDB clusters. The introduction of the resource control feature is a milestone for TiDB. You can divide a distributed database cluster into multiple logical units, map different database users to corresponding resource groups, and set the quota for each resource group as needed. When the cluster resources are limited, all resources used by sessions in the same resource group are limited to the quota. In this way, even if a resource group is over-consumed, the sessions in other resource groups are not affected. + + With this feature, you can combine multiple small and medium-sized applications from different systems into a single TiDB cluster. When the workload of an application grows larger, it does not affect the normal operation of other applications. When the system workload is low, busy applications can still be allocated the required system resources even if they exceed the set quotas, which can achieve the maximum utilization of resources. In addition, the rational use of the resource control feature can reduce the number of clusters, ease the difficulty of operation and maintenance, and save management costs. + + In TiDB v7.1.0, this feature introduces the ability to estimate system capacity based on actual workload or hardware deployment. The estimation ability provides you with a more accurate reference for capacity planning and assists you in better managing TiDB resource allocation to meet the stability needs of enterprise-level scenarios. + + To improve user experience, TiDB Dashboard provides the [Resource Manager page](/dashboard/dashboard-resource-manager.md). You can view the resource group configuration on this page and estimate cluster capacity in a visual way to facilitate reasonable resource allocation. + + For more information, see [documentation](/tidb-resource-control.md). + +* Support the checkpoint mechanism for Fast Online DDL to improve fault tolerance and automatic recovery capability [#42164](https://github.com/pingcap/tidb/issues/42164) @[tangenta](https://github.com/tangenta) + + TiDB v7.1.0 introduces a checkpoint mechanism for [Fast Online DDL](/ddl-introduction.md), which significantly improves the fault tolerance and automatic recovery capability of Fast Online DDL. Even if the TiDB owner node is restarted or changed due to failures, TiDB can still recover progress from checkpoints that are automatically updated on a regular basis, making the DDL execution more stable and efficient. + + For more information, see [documentation](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630). + +* Backup & Restore supports checkpoint restore [#42339](https://github.com/pingcap/tidb/issues/42339) @[Leavrth](https://github.com/Leavrth) + + Snapshot restore or log restore might be interrupted due to recoverable errors, such as disk exhaustion and node crash. Before TiDB v7.1.0, the recovery progress before the interruption would be invalidated even after the error is addressed, and you need to start the restore from scratch. For large clusters, this incurs considerable extra cost. + + Starting from TiDB v7.1.0, Backup & Restore (BR) introduces the checkpoint restore feature, which enables you to continue an interrupted restore. This feature can retain most recovery progress of the interrupted restore. + + For more information, see [documentation](/br/br-checkpoint-restore.md). + +* Optimize the strategy of loading statistics [#42160](https://github.com/pingcap/tidb/issues/42160) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + + TiDB v7.1.0 introduces lightweight statistics initialization as an experimental feature. Lightweight statistics initialization can significantly reduce the number of statistics that must be loaded during startup, thus improving the speed of loading statistics. This feature increases the stability of TiDB in complex runtime environments and reduces the impact on the overall service when TiDB nodes restart. You can set the parameter [`lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) to `true` to enable this feature. + + During TiDB startup, SQL statements executed before the initial statistics are fully loaded might have suboptimal execution plans, thus causing performance issues. To avoid such issues, TiDB v7.1.0 introduces the configuration parameter [`force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710). With this option, you can control whether TiDB provides services only after statistics initialization has been finished during startup. This parameter is disabled by default. + + For more information, see [documentation](/statistics.md#load-statistics). + +* TiCDC supports the data integrity validation feature for single-row data [#8718](https://github.com/pingcap/tiflow/issues/8718) [#42747](https://github.com/pingcap/tidb/issues/42747) @[3AceShowHand](https://github.com/3AceShowHand) @[zyguan](https://github.com/zyguan) + + Starting from v7.1.0, TiCDC introduces the data integrity validation feature, which uses a checksum algorithm to validate the integrity of single-row data. This feature helps verify whether any error occurs in the process of writing data from TiDB, replicating it through TiCDC, and then writing it to a Kafka cluster. The data integrity validation feature only supports changefeeds that use Kafka as the downstream and currently supports the Avro protocol. + + For more information, see [documentation](/ticdc/ticdc-integrity-check.md). + +* TiCDC optimizes DDL replication operations [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/hi-rustin) + + Before v7.1.0, when you perform a DDL operation that affects all rows on a large table (such as adding or deleting a column), the replication latency of TiCDC would significantly increase. Starting from v7.1.0, TiCDC optimizes this replication operation and mitigates the impact of DDL operations on downstream latency. + + For more information, see [documentation](/ticdc/ticdc-faq.md#does-ticdc-replicate-data-changes-caused-by-lossy-ddl-operations-to-the-downstream). + +* Improve the stability of TiDB Lightning when importing TiB-level data [#43510](https://github.com/pingcap/tidb/issues/43510) [#43657](https://github.com/pingcap/tidb/issues/43657) @[D3Hunter](https://github.com/D3Hunter) @[lance6716](https://github.com/lance6716) + + Starting from v7.1.0, TiDB Lightning has added four configuration items to improve stability when importing TiB-level data. + + - `tikv-importer.region-split-batch-size` controls the number of Regions when splitting Regions in a batch. The default value is `4096`. + - `tikv-importer.region-split-concurrency` controls the concurrency when splitting Regions. The default value is the number of CPU cores. + - `tikv-importer.region-check-backoff-limit` controls the number of retries to wait for the Region to come online after the split and scatter operations. The default value is `1800` and the maximum retry interval is two seconds. The number of retries is not increased if any Region becomes online between retries. + - `tikv-importer.pause-pd-scheduler-scope` controls the scope in which TiDB Lightning pauses PD scheduling. Value options are `"table"` and `"global"`. The default value is `"table"`. For TiDB versions earlier than v6.1.0, you can only configure the `"global"` option, which pauses global scheduling during data import. Starting from v6.1.0, the `"table"` option is supported, which means that scheduling is only paused for the Region that stores the target table data. It is recommended to set this configuration item to `"global"` in scenarios with large data volumes to improve stability. + + For more information, see [documentation](/tidb-lightning/tidb-lightning-configuration.md). + +### SQL + +* Support saving TiFlash query results using the `INSERT INTO SELECT` statement (GA) [#37515](https://github.com/pingcap/tidb/issues/37515) @[gengliqi](https://github.com/gengliqi) + + Starting from v6.5.0, TiDB supports pushing down the `SELECT` clause (analytical query) of the `INSERT INTO SELECT` statement to TiFlash. In this way, you can easily save the TiFlash query result to a TiDB table specified by `INSERT INTO` for further analysis, which takes effect as result caching (that is, result materialization). + + In v7.1.0, this feature is generally available. During the execution of the `SELECT` clause in the `INSERT INTO SELECT` statement, the optimizer can intelligently decide whether to push a query down to TiFlash based on the [SQL mode](/sql-mode.md) and the cost estimates of the TiFlash replica. Therefore, the `tidb_enable_tiflash_read_for_write_stmt` system variable introduced during the experimental phase is now deprecated. Note that the computation rules of `INSERT INTO SELECT` statements for TiFlash do not meet the `STRICT SQL Mode` requirement, so TiDB allows the `SELECT` clause in the `INSERT INTO SELECT` statement to be pushed down to TiFlash only when the [SQL mode](/sql-mode.md) of the current session is not strict, which means that the `sql_mode` value does not contain `STRICT_TRANS_TABLES` and `STRICT_ALL_TABLES`. + + For more information, see [documentation](/tiflash/tiflash-results-materialization.md). + +* MySQL-compatible multi-valued indexes become generally available (GA) [#39592](https://github.com/pingcap/tidb/issues/39592) @[xiongjiwei](https://github.com/xiongjiwei) @[qw4990](https://github.com/qw4990) @[YangKeao](https://github.com/YangKeao) + + Filtering the values of an array in a JSON column is a common operation, but normal indexes cannot help speed up such an operation. Creating a multi-valued index on an array can greatly improve filtering performance. If an array in the JSON column has a multi-valued index, you can use the multi-valued index to filter retrieval conditions in `MEMBER OF()`, `JSON_CONTAINS()`, and `JSON_OVERLAPS()` functions, thereby reducing I/O consumption and improving operation speed. + + In v7.1.0, the multi-valued index feature becomes generally available (GA). It supports more complete data types and is compatible with TiDB tools. You can use multi-valued indexes to speed up the search operations on JSON arrays in production environments. + + For more information, see [documentation](/sql-statements/sql-statement-create-index.md#multi-valued-index). + +* Improve the partition management for Hash and Key partitioned tables [#42728](https://github.com/pingcap/tidb/issues/42728) @[mjonss](https://github.com/mjonss) + + Before v7.1.0, Hash and Key partitioned tables in TiDB only support the `TRUNCATE PARTITION` partition management statement. Starting from v7.1.0, Hash and Key partitioned tables also support `ADD PARTITION` and `COALESCE PARTITION` partition management statements. Therefore, you can flexibly adjust the number of partitions in Hash and Key partitioned tables as needed. For example, you can increase the number of partitions with the `ADD PARTITION` statement, or decrease the number of partitions with the `COALESCE PARTITION` statement. + + For more information, see [documentation](/partitioned-table.md#manage-hash-and-key-partitions). + +* The syntax of Range INTERVAL partitioning becomes generally available (GA) [#35683](https://github.com/pingcap/tidb/issues/35683) @[mjonss](https://github.com/mjonss) + + The syntax of Range INTERVAL partitioning (introduced in v6.3.0) becomes GA. With this syntax, you can define Range partitioning by a desired interval without enumerating all partitions, which drastically reduces the length of Range partitioning DDL statements. The syntax is equivalent to that of the original Range partitioning. + + For more information, see [documentation](/partitioned-table.md#range-interval-partitioning). + +* Generated columns become generally available (GA) @[bb7133](https://github.com/bb7133) + + Generated columns are a valuable feature for a database. When creating a table, you can define that the value of a column is calculated based on the values of other columns in the table, rather than being explicitly inserted or updated by users. This generated column can be either a virtual column or a stored column. TiDB has supported MySQL-compatible generated columns since earlier versions, and this feature becomes GA in v7.1.0. + + Using generated columns can improve MySQL compatibility for TiDB, simplifying the process of migrating from MySQL. It also reduces data maintenance complexity and improves data consistency and query efficiency. + + For more information, see [documentation](/generated-columns.md). + +### DB operations + +* Support smooth cluster upgrade without manually canceling DDL operations [#39751](https://github.com/pingcap/tidb/issues/39751) @[zimulala](https://github.com/zimulala) + + Before TiDB v7.1.0, to upgrade a cluster, you must manually cancel its running or queued DDL tasks before the upgrade and then add them back after the upgrade. + + To provide a smoother upgrade experience, TiDB v7.1.0 supports automatically pausing and resuming DDL tasks. Starting from v7.1.0, you can upgrade your clusters without manually canceling DDL tasks in advance. TiDB will automatically pause any running or queued user DDL tasks before the upgrade and resume these tasks after the rolling upgrade, making it easier for you to upgrade your TiDB clusters. + + For more information, see [documentation](/smooth-upgrade-tidb.md). + +### Observability + +* Enhance optimizer diagnostic information [#43122](https://github.com/pingcap/tidb/issues/43122) @[time-and-fate](https://github.com/time-and-fate) + + Obtaining sufficient information is the key to SQL performance diagnostics. In v7.1.0, TiDB continues to add optimizer runtime information to various diagnostic tools, providing better insights into how execution plans are selected and assisting in troubleshooting SQL performance issues. The new information includes: + + * `debug_trace.json` in the output of [`PLAN REPLAYER`](/sql-plan-replayer.md). + * Partial statistics details for `operator info` in the output of [`EXPLAIN`](/explain-walkthrough.md). + * Partial statistics details in the `Stats` field of [slow queries](/identify-slow-queries.md). + + For more information, see [Use `PLAN REPLAYER` to save and restore the on-site information of a cluster](/sql-plan-replayer.md), [`EXPLAIN` walkthrough](/explain-walkthrough.md), and [Identify slow queries](/identify-slow-queries.md). + +### Security + +* Replace the interface used for querying TiFlash system table information [#6941](https://github.com/pingcap/tiflash/issues/6941) @[flowbehappy](https://github.com/flowbehappy) + + Starting from v7.1.0, when providing the query service of [`INFORMATION_SCHEMA.TIFLASH_TABLES`](/information-schema/information-schema-tiflash-tables.md) and [`INFORMATION_SCHEMA.TIFLASH_SEGMENTS`](/information-schema/information-schema-tiflash-segments.md) system tables for TiDB, TiFlash uses the gRPC port instead of the HTTP port, which avoids the security risks of the HTTP service. + +* Support LDAP authentication [#43580](https://github.com/pingcap/tidb/issues/43580) @[YangKeao](https://github.com/YangKeao) + + Starting from v7.1.0, TiDB supports LDAP authentication and provides two authentication plugins: `authentication_ldap_sasl` and `authentication_ldap_simple`. + + For more information, see [documentation](/security-compatibility-with-mysql.md). + +* Enhance the database auditing feature (Enterprise Edition) + + In v7.1.0, TiDB Enterprise Edition enhances the database auditing feature, which significantly expands its capacity and improves the user experience to meet the needs of enterprises for database security compliance: + + - Introduce the concepts of "Filter" and "Rule" for more granular audit event definitions and more fine-grained audit settings. + - Support defining rules in JSON format, providing a more user-friendly configuration method. + - Add automatic log rotation and space management functions, and support configuring log rotation in two dimensions: retention time and log size. + - Support outputting audit logs in both TEXT and JSON formats, facilitating easier integration with third-party tools. + - Support audit log redaction. You can replace all literals to enhance security. + + Database auditing is an important feature in TiDB Enterprise Edition. This feature provides a powerful monitoring and auditing tool for enterprises to ensure data security and compliance. It can help enterprise managers in tracking the source and impact of database operations to prevent illegal data theft or tampering. Furthermore, database auditing can also help enterprises meet various regulatory and compliance requirements, ensuring legal and ethical compliance. This feature has important application value for enterprise information security. + + This feature is included in TiDB Enterprise Edition. To use this feature and its documentation, navigate to the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise) page. + +## Compatibility changes + +> **Note:** +> +> This section provides compatibility changes you need to know when you upgrade from v7.0.0 to the current version (v7.1.0). If you are upgrading from v6.6.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. + +### Behavior changes + +* To improve security, TiFlash deprecates the HTTP service port (default `8123`) and uses the gRPC port as a replacement + + If you have upgraded TiFlash to v7.1.0, then during the TiDB upgrade to v7.1.0, TiDB cannot read the TiFlash system tables ([`INFORMATION_SCHEMA.TIFLASH_TABLES`](/information-schema/information-schema-tiflash-tables.md) and [`INFORMATION_SCHEMA.TIFLASH_SEGMENTS`](/information-schema/information-schema-tiflash-segments.md)). + +* TiDB Lightning in TiDB versions from v6.2.0 to v7.0.0 decides whether to pause global scheduling based on the TiDB cluster version. When TiDB cluster version >= v6.1.0, scheduling is only paused for the Region that stores the target table data and is resumed after the target table import is complete. While for other versions, TiDB Lightning pauses global scheduling. Starting from TiDB v7.1.0, you can control whether to pause global scheduling by configuring [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). By default, TiDB Lightning pauses scheduling for the Region that stores the target table data. If the target cluster version is earlier than v6.1.0, an error occurs. In this case, you can change the value of the parameter to `"global"` and try again. + +* When you use [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-to-timestamp.md) in TiDB v7.1.0, some Regions might remain in the FLASHBACK process even after the completion of the FLASHBACK operation. It is recommended to avoid using this feature in v7.1.0. For more information, see issue [#44292](https://github.com/pingcap/tidb/issues/44292). If you have encountered this issue, you can use the [TiDB snapshot backup and restore](/br/br-snapshot-guide.md) feature to restore data. + +### System variables + +| Variable name | Change type | Description | +|--------|------------------------------|------| +| [`tidb_enable_tiflash_read_for_write_stmt`](/system-variables.md#tidb_enable_tiflash_read_for_write_stmt-new-in-v630) | Deprecated | Changes the default value from `OFF` to `ON`. When [`tidb_allow_mpp = ON`](/system-variables.md#tidb_allow_mpp-new-in-v50), the optimizer intelligently decides whether to push a query down to TiFlash based on the [SQL mode](/sql-mode.md) and the cost estimates of the TiFlash replica. | +| [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size) | Deprecated | Starting from v7.1.0, this system variable is deprecated. You can use [`tidb_session_plan_cache_size`](/system-variables.md#tidb_session_plan_cache_size-new-in-v710) to control the maximum number of plans that can be cached. | +| [`tidb_prepared_plan_cache_size`](/system-variables.md#tidb_prepared_plan_cache_size-new-in-v610) | Deprecated | Starting from v7.1.0, this system variable is deprecated. You can use [`tidb_session_plan_cache_size`](/system-variables.md#tidb_session_plan_cache_size-new-in-v710) to control the maximum number of plans that can be cached. | +| `tidb_ddl_distribute_reorg` | Deleted | This variable is renamed to [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710). | +| [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) | Modified | Introduces two new value options: `authentication_ldap_sasl` and `authentication_ldap_simple`. | +| [`tidb_load_based_replica_read_threshold`](/system-variables.md#tidb_load_based_replica_read_threshold-new-in-v700) | Modified | Takes effect starting from v7.1.0 and controls the threshold for triggering load-based replica read. Changes the default value from `"0s"` to `"1s"` after further tests. | +| [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) | Modified | Changes the default value from `OFF` to `ON`, meaning that the TiFlash late materialization feature is enabled by default. | +| [`authentication_ldap_sasl_auth_method_name`](/system-variables.md#authentication_ldap_sasl_auth_method_name-new-in-v710) | Newly added | Specifies the authentication method name in LDAP SASL authentication. | +| [`authentication_ldap_sasl_bind_base_dn`](/system-variables.md#authentication_ldap_sasl_bind_base_dn-new-in-v710) | Newly added | Limits the search scope within the search tree in LDAP SASL authentication. If a user is created without `AS ...` clause, TiDB automatically searches the `dn` in LDAP server according to the user name. | +| [`authentication_ldap_sasl_bind_root_dn`](/system-variables.md#authentication_ldap_sasl_bind_root_dn-new-in-v710) | Newly added | Specifies the `dn` used to login to the LDAP server to search users in LDAP SASL authentication. | +| [`authentication_ldap_sasl_bind_root_pwd`](/system-variables.md#authentication_ldap_sasl_bind_root_pwd-new-in-v710) | Newly added | Specifies the password used to login to the LDAP server to search users in LDAP SASL authentication. | +| [`authentication_ldap_sasl_ca_path`](/system-variables.md#authentication_ldap_sasl_ca_path-new-in-v710) | Newly added | Specifies the absolute path of the certificate authority file for StartTLS connections in LDAP SASL authentication. | +| [`authentication_ldap_sasl_init_pool_size`](/system-variables.md#authentication_ldap_sasl_init_pool_size-new-in-v710) | Newly added | Specifies the initial connections in the connection pool to the LDAP server in LDAP SASL authentication. | +| [`authentication_ldap_sasl_max_pool_size`](/system-variables.md#authentication_ldap_sasl_max_pool_size-new-in-v710) | Newly added | Specifies the maximum connections in the connection pool to the LDAP server in LDAP SASL authentication. | +| [`authentication_ldap_sasl_server_host`](/system-variables.md#authentication_ldap_sasl_server_host-new-in-v710) | Newly added | Specifies the LDAP server host in LDAP SASL authentication. | +| [`authentication_ldap_sasl_server_port`](/system-variables.md#authentication_ldap_sasl_server_port-new-in-v710) | Newly added | Specifies the LDAP server TCP/IP port number in LDAP SASL authentication. | +| [`authentication_ldap_sasl_tls`](/system-variables.md#authentication_ldap_sasl_tls-new-in-v710) | Newly added | Specifies whether connections by the plugin to the LDAP server are protected with StartTLS in LDAP SASL authentication. | +| [`authentication_ldap_simple_auth_method_name`](/system-variables.md#authentication_ldap_simple_auth_method_name-new-in-v710) | Newly added | Specifies the authentication method name in LDAP simple authentication. It only supports `SIMPLE`. | +| [`authentication_ldap_simple_bind_base_dn`](/system-variables.md#authentication_ldap_simple_bind_base_dn-new-in-v710) | Newly added | Limits the search scope within the search tree in LDAP simple authentication. If a user is created without `AS ...` clause, TiDB will automatically search the `dn` in LDAP server according to the user name. | +| [`authentication_ldap_simple_bind_root_dn`](/system-variables.md#authentication_ldap_simple_bind_root_dn-new-in-v710) | Newly added | Specifies the `dn` used to login to the LDAP server to search users in LDAP simple authentication. | +| [`authentication_ldap_simple_bind_root_pwd`](/system-variables.md#authentication_ldap_simple_bind_root_pwd-new-in-v710) | Newly added | Specifies the password used to login to the LDAP server to search users in LDAP simple authentication. | +| [`authentication_ldap_simple_ca_path`](/system-variables.md#authentication_ldap_simple_ca_path-new-in-v710) | Newly added | Specifies the absolute path of the certificate authority file for StartTLS connections in LDAP simple authentication. | +| [`authentication_ldap_simple_init_pool_size`](/system-variables.md#authentication_ldap_simple_init_pool_size-new-in-v710) | Newly added | Specifies the initial connections in the connection pool to the LDAP server in LDAP simple authentication. | +| [`authentication_ldap_simple_max_pool_size`](/system-variables.md#authentication_ldap_simple_max_pool_size-new-in-v710) | Newly added | Specifies the maximum connections in the connection pool to the LDAP server in LDAP simple authentication. | +| [`authentication_ldap_simple_server_host`](/system-variables.md#authentication_ldap_simple_server_host-new-in-v710) | Newly added | Specifies the LDAP server host in LDAP simple authentication. | +| [`authentication_ldap_simple_server_port`](/system-variables.md#authentication_ldap_simple_server_port-new-in-v710) | Newly added | Specifies the LDAP server TCP/IP port number in LDAP simple authentication. | +| [`authentication_ldap_simple_tls`](/system-variables.md#authentication_ldap_simple_tls-new-in-v710) | Newly added | Specifies whether connections by the plugin to the LDAP server are protected with StartTLS in LDAP simple authentication. | +| [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) | Newly added | Controls whether to enable the distributed execution framework. After enabling distributed execution, DDL, import, and other supported backend tasks will be jointly completed by multiple TiDB nodes in the cluster. This variable was renamed from `tidb_ddl_distribute_reorg`. | +| [`tidb_enable_non_prepared_plan_cache_for_dml`](/system-variables.md#tidb_enable_non_prepared_plan_cache_for_dml-new-in-v710) | Newly added | Controls whether to enable the [Non-prepared plan cache](/sql-non-prepared-plan-cache.md) feature for DML statements. | +| [`tidb_enable_row_level_checksum`](/system-variables.md#tidb_enable_row_level_checksum-new-in-v710) | Newly added | Controls whether to enable the TiCDC data integrity validation for single-row data feature.| +| [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v710) | Newly added | This variable provides more fine-grained control over the optimizer and helps to prevent performance regression after upgrading caused by behavior changes in the optimizer. | +| [`tidb_plan_cache_invalidation_on_fresh_stats`](/system-variables.md#tidb_plan_cache_invalidation_on_fresh_stats-new-in-v710) | Newly added | Controls whether to invalidate the plan cache automatically when statistics on related tables are updated. | +| [`tidb_plan_cache_max_plan_size`](/system-variables.md#tidb_plan_cache_max_plan_size-new-in-v710) | Newly added | Controls the maximum size of a plan that can be cached in prepared or non-prepared plan cache. | +| [`tidb_prefer_broadcast_join_by_exchange_data_size`](/system-variables.md#tidb_prefer_broadcast_join_by_exchange_data_size-new-in-v710) | Newly added | Controls whether to use the algorithm with the minimum overhead of network transmission. If this variable is enabled, TiDB estimates the size of the data to be exchanged in the network using `Broadcast Hash Join` and `Shuffled Hash Join` respectively, and then chooses the one with the smaller size. [`tidb_broadcast_join_threshold_count`](/system-variables.md#tidb_broadcast_join_threshold_count-new-in-v50) and [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) will not take effect after this variable is enabled. | +| [`tidb_session_plan_cache_size`](/system-variables.md#tidb_session_plan_cache_size-new-in-v710) | Newly added | Controls the maximum number of plans that can be cached. Prepared plan cache and non-prepared plan cache share the same cache. | + +### Configuration file parameters + +| Configuration file | Configuration parameter | Change type | Description | +| -------- | -------- | -------- | -------- | +| TiDB | [`performance.force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v710) | Newly added | Controls whether to wait for statistics initialization to finish before providing services during TiDB startup. | +| TiDB | [`performance.lite-init-stats`](/tidb-configuration-file.md#lite-init-stats-new-in-v710) | Newly added | Controls whether to use lightweight statistics initialization during TiDB startup. | +| TiDB | [`log.timeout`](/tidb-configuration-file.md#timeout-new-in-v710) | Newly added | Sets the timeout for log-writing operations in TiDB. In case of a disk failure that prevents logs from being written, this configuration item can trigger the TiDB process to panic instead of hang. The default value is `0`, which means no timeout is set. | +| TiKV | [rocksdb.\[defaultcf\|writecf\|lockcf\].optimize-filters-for-memory](/tikv-configuration-file.md#optimize-filters-for-memory-new-in-v710) | Newly added | Controls whether to generate Bloom/Ribbon filters that minimize memory internal fragmentation. | +| TiKV | [rocksdb.\[defaultcf\|writecf\|lockcf\].ribbon-filter-above-level](/tikv-configuration-file.md#ribbon-filter-above-level-new-in-v710) | Newly added | Controls whether to use Ribbon filters for levels greater than or equal to this value and use non-block-based bloom filters for levels less than this value. | +| TiKV | [`split.byte-threshold`](/tikv-configuration-file.md#byte-threshold-new-in-v50) | Modified | Changes the default value from `30MiB` to `100MiB` when [`region-split-size`](/tikv-configuration-file.md#region-split-size) is greater than or equal to 4 GB. | +| TiKV | [`split.qps-threshold`](/tikv-configuration-file.md#qps-threshold) | Modified | Changes the default value from `3000` to `7000` when [`region-split-size`](/tikv-configuration-file.md#region-split-size) is greater than or equal to 4 GB. | +| TiKV | [`split.region-cpu-overload-threshold-ratio`](/tikv-configuration-file.md#region-cpu-overload-threshold-ratio-new-in-v620) | Modified | Changes the default value from `0.25` to `0.75` when [`region-split-size`](/tikv-configuration-file.md#region-split-size) is greater than or equal to 4 GB. | +| PD | [`store-limit-version`](/pd-configuration-file.md#store-limit-version-new-in-v710) | Newly added | Controls the mode of store limit. Value options are `"v1"` and `"v2"`. | +| PD | [`schedule.enable-diagnostic`](/pd-configuration-file.md#enable-diagnostic-new-in-v630) | Modified | Changes the default value from `false` to `true`, meaning that the diagnostic feature of scheduler is enabled by default. | +| TiFlash | `http_port` | Deleted | Deprecates the HTTP service port (default `8123`). | +| TiDB Lightning | [`tikv-importer.pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md) | Newly added | Controls the scope in which TiDB Lightning pauses PD scheduling. The default value is `"table"` and value options are `"global"` and `"table"`. | +| TiDB Lightning | [`tikv-importer.region-check-backoff-limit`](/tidb-lightning/tidb-lightning-configuration.md) | Newly added | Controls the number of retries to wait for the Region to come online after the split and scatter operations. The default value is `1800`. The maximum retry interval is two seconds. The number of retries is not increased if any Region becomes online between retries.| +| TiDB Lightning | [`tikv-importer.region-split-batch-size`](/tidb-lightning/tidb-lightning-configuration.md) | Newly added | Controls the number of Regions when splitting Regions in a batch. The default value is `4096`. | +| TiDB Lightning | [`tikv-importer.region-split-concurrency`](/tidb-lightning/tidb-lightning-configuration.md) | Newly added | Controls the concurrency when splitting Regions. The default value is the number of CPU cores. | +| TiCDC | [`insecure-skip-verify`](/ticdc/ticdc-sink-to-kafka.md) | Newly added | Controls whether the authentication algorithm is set when TLS is enabled in the scenario of replicating data to Kafka. | +| TiCDC | [`integrity.corruption-handle-level`](/ticdc/ticdc-changefeed-config.md#cli-and-configuration-parameters-of-ticdc-changefeeds) | Newly added | Specifies the log level of the Changefeed when the checksum validation for single-row data fails. The default value is `"warn"`. Value options are `"warn"` and `"error"`. | +| TiCDC | [`integrity.integrity-check-level`](/ticdc/ticdc-changefeed-config.md#cli-and-configuration-parameters-of-ticdc-changefeeds) | Newly added | Controls whether to enable the checksum validation for single-row data. The default value is `"none"`, which means to disable the feature. | +| TiCDC | [`sink.enable-partition-separator`](/ticdc/ticdc-changefeed-config.md#cli-and-configuration-parameters-of-ticdc-changefeeds) | Modified | Changes the default value from `false` to `true` after further tests, meaning that partitions in a table are stored in separate directories by default. It is recommended that you keep the value as `true` to avoid the potential issue of data loss during replication of partitioned tables to storage services. | + +## Improvements + ++ TiDB + + - Display the number of distinct values for the corresponding column in the Cardinality column of the `SHOW INDEX` result [#42227](https://github.com/pingcap/tidb/issues/42227) @[winoros](https://github.com/winoros) + - Use `SQL_NO_CACHE` to prevent TTL Scan queries from impacting the TiKV block cache [#43206](https://github.com/pingcap/tidb/issues/43206) @[lcwangchao](https://github.com/lcwangchao) + - Improve an error message related to `MAX_EXECUTION_TIME` to make it compatible with MySQL [#43031](https://github.com/pingcap/tidb/issues/43031) @[dveeden](https://github.com/dveeden) + - Support using the MergeSort operator on partitioned tables in IndexLookUp [#26166](https://github.com/pingcap/tidb/issues/26166) @[Defined2014](https://github.com/Defined2014) + - Enhance `caching_sha2_password` to make it compatible with MySQL [#43576](https://github.com/pingcap/tidb/issues/43576) @[asjdf](https://github.com/asjdf) + ++ TiKV + + - Reduce the impact of split operations on write QPS when using partitioned Raft KV [#14447](https://github.com/tikv/tikv/issues/14447) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Optimize the space occupied by snapshots when using partitioned Raft KV [#14581](https://github.com/tikv/tikv/issues/14581) @[bufferflies](https://github.com/bufferflies) + - Provide more detailed time information for each stage of processing requests in TiKV [#12362](https://github.com/tikv/tikv/issues/12362) @[cfzjywxk](https://github.com/cfzjywxk) + - Use PD as metastore in log backup [#13867](https://github.com/tikv/tikv/issues/13867) @[YuJuncen](https://github.com/YuJuncen) + ++ PD + + - Add a controller that automatically adjusts the size of the store limit based on the execution details of the snapshot. To enable this controller, set `store-limit-version` to `v2`. Once enabled, you do not need to manually adjust the `store limit` configuration to control the speed of scaling in or scaling out [#6147](https://github.com/tikv/pd/issues/6147) @[bufferflies](https://github.com/bufferflies) + - Add historical load information to avoid frequent scheduling of Regions with unstable loads by the hotspot scheduler when the storage engine is raft-kv2 [#6297](https://github.com/tikv/pd/issues/6297) @[bufferflies](https://github.com/bufferflies) + - Add a leader health check mechanism. When the PD server where the etcd leader is located cannot be elected as the leader, PD actively switches the etcd leader to ensure that the PD leader is available [#6403](https://github.com/tikv/pd/issues/6403) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Improve TiFlash performance and stability in the disaggregated storage and compute architecture [#6882](https://github.com/pingcap/tiflash/issues/6882) @[JaySon-Huang](https://github.com/JaySon-Huang) @[breezewish](https://github.com/breezewish) @[JinheLin](https://github.com/JinheLin) + - Support optimizing query performance in Semi Join or Anti Semi Join by selecting the smaller table as the build side [#7280](https://github.com/pingcap/tiflash/issues/7280) @[yibin87](https://github.com/yibin87) + - Improve performance of data import from BR and TiDB Lightning to TiFlash with default configurations [#7272](https://github.com/pingcap/tiflash/issues/7272) @[breezewish](https://github.com/breezewish) + ++ Tools + + + Backup & Restore (BR) + + - Support modifying the TiKV configuration item `log-backup.max-flush-interval` during log backup [#14433](https://github.com/tikv/tikv/issues/14433) @[joccau](https://github.com/joccau) + + + TiCDC + + - Optimize the directory structure when DDL events occur in the scenario of replicating data to object storage [#8890](https://github.com/pingcap/tiflow/issues/8890) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Optimize the method of setting GC TLS for the upstream when the TiCDC replication task fails [#8403](https://github.com/pingcap/tiflow/issues/8403) @[charleszheng44](https://github.com/charleszheng44) + - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Support using the open-protocol protocol to only replicate the changed columns after an update occurs when replicating data to Kafka [#8706](https://github.com/pingcap/tiflow/issues/8706) @[sdojjy](https://github.com/sdojjy) + - Optimize the error handling of TiCDC in the downstream failures or other scenarios [#8657](https://github.com/pingcap/tiflow/issues/8657) @[hicqu](https://github.com/hicqu) + - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/hi-rustin) + + + TiDB Lightning + + - Change the severity level of the precheck item related to uneven Region distribution from `Critical` to `Warn` to avoid blocking users from importing data [#42836](https://github.com/pingcap/tidb/issues/42836) @[okJiang](https://github.com/okJiang) + - Add a retry mechanism when encountering an `unknown RPC` error during data import [#43291](https://github.com/pingcap/tidb/issues/43291) @[D3Hunter](https://github.com/D3Hunter) + - Enhance the retry mechanism for Region jobs [#43682](https://github.com/pingcap/tidb/issues/43682) @[lance6716](https://github.com/lance6716) + +## Bug fixes + ++ TiDB + + - Fix the issue that there is no prompt about manually executing `ANALYZE TABLE` after reorganizing partitions [#42183](https://github.com/pingcap/tidb/issues/42183) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue of missing table names in the `ADMIN SHOW DDL JOBS` result when a `DROP TABLE` operation is being executed [#42268](https://github.com/pingcap/tidb/issues/42268) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that `Ignore Event Per Minute` and `Stats Cache LRU Cost` charts might not be displayed normally in the Grafana monitoring panel [#42562](https://github.com/pingcap/tidb/issues/42562) @[pingandb](https://github.com/pingandb) + - Fix the issue that the `ORDINAL_POSITION` column returns incorrect results when querying the `INFORMATION_SCHEMA.COLUMNS` table [#43379](https://github.com/pingcap/tidb/issues/43379) @[bb7133](https://github.com/bb7133) + - Fix the case sensitivity issue in some columns of the permission table [#41048](https://github.com/pingcap/tidb/issues/41048) @[bb7133](https://github.com/bb7133) + - Fix the issue that after a new column is added in the cache table, the value is `NULL` instead of the default value of the column [#42928](https://github.com/pingcap/tidb/issues/42928) @[lqs](https://github.com/lqs) + - Fix the issue that CTE results are incorrect when pushing down predicates [#43645](https://github.com/pingcap/tidb/issues/43645) @[winoros](https://github.com/winoros) + - Fix the issue of DDL retry caused by write conflict when executing `TRUNCATE TABLE` for partitioned tables with many partitions and TiFlash replicas [#42940](https://github.com/pingcap/tidb/issues/42940) @[mjonss](https://github.com/mjonss) + - Fix the issue that there is no warning when using `SUBPARTITION` in creating partitioned tables [#41198](https://github.com/pingcap/tidb/issues/41198) [#41200](https://github.com/pingcap/tidb/issues/41200) @[mjonss](https://github.com/mjonss) + - Fix the incompatibility issue with MySQL when dealing with value overflow issues in generated columns [#40066](https://github.com/pingcap/tidb/issues/40066) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that `REORGANIZE PARTITION` cannot be concurrently executed with other DDL operations [#42442](https://github.com/pingcap/tidb/issues/42442) @[bb7133](https://github.com/bb7133) + - Fix the issue that canceling the partition reorganization task in DDL might cause subsequent DDL operations to fail [#42448](https://github.com/pingcap/tidb/issues/42448) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that assertions on delete operations are incorrect under certain conditions [#42426](https://github.com/pingcap/tidb/issues/42426) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB server cannot start due to an error in reading the cgroup information with the error message "can't read file memory.stat from cgroup v1: open /sys/memory.stat no such file or directory" [#42659](https://github.com/pingcap/tidb/issues/42659) @[hawkingrei](https://github.com/hawkingrei) + - Fix the `Duplicate Key` issue that occurs when updating the partition key of a row on a partitioned table with a global index [#42312](https://github.com/pingcap/tidb/issues/42312) @[L-maple](https://github.com/L-maple) + - Fix the issue that the `Scan Worker Time By Phase` chart in the TTL monitoring panel does not display data [#42515](https://github.com/pingcap/tidb/issues/42515) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that some queries on partitioned tables with a global index return incorrect results [#41991](https://github.com/pingcap/tidb/issues/41991) [#42065](https://github.com/pingcap/tidb/issues/42065) @[L-maple](https://github.com/L-maple) + - Fix the issue of displaying some error logs during the process of reorganizing a partitioned table [#42180](https://github.com/pingcap/tidb/issues/42180) @[mjonss](https://github.com/mjonss) + - Fix the issue that the data length in the `QUERY` column of the `INFORMATION_SCHEMA.DDL_JOBS` table might exceed the column definition [#42440](https://github.com/pingcap/tidb/issues/42440) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `INFORMATION_SCHEMA.CLUSTER_HARDWARE` table might display incorrect values in containers [#42851](https://github.com/pingcap/tidb/issues/42851) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an incorrect result is returned when you query a partitioned table using `ORDER BY` + `LIMIT` [#43158](https://github.com/pingcap/tidb/issues/43158) @[Defined2014](https://github.com/Defined2014) + - Fix the issue of multiple DDL tasks running simultaneously using the ingest method [#42903](https://github.com/pingcap/tidb/issues/42903) @[tangenta](https://github.com/tangenta) + - Fix the wrong value returned when querying a partitioned table using `Limit` [#24636](https://github.com/pingcap/tidb/issues/24636) + - Fix the issue of displaying the incorrect TiDB address in IPv6 environment [#43260](https://github.com/pingcap/tidb/issues/43260) @[nexustar](https://github.com/nexustar) + - Fix the issue of displaying incorrect values for system variables `tidb_enable_tiflash_read_for_write_stmt` and `tidb_enable_exchange_partition` [#43281](https://github.com/pingcap/tidb/issues/43281) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the proxy protocol reports the `Header read timeout` error when processing certain erroneous data [#43205](https://github.com/pingcap/tidb/issues/43205) @[blacktear23](https://github.com/blacktear23) + - Fix the issue that when `tidb_scatter_region` is enabled, Region does not automatically split after a partition is truncated [#43174](https://github.com/pingcap/tidb/issues/43174) [#43028](https://github.com/pingcap/tidb/issues/43028) @[jiyfhust](https://github.com/jiyfhust) + - Add checks on the tables with generated columns and report errors for unsupported DDL operations on these columns [#38988](https://github.com/pingcap/tidb/issues/38988) [#24321](https://github.com/pingcap/tidb/issues/24321) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the error message is incorrect in certain type conversion errors [#41730](https://github.com/pingcap/tidb/issues/41730) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that after a TiDB node is normally shutdown, DDL tasks triggered on this node will be canceled [#43854](https://github.com/pingcap/tidb/issues/43854) @[zimulala](https://github.com/zimulala) + - Fix the issue that when the PD member address changes, allocating ID for the `AUTO_INCREMENT` column will be blocked for a long time [#42643](https://github.com/pingcap/tidb/issues/42643) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue of reporting the `GC lifetime is shorter than transaction duration` error during DDL execution [#40074](https://github.com/pingcap/tidb/issues/40074) @[tangenta](https://github.com/tangenta) + - Fix the issue that metadata locks unexpectedly block the DDL execution [#43755](https://github.com/pingcap/tidb/issues/43755) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the cluster cannot query some system views in IPv6 environment [#43286](https://github.com/pingcap/tidb/issues/43286) @[Defined2014](https://github.com/Defined2014) + - Fix the issue of not finding the partition during inner join in dynamic pruning mode [#43686](https://github.com/pingcap/tidb/issues/43686) @[mjonss](https://github.com/mjonss) + - Fix the issue that TiDB reports syntax errors when analyzing tables [#43392](https://github.com/pingcap/tidb/issues/43392) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiCDC might lose some row changes during table renaming [#43338](https://github.com/pingcap/tidb/issues/43338) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB server crashes when the client uses cursor reads [#38116](https://github.com/pingcap/tidb/issues/38116) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that `ADMIN SHOW DDL JOBS LIMIT` returns incorrect results [#42298](https://github.com/pingcap/tidb/issues/42298) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the TiDB panic issue that occurs when querying union views and temporary tables with `UNION` [#42563](https://github.com/pingcap/tidb/issues/42563) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that renaming tables does not take effect when committing multiple statements in a transaction [#39664](https://github.com/pingcap/tidb/issues/39664) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the incompatibility issue between the behavior of prepared plan cache and non-prepared plan cache during time conversion [#42439](https://github.com/pingcap/tidb/issues/42439) @[qw4990](https://github.com/qw4990) + - Fix the wrong results caused by plan cache for Decimal type [#43311](https://github.com/pingcap/tidb/issues/43311) @[qw4990](https://github.com/qw4990) + - Fix the TiDB panic issue in null-aware anti join (NAAJ) due to the wrong field type check [#42459](https://github.com/pingcap/tidb/issues/42459) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that DML execution failures in pessimistic transactions at the RC isolation level might cause inconsistency between data and indexes [#43294](https://github.com/pingcap/tidb/issues/43294) @[ekexium](https://github.com/ekexium) + - Fix the issue that in some extreme cases, when the first statement of a pessimistic transaction is retried, resolving locks on this transaction might affect transaction correctness [#42937](https://github.com/pingcap/tidb/issues/42937) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that in some rare cases, residual pessimistic locks of pessimistic transactions might affect data correctness when GC resolves locks [#43243](https://github.com/pingcap/tidb/issues/43243) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the `LOCK` to `PUT` optimization leads to duplicate data being returned in specific queries [#28011](https://github.com/pingcap/tidb/issues/28011) @[zyguan](https://github.com/zyguan) + - Fix the issue that when data is changed, the locking behavior of the unique index is not consistent with that when the data is unchanged [#36438](https://github.com/pingcap/tidb/issues/36438) @[zyguan](https://github.com/zyguan) + ++ TiKV + + - Fix the issue that when you enable `tidb_pessimistic_txn_fair_locking`, in some extreme cases, expired requests caused by failed RPC retries might affect data correctness during the resolve lock operation [#14551](https://github.com/tikv/tikv/issues/14551) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that when you enable `tidb_pessimistic_txn_fair_locking`, in some extreme cases, expired requests caused by failed RPC retries might cause transaction conflicts to be ignored, thus affecting transaction consistency [#14311](https://github.com/tikv/tikv/issues/14311) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that encryption key ID conflict might cause the deletion of the old keys [#14585](https://github.com/tikv/tikv/issues/14585) @[tabokie](https://github.com/tabokie) + - Fix the performance degradation issue caused by accumulated lock records when a cluster is upgraded from a previous version to v6.5 or later versions [#14780](https://github.com/tikv/tikv/issues/14780) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the `raft entry is too large` error occurs during the PITR recovery process [#14313](https://github.com/tikv/tikv/issues/14313) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV panics during the PITR recovery process due to `log_batch` exceeding 2 GB [#13848](https://github.com/tikv/tikv/issues/13848) @[YuJuncen](https://github.com/YuJuncen) + ++ PD + + - Fix the issue that the number of `low space store` in the PD monitoring panel is abnormal after TiKV panics [#6252](https://github.com/tikv/pd/issues/6252) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that Region Health monitoring data is deleted after PD leader switch [#6366](https://github.com/tikv/pd/issues/6366) @[iosmanthus](https://github.com/iosmanthus) + - Fix the issue that the rule checker cannot repair unhealthy Regions with the `schedule=deny` label [#6426](https://github.com/tikv/pd/issues/6426) @[nolouch](https://github.com/nolouch) + - Fix the issue that some existing labels are lost after TiKV or TiFlash restarts [#6467](https://github.com/tikv/pd/issues/6467) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the replication status cannot be switched when there are learner nodes in the replication mode [#14704](https://github.com/tikv/tikv/issues/14704) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Fix the issue that querying data in the `TIMESTAMP` or `TIME` type returns errors after enabling late materialization [#7455](https://github.com/pingcap/tiflash/issues/7455) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that large update transactions might cause TiFlash to repeatedly report errors and restart [#7316](https://github.com/pingcap/tiflash/issues/7316) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue of backup slowdown when a TiKV node crashes in a cluster [#42973](https://github.com/pingcap/tidb/issues/42973) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue of inaccurate error messages caused by a backup failure in some cases [#43236](https://github.com/pingcap/tidb/issues/43236) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiCDC cannot automatically recover when PD address or leader fails [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that checkpoint lag increases when one of the upstream TiKV nodes crashes [#8858](https://github.com/pingcap/tiflow/issues/8858) @[hicqu](https://github.com/hicqu) + - Fix the issue that when replicating data to object storage, the `EXCHANGE PARTITION` operation in the upstream cannot be properly replicated to the downstream [#8914](https://github.com/pingcap/tiflow/issues/8914) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the OOM issue caused by excessive memory usage of the sorter component in some special scenarios [#8974](https://github.com/pingcap/tiflow/issues/8974) @[hicqu](https://github.com/hicqu) + - Fix the TiCDC node panic that occurs when the downstream Kafka sinks are rolling restarted [#9023](https://github.com/pingcap/tiflow/issues/9023) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that latin1 data might be corrupted during replication [#7028](https://github.com/pingcap/tiflow/issues/7028) @[lance6716](https://github.com/lance6716) + + + TiDB Dumpling + + - Fix the issue that the `UNSIGNED INTEGER` type primary key cannot be used for splitting chunks [#42620](https://github.com/pingcap/tidb/issues/42620) @[lichunzhu](https://github.com/lichunzhu) + - Fix the issue that TiDB Dumpling might panic when `--output-file-template` is incorrectly set [#42391](https://github.com/pingcap/tidb/issues/42391) @[lichunzhu](https://github.com/lichunzhu) + + + TiDB Binlog + + - Fix the issue that an error might occur when encountering a failed DDL statement [#1228](https://github.com/pingcap/tidb-binlog/issues/1228) @[okJiang](https://github.com/okJiang) + + + TiDB Lightning + + - Fix the performance degradation issue during data import [#42456](https://github.com/pingcap/tidb/issues/42456) @[lance6716](https://github.com/lance6716) + - Fix the issue of `write to tikv with no leader returned` when importing a large amount of data [#43055](https://github.com/pingcap/tidb/issues/43055) @[lance6716](https://github.com/lance6716) + - Fix the issue of excessive `keys within region is empty, skip doIngest` logs during data import [#43197](https://github.com/pingcap/tidb/issues/43197) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that panic might occur during partial write [#43363](https://github.com/pingcap/tidb/issues/43363) @[lance6716](https://github.com/lance6716) + - Fix the issue that OOM might occur when importing a wide table [#43728](https://github.com/pingcap/tidb/issues/43728) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue of missing data in the TiDB Lightning Grafana dashboard [#43357](https://github.com/pingcap/tidb/issues/43357) @[lichunzhu](https://github.com/lichunzhu) + - Fix the import failure due to incorrect setting of `keyspace-name` [#43684](https://github.com/pingcap/tidb/issues/43684) @[zeminzhou](https://github.com/zeminzhou) + - Fix the issue that data import might be skipped during range partial write in some cases [#43768](https://github.com/pingcap/tidb/issues/43768) @[lance6716](https://github.com/lance6716) + +## Performance test + +To learn about the performance of TiDB v7.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc) of the TiDB Dedicated cluster. + +## Contributors + +We would like to thank the following contributors from the TiDB community: + +- [blacktear23](https://github.com/blacktear23) +- [ethercflow](https://github.com/ethercflow) +- [hihihuhu](https://github.com/hihihuhu) +- [jiyfhust](https://github.com/jiyfhust) +- [L-maple](https://github.com/L-maple) +- [lqs](https://github.com/lqs) +- [pingandb](https://github.com/pingandb) +- [yorkhellen](https://github.com/yorkhellen) +- [yujiarista](https://github.com/yujiarista) (First-time contributor) diff --git a/sql-statements/sql-statement-alter-resource-group.md b/sql-statements/sql-statement-alter-resource-group.md index 088f2b5759abe..017c26d117f86 100644 --- a/sql-statements/sql-statement-alter-resource-group.md +++ b/sql-statements/sql-statement-alter-resource-group.md @@ -9,7 +9,7 @@ summary: Learn the usage of ALTER RESOURCE GROUP in TiDB. > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/sql-statements/sql-statement-calibrate-resource.md b/sql-statements/sql-statement-calibrate-resource.md new file mode 100644 index 0000000000000..8edf202d22aee --- /dev/null +++ b/sql-statements/sql-statement-calibrate-resource.md @@ -0,0 +1,123 @@ +--- +title: CALIBRATE RESOURCE +summary: An overview of the usage of CALIBRATE RESOURCE for the TiDB database. +--- + +# `CALIBRATE RESOURCE` + +The `CALIBRATE RESOURCE` statement is used to estimate and output the ['Request Unit (RU)`](/tidb-resource-control#what-is-request-unit-ru) capacity of the current cluster. + + + +> **Note:** +> +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). + + + +## Synopsis + +```ebnf+diagram +CalibrateResourceStmt ::= 'CALIBRATE' 'RESOURCE' WorkloadOption + +WorkloadOption ::= +( 'WORKLOAD' ('TPCC' | 'OLTP_READ_WRITE' | 'OLTP_READ_ONLY' | 'OLTP_WRITE_ONLY') ) +| ( 'START_TIME' 'TIMESTAMP' ('DURATION' stringLit | 'END_TIME' 'TIMESTAMP')?)? + +``` + +## Privileges + +To execute this command, make sure that the following requirements are met: + +- You have enabled [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660). +- The user has `SUPER` or `RESOURCE_GROUP_ADMIN` privilege. +- The user has the `SELECT` privilege for all tables in the `METRICS_SCHEMA` schema. + +## Methods for estimating capacity + +TiDB provides two methods for estimation: + +### Estimate capacity based on actual workload + +If your application is already running in a production environment, or you can run actual business tests, it is recommended to use the actual workload over a period of time to estimate the total capacity. To improve the accuracy of the estimation, observe the following constraints: + +- Use the `START_TIME` parameter to specify the time point at which the estimation starts, in the format of `2006-01-02 15:04:05`. The default estimation end time is the current time. +- After specifying the `START_TIME` parameter, you can use the `END_TIME` parameter to specify the estimation end time, or use the `DURATION` parameter to specify the estimation time window from `START_TIME`. +- The time window ranges from 10 minutes to 24 hours. +- In the specified time window, if the CPU utilization of TiDB and TiKV is too low, you cannot estimate the capacity. + +### Estimate capacity based on hardware deployment + +This method mainly estimates capacity based on the current cluster configuration, combined with the empirical values observed for different workloads. Because different types of workloads require different ratios of hardware, the output capacity of the same configuration of hardware might be different. The `WORKLOAD` parameter here accepts the following different workload types. The default value is `TPCC`. + +- `TPCC`: applies to workloads with heavy data write. It is estimated based on a workload model similar to `TPC-C`. +- `OLTP_WRITE_ONLY`: applies to workloads with heavy data write. It is estimated based on a workload model similar to `sysbench oltp_write_only`. +- `OLTP_READ_WRITE`: applies to workloads with even data read and write. It is estimated based on a workload model similar to `sysbench oltp_read_write`. +- `OLTP_READ_ONLY`: applies to workloads with heavy data read. It is estimated based on a workload model similar to `sysbench oltp_read_only`. + +> **Note:** +> +> The RU capacity of a cluster varies with the topology of the cluster and the hardware and software configuration of each component. The actual RU that each cluster can provide is also related to the actual workload. The estimated value based on hardware deployment is for reference only and might differ from the actual maximum value. It is recommended to [estimate capacity based on actual workload](#estimate-capacity-based-on-actual-workload). + +## Examples + +Specify the start time `START_TIME` and the time window `DURATION` to view the RU capacity according to the actual workload. + +```sql +CALIBRATE RESOURCE START_TIME '2023-04-18 08:00:00' DURATION '20m'; ++-------+ +| QUOTA | ++-------+ +| 27969 | ++-------+ +1 row in set (0.01 sec) +``` + +Specify the start time `START_TIME` and the end time `END_TIME` to view the RU capacity according to the actual workload. + +```sql +CALIBRATE RESOURCE START_TIME '2023-04-18 08:00:00' END_TIME '2023-04-18 08:20:00'; ++-------+ +| QUOTA | ++-------+ +| 27969 | ++-------+ +1 row in set (0.01 sec) +``` + +When the time window range `DURATION` does not fall between 10 minutes and 24 hours, an error occurs. + +```sql +CALIBRATE RESOURCE START_TIME '2023-04-18 08:00:00' DURATION '25h'; +ERROR 1105 (HY000): the duration of calibration is too long, which could lead to inaccurate output. Please make the duration between 10m0s and 24h0m0s +CALIBRATE RESOURCE START_TIME '2023-04-18 08:00:00' DURATION '9m'; +ERROR 1105 (HY000): the duration of calibration is too short, which could lead to inaccurate output. Please make the duration between 10m0s and 24h0m0s +``` + +When the workload within the time window is too low, an error occurs. + +```sql +CALIBRATE RESOURCE START_TIME '2023-04-18 08:00:00' DURATION '60m'; +ERROR 1105 (HY000): The workload in selected time window is too low, with which TiDB is unable to reach a capacity estimation; please select another time window with higher workload, or calibrate resource by hardware instead +``` + +Specify `WORKLOAD` to view the RU capacity. The default value is `TPCC`. + +```sql +CALIBRATE RESOURCE; ++-------+ +| QUOTA | ++-------+ +| 190470 | ++-------+ +1 row in set (0.01 sec) + +CALIBRATE RESOURCE WORKLOAD OLTP_WRITE_ONLY; ++-------+ +| QUOTA | ++-------+ +| 27444 | ++-------+ +1 row in set (0.01 sec) +``` diff --git a/sql-statements/sql-statement-create-resource-group.md b/sql-statements/sql-statement-create-resource-group.md index 39ab52ac72e16..ebc0ad761f252 100644 --- a/sql-statements/sql-statement-create-resource-group.md +++ b/sql-statements/sql-statement-create-resource-group.md @@ -9,7 +9,7 @@ summary: Learn the usage of CREATE RESOURCE GROUP in TiDB. > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/sql-statements/sql-statement-drop-resource-group.md b/sql-statements/sql-statement-drop-resource-group.md index 2edd398c59be8..669d23b379e52 100644 --- a/sql-statements/sql-statement-drop-resource-group.md +++ b/sql-statements/sql-statement-drop-resource-group.md @@ -9,7 +9,7 @@ summary: Learn the usage of DROP RESOURCE GROUP in TiDB. > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/sql-statements/sql-statement-flashback-to-timestamp.md b/sql-statements/sql-statement-flashback-to-timestamp.md index 3843499c35fce..c8159cbe17a18 100644 --- a/sql-statements/sql-statement-flashback-to-timestamp.md +++ b/sql-statements/sql-statement-flashback-to-timestamp.md @@ -11,7 +11,7 @@ TiDB v6.4.0 introduces the `FLASHBACK CLUSTER TO TIMESTAMP` syntax. You can use > **Warning:** > -> The `FLASHBACK CLUSTER TO TIMESTAMP` syntax is not applicable to TiDB Cloud [Serverless Tier](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta) clusters. Do not execute this statement on Serverless Tier clusters to avoid unexpected results. +> The `FLASHBACK CLUSTER TO TIMESTAMP` syntax is not applicable to [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta) clusters. Do not execute this statement on TiDB Serverless clusters to avoid unexpected results. diff --git a/sql-statements/sql-statement-load-data.md b/sql-statements/sql-statement-load-data.md index 1c1d0dd6c0a02..1ed4c90e165b6 100644 --- a/sql-statements/sql-statement-load-data.md +++ b/sql-statements/sql-statement-load-data.md @@ -7,6 +7,26 @@ summary: An overview of the usage of LOAD DATA for the TiDB database. The `LOAD DATA` statement batch loads data into a TiDB table. +<<<<<<< HEAD +======= +In TiDB v7.0.0, the `LOAD DATA` SQL statement supports the following features: + +- Support importing data from S3 and GCS +- Add a new parameter `FIELDS DEFINED NULL BY` + +> **Warning:** +> +> The new parameter `FIELDS DEFINED NULL BY` and support for importing data from S3 and GCS in v7.0.0 are experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + + + +> **Note:** +> +> This feature is only available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). + + + +>>>>>>> 76416ca7e (tidb: rename products (#13692)) ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-set-resource-group.md b/sql-statements/sql-statement-set-resource-group.md new file mode 100644 index 0000000000000..e2c32677fa839 --- /dev/null +++ b/sql-statements/sql-statement-set-resource-group.md @@ -0,0 +1,96 @@ +--- +title: SET RESOURCE GROUP +summary: An overview of the usage of SET RESOURCE GROUP in the TiDB database. +--- + +# SET RESOURCE GROUP + +`SET RESOURCE GROUP` is used to set the resource group for the current session. + + + +> **Note:** +> +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). + + + +## Synopsis + +**SetResourceGroupStmt:** + +```ebnf+diagram +SetResourceGroupStmt ::= + "SET" "RESOURCE" "GROUP" ResourceGroupName + +ResourceGroupName ::= + Identifier +``` + +## Examples + +Create a user `user1`, create two resource groups `rg1` and `rg2`, and bind the user `user1` to the resource group `rg1`. + +```sql +CREATE USER 'user1'; +CREATE RESOURCE GROUP 'rg1' RU_PER_SEC = 1000; +ALTER USER 'user1' RESOURCE GROUP `rg1`; +``` + +Use `user1` to log in and view the resource group bound to the current user. + +```sql +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg1 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +Execute `SET RESOURCE GROUP` to set the resource group for the current session to `rg2`. + +```sql +SET RESOURCE GROUP `rg2`; +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg2 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +Execute `SET RESOURCE GROUP` to specify the current session to use the default resource group. + +```sql +SET RESOURCE GROUP ``; +SELECT CURRENT_RESOURCE_GROUP(); +``` + +```sql ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| default | ++--------------------------+ +1 row in set (0.00 sec) +``` + +## MySQL compatibility + +MySQL also supports [SET RESOURCE GROUP](https://dev.mysql.com/doc/refman/8.0/en/set-resource-group.html). But the accepted parameters are different from that of TiDB. They are not compatible. + +## See also + +* [CREATE RESOURCE GROUP](/sql-statements/sql-statement-create-resource-group.md) +* [DROP RESOURCE GROUP](/sql-statements/sql-statement-drop-resource-group.md) +* [ALTER RESOURCE GROUP](/sql-statements/sql-statement-alter-resource-group.md) +* [Resource Control](/tidb-resource-control.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-create-resource-group.md b/sql-statements/sql-statement-show-create-resource-group.md index ab3523a14df24..2c49734a91609 100644 --- a/sql-statements/sql-statement-show-create-resource-group.md +++ b/sql-statements/sql-statement-show-create-resource-group.md @@ -9,7 +9,7 @@ summary: Learn the usage of SHOW CREATE RESOURCE GROUP in TiDB. > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). diff --git a/statement-summary-tables.md b/statement-summary-tables.md index d538e955c6758..b9a9854092fcd 100644 --- a/statement-summary-tables.md +++ b/statement-summary-tables.md @@ -19,7 +19,7 @@ Therefore, starting from v4.0.0-rc.1, TiDB provides system tables in `informatio > **Note:** > -> The following tables are unavailable for [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta): `statements_summary`, `statements_summary_history`, `cluster_statements_summary`, and `cluster_statements_summary_history`. +> The following tables are unavailable for [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta): `statements_summary`, `statements_summary_history`, `cluster_statements_summary`, and `cluster_statements_summary_history`. @@ -200,7 +200,7 @@ To address this issue, TiDB v6.6.0 experimentally introduces the [statement summ -This section is only applicable to on-premises TiDB. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. +This section is only applicable to TiDB Self-Hosted. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. diff --git a/statistics.md b/statistics.md index 2ae4f3c389127..cbbf4c0641e06 100644 --- a/statistics.md +++ b/statistics.md @@ -11,7 +11,7 @@ TiDB uses statistics to decide [which index to choose](/choose-index.md). The `tidb_analyze_version` variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. -- For on-premises TiDB, the default value of this variable is `1` before v5.1.0. In v5.3.0 and later versions, the default value of this variable is `2`. If your cluster is upgraded from a version earlier than v5.3.0 to v5.3.0 or later, the default value of `tidb_analyze_version` does not change. +- For TiDB Self-Hosted, the default value of this variable is `1` before v5.1.0. In v5.3.0 and later versions, the default value of this variable is `2`. If your cluster is upgraded from a version earlier than v5.3.0 to v5.3.0 or later, the default value of `tidb_analyze_version` does not change. - For TiDB Cloud, the default value of this variable is `1`. Compared to Version 1, Version 2 statistics avoids the potential inaccuracy caused by hash collision when the data volume is huge. It also maintains the estimate precision in most scenarios. diff --git a/system-variables.md b/system-variables.md index 56c740e96a0cd..af932373d0b2e 100644 --- a/system-variables.md +++ b/system-variables.md @@ -772,7 +772,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Type: Integer -- Default value: `2` for on-premises TiDB and `1` for TiDB Cloud +- Default value: `2` for TiDB Self-Hosted and `1` for TiDB Cloud - Range: `[1, 2]` - Controls how TiDB collects statistics. @@ -1139,7 +1139,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a > **Note:** > -> To improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. For [Serverless Tier](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta) clusters, this feature is unavailable. +> To improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta) clusters, this feature is unavailable. diff --git a/tidb-distributed-execution-framework.md b/tidb-distributed-execution-framework.md new file mode 100644 index 0000000000000..3678837d58308 --- /dev/null +++ b/tidb-distributed-execution-framework.md @@ -0,0 +1,124 @@ +--- +title: TiDB Backend Task Distributed Execution Framework +summary: Learn the use cases, limitations, usage, and implementation principles of the TiDB backend task distributed execution framework. +--- + +# TiDB Backend Task Distributed Execution Framework + +> **Warning:** +> +> This feature is an experimental feature. It is not recommended to use it in production environments. + + + +> **Note:** +> +> Currently, this feature is only applicable to TiDB Dedicated clusters. You cannot use it on TiDB Serverless clusters. + + + +TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a backend task distributed execution framework to further leverage the resource advantages of the distributed architecture. The goal of this framework is to implement unified scheduling and distributed execution of all backend tasks, and to provide unified resource management capabilities for both overall and individual backend tasks, which better meets users' expectations for resource usage. + +This document describes the use cases, limitations, usage, and implementation principles of the TiDB backend task distributed execution framework. + +> **Note:** +> +> This framework does not support the distributed execution of SQL queries. + +## Use cases and limitations + +In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, Load Data, TTL, Analyze, and Backup/Restore, which are called **backend tasks**. These backend tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: + +- Need to process all data in a schema or a database object (table). +- Might need to be executed periodically, but at a low frequency. +- If the resources are not properly controlled, they are prone to affect TP and AP tasks, lowering the database service quality. + +Enabling the TiDB backend task distributed execution framework can solve the above problems and has the following three advantages: + +- The framework provides unified capabilities for high scalability, high availability, and high performance. +- The framework supports distributed execution of backend tasks, which can flexibly schedule the available computing resources of the entire TiDB cluster, thereby better utilizing the computing resources in a TiDB cluster. +- The framework provides unified resource usage and management capabilities for both overall and individual backend tasks. + +Currently, the TiDB backend task distributed execution framework only supports the distributed execution of `ADD INDEX` statements, that is, the DDL statements for creating indexes. For example, the following SQL statements are supported: + +```sql +ALTER TABLE t1 ADD INDEX idx1(c1); +CREATE INDEX idx1 ON table t1(c1); +``` + +## Prerequisites + +Before using the distributed framework, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. + + + +1. Adjust the following system variables related to Fast Online DDL: + + * [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630): used to enable Fast Online DDL mode. It is enabled by default starting from TiDB v6.5.0. + * [`tidb_ddl_disk_quota`](/system-variables.md#tidb_ddl_disk_quota-new-in-v630): used to control the maximum quota of local disks that can be used in Fast Online DDL mode. + +2. Adjust the following configuration item related to Fast Online DDL: + + * [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630): specifies the local disk path that can be used in Fast Online DDL mode. + +> **Note:** +> +> Before you upgrade TiDB to v6.5.0 or later, it is recommended that you check whether the [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630) path of TiDB is correctly mounted to an SSD disk. This path is a TiDB configuration item, which takes effect after TiDB is restarted. Therefore, setting this configuration item in advance before upgrading can avoid another restart. + + + + + +Adjust the following system variables related to Fast Online DDL: + +* [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630): used to enable Fast Online DDL mode. It is enabled by default starting from TiDB v6.5.0. +* [`tidb_ddl_disk_quota`](/system-variables.md#tidb_ddl_disk_quota-new-in-v630): used to control the maximum quota of local disks that can be used in Fast Online DDL mode. + + + +## Usage + +1. To enable the distributed framework, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: + + ```sql + SET GLOBAL tidb_enable_dist_task = ON; + ``` + + When backend tasks are running, the DDL statements supported by the framework are executed in a distributed manner. + +2. Adjust the following system variables that might affect the distributed execution of DDL tasks according to your needs: + + * [`tidb_ddl_reorg_worker_cnt`](/system-variables.md#tidb_ddl_reorg_worker_cnt): use the default value `4`. The recommended maximum value is `16`. + * [`tidb_ddl_reorg_priority`](/system-variables.md#tidb_ddl_reorg_priority) + * [`tidb_ddl_error_count_limit`](/system-variables.md#tidb_ddl_error_count_limit) + * [`tidb_ddl_reorg_batch_size`](/system-variables.md#tidb_ddl_reorg_batch_size): use the default value. The recommended maximum value is `1024`. + +> **Tip:** +> +> For distributed execution of `ADD INDEX` statements, you only need to set `tidb_ddl_reorg_worker_cnt`. + +## Implementation principles + +The architecture of the TiDB backend task distributed execution framework is as follows: + +![Architecture of the TiDB backend task distributed execution framework](/media/dist-task/dist-task-architect.jpg) + +As shown in the preceding diagram, the execution of backend tasks in the distributed framework is mainly handled by the following modules: + +- Dispatcher: generates the distributed execution plan for each task, manages the execution process, converts the task status, and collects and feeds back the runtime task information. +- Scheduler: replicates the execution of distributed tasks among TiDB nodes to improve the efficiency of backend task execution. +- Subtask Executor: the actual executor of distributed subtasks. In addition, the Subtask Executor returns the execution status of subtasks to the Scheduler, and the Scheduler updates the execution status of subtasks in a unified manner. +- Resource pool: provides the basis for quantifying resource usage and management by pooling computing resources of the above modules. + +## See also + + + +* [Execution Principles and Best Practices of DDL Statements](/ddl-introduction.md) + + + + +* [Execution Principles and Best Practices of DDL Statements](https://docs.pingcap.com/tidb/stable/ddl-introduction) + + diff --git a/tidb-resource-control.md b/tidb-resource-control.md index 5c0f77f0a441e..b39314af1fe5c 100644 --- a/tidb-resource-control.md +++ b/tidb-resource-control.md @@ -13,7 +13,7 @@ summary: Learn how to use the resource control feature to control and schedule a > **Note:** > -> This feature is not available on [Serverless Tier clusters](/tidb-cloud/select-cluster-tier.md#serverless-tier-beta). +> This feature is not available on [TiDB Serverless clusters](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). @@ -78,7 +78,11 @@ The resource control feature introduces two new global variables. +<<<<<<< HEAD * TiKV: For on-premises TiDB, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `false` by default and does not support dynamic modification. If you need to enable it for TiDB Cloud Dedicated Tier clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +======= +* TiKV: For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. +>>>>>>> 76416ca7e (tidb: rename products (#13692)) @@ -156,6 +160,60 @@ After you complete the above operations of creating resource groups and binding If there are too many requests that result in insufficient resources for the resource group, the client's requests will wait. If the wait time is too long, the requests will report an error. +<<<<<<< HEAD +======= +> **Note:** +> +> - When you bind a user to a resource group by using `CREATE USER` or `ALTER USER`, it will not take effect for the user's existing sessions, but only for the user's new sessions. +> - TiDB automatically creates a `default` resource group during cluster initialization. For this resource group, the default value of `RU_PER_SEC` is `UNLIMITED` (equivalent to the maximum value of the `INT` type, that is, `2147483647`) and it is in `BURSTABLE` mode. Statements that are not bound to a resource group are automatically bound to this resource group. This resource group does not support deletion, but you can modify the configuration of its RU. + +#### Bind the current session to a resource group + +By binding a session to a resource group, the resource usage of the corresponding session is limited by the specified usage (RU). + +The following example binds the current session to the resource group `rg1`. + +```sql +SET RESOURCE GROUP rg1; +``` + +#### Bind the current statement to a resource group + +By adding the [`RESOURCE_GROUP(resource_group_name)`](/optimizer-hints.md#resource_groupresource_group_name) hint to a SQL statement, you can specify the resource group to which the statement is bound. This hint supports `SELECT`, `INSERT`, `UPDATE`, and `DELETE` statements. + +The following example binds the current statement to the resource group `rg1`. + +```sql +SELECT /*+ RESOURCE_GROUP(rg1) */ * FROM t limit 10; +``` + +## Disable resource control + + + +1. Execute the following statement to disable the resource control feature. + + ```sql + SET GLOBAL tidb_enable_resource_control = 'OFF'; + ``` + +2. Set the TiKV parameter [`resource-control.enabled`](/tikv-configuration-file.md#resource-control) to `false` to disable scheduling based on the RU of the resource group. + + + + + +1. Execute the following statement to disable the resource control feature. + + ```sql + SET GLOBAL tidb_enable_resource_control = 'OFF'; + ``` + +2. For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + + + +>>>>>>> 76416ca7e (tidb: rename products (#13692)) ## Monitoring metrics and charts @@ -170,7 +228,7 @@ TiKV also records the request QPS from different resource groups. For more detai > **Note:** > -> This section is only applicable to on-premises TiDB. Currently, TiDB Cloud does not provide resource control metrics. +> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide resource control metrics. TiDB regularly collects runtime information about resource control and provides visual charts of the metrics in Grafana's **TiDB** > **Resource Control** dashboard. diff --git a/time-to-live.md b/time-to-live.md index f50d1c6d98d3a..5663694194944 100644 --- a/time-to-live.md +++ b/time-to-live.md @@ -162,7 +162,7 @@ The preceding statement allows TTL jobs to be scheduled only between 1:00 and 5: > **Note:** > -> This section is only applicable to on-premises TiDB. Currently, TiDB Cloud does not provide TTL metrics. +> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide TTL metrics. @@ -240,6 +240,11 @@ Currently, the TTL feature has the following limitations: * The TTL attribute cannot be set on temporary tables, including local temporary tables and global temporary tables. * A table with the TTL attribute does not support being referenced by other tables as the primary table in a foreign key constraint. * It is not guaranteed that all expired data is deleted immediately. The time when expired data is deleted depends on the scheduling interval and scheduling window of the background cleanup job. +<<<<<<< HEAD +======= +* For tables that use [clustered indexes](/clustered-indexes.md), if the primary key is neither an integer nor a binary string type, the TTL job cannot be split into multiple tasks. This will cause the TTL job to be executed sequentially on a single TiDB node. If the table contains a large amount of data, the execution of the TTL job might become slow. +* TTL is not available for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless-beta). +>>>>>>> 76416ca7e (tidb: rename products (#13692)) ## FAQs