diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b43061138a84e..6d275890edbd6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -19,6 +19,7 @@ By default, **CHOOSE MASTER ONLY** so your changes will be applied to the next T For details, see [tips for choosing the affected versions](https://github.com/pingcap/docs/blob/master/CONTRIBUTING.md#guideline-for-choosing-the-affected-versions). - [ ] master (the latest development version) +- [ ] v5.3 (TiDB 5.3 versions) - [ ] v5.2 (TiDB 5.2 versions) - [ ] v5.1 (TiDB 5.1 versions) - [ ] v5.0 (TiDB 5.0 versions) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000000..1eb80fe0f274b --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,18 @@ +name: ci + +on: [pull_request] + +jobs: + + pull: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: '12' + - name: Verify internal links + run: ./scripts/verify-links.sh + - name: Verify internal link anchors + run: ./scripts/verify-link-anchors.sh diff --git a/.github/workflows/link-check.yaml b/.github/workflows/link-check.yaml index 5e4a64bbaca34..74b0a30887183 100644 --- a/.github/workflows/link-check.yaml +++ b/.github/workflows/link-check.yaml @@ -1,31 +1,31 @@ -# name: external-link-check +name: external-link-check -# on: [pull_request] +on: [pull_request] -# jobs: -# base: -# runs-on: ubuntu-18.04 -# steps: -# - name: Check out code -# uses: actions/checkout@v2 +jobs: + base: + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v2 -# # NOTE: -# # - the check process can be very fast, -# # but if `-c` is too large then `Too Many Requests (HTTP error 429)` may be reported from `https://github.com/*`. -# # - we hardcode `--document-root` to `/github/workspace` in the container now. -# # - we use `http://172.*` as sample addresses in some docs, so we need to exclude them. -# - name: Check external links -# id: lc -# uses: peter-evans/link-checker@v1 -# with: -# args: -c 32 -d /github/workspace -r -x "http://172.*|https://github.com/.*/pull/.*|https://github.com/.*/issues/.*|https://github.com/[^.\n]*$|http://127.0.0.1.*|http://localhost.*|http://.*:.*|https://static.googleusercontent.com/media/research.google.com/.*|https://www.crunchbase.com/organization/.*|http://www.brendangregg.com.*|https://blog.csdn.net.*|http://mysql-cocoa.sourceforge.net.*|https://tools.ietf.org.*|http://code.google.com.*|https://spark.apache.org.*|https://www.caasdata.com/|https://www.linkedin.com/company/.*|https://sea-group.org/.*|https://octave.sourceforge.io/database/index.html|https://www.teld.cn/|http://gnade.sourceforge.net/|https://en.wikipedia.org/wiki/WeBank_\(China|https://twitter.com/PingCAP|http://www.webgraphviz.com/|http://octave.sourceforge.net/database/index.html|http://www.ymt.com/|http://www.alhem.net/project/mysql/|https://www.reddit.com/r/TiDB|http://www.189.cn/sh/|https://dev.mysql.com/doc/refman/5.7/en/connector-odbc-info.html|https://asktug.com/_/tidb-performance-map/#/|https://commons.apache.org/proper/commons-dbcp/|https://tomcat.apache.org/tomcat-7.0-doc/jdbc-pool.html|https://en.wikipedia.org/wiki/Plane_\(Unicode|.*.md" . + # NOTE: + # - the check process can be very fast, + # but if `-c` is too large then `Too Many Requests (HTTP error 429)` may be reported from `https://github.com/*`. + # - we hardcode `--document-root` to `/github/workspace` in the container now. + # - we use `http://172.*` as sample addresses in some docs, so we need to exclude them. + - name: Check external links + id: lc + uses: peter-evans/link-checker@v1 + with: + args: -c 32 -d /github/workspace -r -x "http://172.*|https://github.com/.*/pull/.*|https://github.com/.*/issues/.*|https://github.com/[^.\n]*$|http://127.0.0.1.*|http://localhost.*|http://.*:.*|https://static.googleusercontent.com/media/research.google.com/.*|https://www.crunchbase.com/organization/.*|http://www.brendangregg.com.*|https://blog.csdn.net.*|http://mysql-cocoa.sourceforge.net.*|https://tools.ietf.org.*|http://code.google.com.*|https://spark.apache.org.*|https://www.caasdata.com/|https://www.linkedin.com/company/.*|https://sea-group.org/.*|https://octave.sourceforge.io/database/index.html|https://www.teld.cn/|http://gnade.sourceforge.net/|https://en.wikipedia.org/wiki/WeBank_\(China|https://twitter.com/PingCAP|http://www.webgraphviz.com/|http://octave.sourceforge.net/database/index.html|http://www.ymt.com/|http://www.alhem.net/project/mysql/|https://www.reddit.com/r/TiDB|http://www.189.cn/sh/|https://dev.mysql.com/doc/refman/5.7/en/connector-odbc-info.html|https://asktug.com/_/tidb-performance-map/#/|https://commons.apache.org/proper/commons-dbcp/|https://tomcat.apache.org/tomcat-7.0-doc/jdbc-pool.html|https://en.wikipedia.org/wiki/Plane_\(Unicode|.*.md" . -# - name: Fail if there were external link errors -# if: ${{ steps.lc.outputs.exit_code }} -# run: | -# echo "Please check the broken links reported in the previous step \"Check external links\"." -# exit ${{ steps.lc.outputs.exit_code }} -# # Debug via SSH if previous steps failed -# #- name: Set up tmate session -# # if: ${{ failure() }} -# # uses: mxschmitt/action-tmate@v2 + - name: Fail if there were external link errors + if: ${{ steps.lc.outputs.exit_code }} + run: | + echo "Please check the broken links reported in the previous step \"Check external links\"." + exit ${{ steps.lc.outputs.exit_code }} + # Debug via SSH if previous steps failed + #- name: Set up tmate session + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df71d03af55db..6ac2c012e9be1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,7 @@ Currently, we maintain the following versions of TiDB documentation, each with a | Docs branch name | Version description | | :--- | :--- | | `master` branch | the latest development version | +| `release-5.3` branch | the 5.3 stable version | | `release-5.2` branch | the 5.2 stable version | | `release-5.1` branch | the 5.1 stable version | | `release-5.0` branch | the 5.0 stable version | @@ -53,6 +54,7 @@ Currently, we maintain the following versions of TiDB documentation, each with a - If your changes apply to only one docs version, just submit a PR to the corresponding version branch. - If your changes apply to multiple docs versions, you don't have to submit a PR to each branch. Instead, after you submit your PR, trigger the ti-chi-bot to submit a PR to other version branches by adding one or several of the following labels as needed. Once the current PR is merged, ti-chi-bot will start to work. + - `needs-cherry-pick-5.3` label: ti-chi-bot will submit a PR to the `release-5.3` branch. - `needs-cherry-pick-5.2` label: ti-chi-bot will submit a PR to the `release-5.2` branch. - `needs-cherry-pick-5.1` label: ti-chi-bot will submit a PR to the `release-5.1` branch. - `needs-cherry-pick-5.0` label: ti-chi-bot will submit a PR to the `release-5.0` branch. diff --git a/README.md b/README.md index ee9284332a9be..daf0c159287c2 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Currently, we maintain the following versions of TiDB documentation in different | Branch name | TiDB docs version | | :---------|:----------| | [`master`](https://github.com/pingcap/docs/tree/master) | The latest development version | +| [`release-5.3`](https://github.com/pingcap/docs/tree/release-5.3) | 5.3 stable version | | [`release-5.2`](https://github.com/pingcap/docs/tree/release-5.2) | 5.2 stable version | | [`release-5.1`](https://github.com/pingcap/docs/tree/release-5.1) | 5.1 stable version | | [`release-5.0`](https://github.com/pingcap/docs/tree/release-5.0) | 5.0 stable version | diff --git a/TOC.md b/TOC.md index 9308397b8707e..4788da03df94a 100644 --- a/TOC.md +++ b/TOC.md @@ -1,692 +1,713 @@ -# TiDB Documentation - - - -## Documentation List + -+ About TiDB - + [TiDB Introduction](/overview.md) - + [TiDB 5.2 Release Notes](/releases/release-5.2.0.md) - + [Basic Features](/basic-features.md) - + [Experimental Features](/experimental-features.md) - + Benchmarks - + [v5.2 Sysbench Performance Test Report](/benchmark/benchmark-sysbench-v5.2.0-vs-v5.1.1.md) - + [v5.2 TPC-C Performance Test Report](/benchmark/v5.2-performance-benchmarking-with-tpcc.md) - + [v5.2 MPP mode TPC-H 100GB Performance Test](/benchmark/v5.2-performance-benchmarking-with-tpch.md) - + [MySQL Compatibility](/mysql-compatibility.md) - + [TiDB Limitations](/tidb-limitations.md) - + [TiDB Adopters](/adopters.md) - + [Credits](/credits.md) -+ Quick Start - + [Try Out TiDB](/quick-start-with-tidb.md) - + [Try Out HTAP](/quick-start-with-htap.md) - + [Learn TiDB SQL](/basic-sql-operations.md) - + [Learn HTAP](/explore-htap.md) - + [Import Example Database](/import-example-data.md) -+ Deploy - + [Software and Hardware Requirements](/hardware-and-software-requirements.md) - + [Environment Configuration Checklist](/check-before-deployment.md) - + Plan Cluster Topology - + [Minimal Topology](/minimal-deployment-topology.md) - + [TiFlash Topology](/tiflash-deployment-topology.md) - + [TiCDC Topology](/ticdc-deployment-topology.md) - + [TiDB Binlog Topology](/tidb-binlog-deployment-topology.md) - + [TiSpark Topology](/tispark-deployment-topology.md) - + [Cross-DC Topology](/geo-distributed-deployment-topology.md) - + [Hybrid Topology](/hybrid-deployment-topology.md) - + Install and Start - + [Use TiUP (Recommended)](/production-deployment-using-tiup.md) - + [Deploy in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable) - + [Verify Cluster Status](/post-installation-check.md) - + Test Cluster Performance - + [Test TiDB Using Sysbench](/benchmark/benchmark-tidb-using-sysbench.md) - + [Test TiDB Using TPC-C](/benchmark/benchmark-tidb-using-tpcc.md) -+ Migrate - + [Overview](/migration-overview.md) - + Migrate from MySQL - + [Migrate from Amazon Aurora MySQL Using TiDB Lightning](/migrate-from-aurora-using-lightning.md) - + [Migrate from MySQL SQL Files Using TiDB Lightning](/migrate-from-mysql-dumpling-files.md) - + [Migrate from Amazon Aurora MySQL Using DM](/migrate-from-aurora-mysql-database.md) - + Migrate from CSV Files - + [Use TiDB Lightning](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) - + [Use `LOAD DATA` Statement](/sql-statements/sql-statement-load-data.md) - + [Migrate from SQL Files](/migrate-from-mysql-dumpling-files.md) - + [Replicate Incremental Data between TiDB Clusters in Real Time](/incremental-replication-between-clusters.md) -+ Maintain - + Upgrade - + [Use TiUP (Recommended)](/upgrade-tidb-using-tiup.md) - + [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/upgrade-a-tidb-cluster) - + Scale - + [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) - + [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/scale-a-tidb-cluster) - + Backup and Restore - + Use BR Tool (Recommended) - + [BR Tool Overview](/br/backup-and-restore-tool.md) - + [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) - + [BR Use Cases](/br/backup-and-restore-use-cases.md) - + [External Storages](/br/backup-and-restore-storages.md) - + [BR FAQ](/br/backup-and-restore-faq.md) - + [Configure Time Zone](/configure-time-zone.md) - + [Daily Checklist](/daily-check.md) - + [Maintain TiFlash](/tiflash/maintain-tiflash.md) - + [Maintain TiDB Using TiUP](/maintain-tidb-using-tiup.md) - + [Modify Configuration Online](/dynamic-config.md) -+ Monitor and Alert - + [Monitoring Framework Overview](/tidb-monitoring-framework.md) - + [Monitoring API](/tidb-monitoring-api.md) - + [Deploy Monitoring Services](/deploy-monitoring-services.md) - + [Export Grafana Snapshots](/exporting-grafana-snapshots.md) - + [TiDB Cluster Alert Rules](/alert-rules.md) - + [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) -+ Troubleshoot - + [TiDB Troubleshooting Map](/tidb-troubleshooting-map.md) - + [Identify Slow Queries](/identify-slow-queries.md) - + [Analyze Slow Queries](/analyze-slow-queries.md) - + [SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) - + [Identify Expensive Queries](/identify-expensive-queries.md) - + [Statement Summary Tables](/statement-summary-tables.md) - + [Troubleshoot Hotspot Issues](/troubleshoot-hot-spot-issues.md) - + [Troubleshoot Increased Read and Write Latency](/troubleshoot-cpu-issues.md) - + [Troubleshoot Cluster Setup](/troubleshoot-tidb-cluster.md) - + [Troubleshoot High Disk I/O Usage](/troubleshoot-high-disk-io.md) - + [Troubleshoot Lock Conflicts](/troubleshoot-lock-conflicts.md) - + [Troubleshoot TiFlash](/tiflash/troubleshoot-tiflash.md) - + [Troubleshoot Write Conflicts in Optimistic Transactions](/troubleshoot-write-conflicts.md) -+ Performance Tuning - + System Tuning - + [Operating System Tuning](/tune-operating-system.md) - + Software Tuning - + Configuration - + [Tune TiDB Memory](/configure-memory-usage.md) - + [Tune TiKV Threads](/tune-tikv-thread-performance.md) - + [Tune TiKV Memory](/tune-tikv-memory-performance.md) - + [TiKV Follower Read](/follower-read.md) - + [TiFlash Tuning](/tiflash/tune-tiflash-performance.md) - + [Coprocessor Cache](/coprocessor-cache.md) - + SQL Tuning - + [Overview](/sql-tuning-overview.md) - + Understanding the Query Execution Plan - + [Overview](/explain-overview.md) - + [`EXPLAIN` Walkthrough](/explain-walkthrough.md) - + [Indexes](/explain-indexes.md) - + [Joins](/explain-joins.md) - + [MPP Queries](/explain-mpp.md) - + [Subqueries](/explain-subqueries.md) - + [Aggregation](/explain-aggregation.md) - + [Views](/explain-views.md) - + [Partitions](/explain-partitions.md) - + SQL Optimization Process - + [Overview](/sql-optimization-concepts.md) - + Logic Optimization - + [Overview](/sql-logical-optimization.md) - + [Subquery Related Optimizations](/subquery-optimization.md) - + [Column Pruning](/column-pruning.md) - + [Decorrelation of Correlated Subquery](/correlated-subquery-optimization.md) - + [Eliminate Max/Min](/max-min-eliminate.md) - + [Predicates Push Down](/predicate-push-down.md) - + [Partition Pruning](/partition-pruning.md) - + [TopN and Limit Push Down](/topn-limit-push-down.md) - + [Join Reorder](/join-reorder.md) - + Physical Optimization - + [Overview](/sql-physical-optimization.md) - + [Index Selection](/choose-index.md) - + [Statistics](/statistics.md) - + [Wrong Index Solution](/wrong-index-solution.md) - + [Distinct Optimization](/agg-distinct-optimization.md) - + [Prepare Execution Plan Cache](/sql-prepare-plan-cache.md) - + Control Execution Plans - + [Overview](/control-execution-plan.md) - + [Optimizer Hints](/optimizer-hints.md) - + [SQL Plan Management](/sql-plan-management.md) - + [The Blocklist of Optimization Rules and Expression Pushdown](/blocklist-control-plan.md) -+ Tutorials - + [Multiple Data Centers in One City Deployment](/multi-data-centers-in-one-city-deployment.md) - + [Three Data Centers in Two Cities Deployment](/three-data-centers-in-two-cities-deployment.md) - + [Two Data Centers in One City Deployment](/two-data-centers-in-one-city-deployment.md) - + Read Historical Data - + Use Stale Read (Recommended) - + [Usage Scenarios of Stale Read](/stale-read.md) - + [Perform Stale Read Using `As OF TIMESTAMP`](/as-of-timestamp.md) - + [Use the `tidb_snapshot` System Variable](/read-historical-data.md) - + Best Practices - + [Use TiDB](/best-practices/tidb-best-practices.md) - + [Java Application Development](/best-practices/java-app-best-practices.md) - + [Use HAProxy](/best-practices/haproxy-best-practices.md) - + [Highly Concurrent Write](/best-practices/high-concurrency-best-practices.md) - + [Grafana Monitoring](/best-practices/grafana-monitor-best-practices.md) - + [PD Scheduling](/best-practices/pd-scheduling-best-practices.md) - + [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) - + [Three-node Hybrid Deployment](/best-practices/three-nodes-hybrid-deployment.md) - + [Local Read Under Three Data Centers Deployment](/best-practices/three-dc-local-read.md) - + [Use Placement Rules](/configure-placement-rules.md) - + [Use Load Base Split](/configure-load-base-split.md) - + [Use Store Limit](/configure-store-limit.md) -+ TiDB Ecosystem Tools - + [Overview](/ecosystem-tool-user-guide.md) - + [Use Cases](/ecosystem-tool-user-case.md) - + [Download](/download-ecosystem-tools.md) - + Backup & Restore (BR) - + [BR Tool Overview](/br/backup-and-restore-tool.md) - + [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) - + [BR Use Cases](/br/backup-and-restore-use-cases.md) - + [External Storages](/br/backup-and-restore-storages.md) - + [BR FAQ](/br/backup-and-restore-faq.md) - + TiDB Binlog - + [Overview](/tidb-binlog/tidb-binlog-overview.md) - + [Quick Start](/tidb-binlog/get-started-with-tidb-binlog.md) - + [Deploy](/tidb-binlog/deploy-tidb-binlog.md) - + [Maintain](/tidb-binlog/maintain-tidb-binlog-cluster.md) - + [Configure](/tidb-binlog/tidb-binlog-configuration-file.md) - + [Pump](/tidb-binlog/tidb-binlog-configuration-file.md#pump) - + [Drainer](/tidb-binlog/tidb-binlog-configuration-file.md#drainer) - + [Upgrade](/tidb-binlog/upgrade-tidb-binlog.md) - + [Monitor](/tidb-binlog/monitor-tidb-binlog-cluster.md) - + [Reparo](/tidb-binlog/tidb-binlog-reparo.md) - + [binlogctl](/tidb-binlog/binlog-control.md) - + [Binlog Consumer Client](/tidb-binlog/binlog-consumer-client.md) - + [TiDB Binlog Relay Log](/tidb-binlog/tidb-binlog-relay-log.md) - + [Bidirectional Replication Between TiDB Clusters](/tidb-binlog/bidirectional-replication-between-tidb-clusters.md) - + [Glossary](/tidb-binlog/tidb-binlog-glossary.md) - + Troubleshoot - + [Troubleshoot](/tidb-binlog/troubleshoot-tidb-binlog.md) - + [Handle Errors](/tidb-binlog/handle-tidb-binlog-errors.md) - + [FAQ](/tidb-binlog/tidb-binlog-faq.md) - + TiDB Lightning - + [Overview](/tidb-lightning/tidb-lightning-overview.md) - + [Tutorial](/get-started-with-tidb-lightning.md) - + [Deploy](/tidb-lightning/deploy-tidb-lightning.md) - + [Configure](/tidb-lightning/tidb-lightning-configuration.md) - + Key Features - + [Checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md) - + [Table Filter](/table-filter.md) - + [CSV Support](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) - + [Backends](/tidb-lightning/tidb-lightning-backends.md) - + [Web Interface](/tidb-lightning/tidb-lightning-web-interface.md) - + [Monitor](/tidb-lightning/monitor-tidb-lightning.md) - + [FAQ](/tidb-lightning/tidb-lightning-faq.md) - + [Glossary](/tidb-lightning/tidb-lightning-glossary.md) - + TiCDC - + [Overview](/ticdc/ticdc-overview.md) - + [Deploy](/ticdc/deploy-ticdc.md) - + [Maintain](/ticdc/manage-ticdc.md) - + [Troubleshoot](/ticdc/troubleshoot-ticdc.md) - + [Monitor](/ticdc/monitor-ticdc.md) - + [TiCDC OpenAPI](/ticdc/ticdc-open-api.md) - + [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) - + [Integrate TiDB with Confluent Platform](/ticdc/integrate-confluent-using-ticdc.md) - + [Glossary](/ticdc/ticdc-glossary.md) - + [Dumpling](/dumpling-overview.md) - + sync-diff-inspector - + [Overview](/sync-diff-inspector/sync-diff-inspector-overview.md) - + [Data Check for Tables with Different Schema/Table Names](/sync-diff-inspector/route-diff.md) - + [Data Check in Sharding Scenarios](/sync-diff-inspector/shard-diff.md) - + [Data Check for TiDB Upstream/Downstream Clusters](/sync-diff-inspector/upstream-downstream-diff.md) - + TiSpark - + [Quick Start](/get-started-with-tispark.md) - + [User Guide](/tispark-overview.md) -+ Reference - + Cluster Architecture - + [Overview](/tidb-architecture.md) - + [Storage](/tidb-storage.md) - + [Computing](/tidb-computing.md) - + [Scheduling](/tidb-scheduling.md) - + Key Monitoring Metrics - + [Overview](/grafana-overview-dashboard.md) - + [TiDB](/grafana-tidb-dashboard.md) - + [PD](/grafana-pd-dashboard.md) - + [TiKV](/grafana-tikv-dashboard.md) - + [TiFlash](/tiflash/monitor-tiflash.md) - + [TiCDC](/ticdc/monitor-ticdc.md) - + Secure - + [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) - + [Enable TLS Between TiDB Components](/enable-tls-between-components.md) - + [Generate Self-signed Certificates](/generate-self-signed-certificates.md) - + [Encryption at Rest](/encryption-at-rest.md) - + [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) - + [Log Redaction](/log-redaction.md) - + Privileges - + [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) - + [Privilege Management](/privilege-management.md) - + [User Account Management](/user-account-management.md) - + [Role-Based Access Control](/role-based-access-control.md) - + [Certificate-Based Authentication](/certificate-authentication.md) - + SQL - + SQL Language Structure and Syntax - + Attributes - + [AUTO_INCREMENT](/auto-increment.md) - + [AUTO_RANDOM](/auto-random.md) - + [SHARD_ROW_ID_BITS](/shard-row-id-bits.md) - + [Literal Values](/literal-values.md) - + [Schema Object Names](/schema-object-names.md) - + [Keywords and Reserved Words](/keywords.md) - + [User-Defined Variables](/user-defined-variables.md) - + [Expression Syntax](/expression-syntax.md) - + [Comment Syntax](/comment-syntax.md) - + SQL Statements - + [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - + [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - + [`ADMIN`](/sql-statements/sql-statement-admin.md) - + [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - + [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) - + [`ADMIN CHECK [TABLE|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) - + [`ADMIN SHOW DDL [JOBS|QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - + [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) - + [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - + [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - + [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - + [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - + [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - + [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - + [`BACKUP`](/sql-statements/sql-statement-backup.md) - + [`BEGIN`](/sql-statements/sql-statement-begin.md) - + [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) - + [`COMMIT`](/sql-statements/sql-statement-commit.md) - + [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) - + [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) - + [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) - + [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) - + [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) - + [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) - + [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) - + [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) - + [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) - + [`CREATE USER`](/sql-statements/sql-statement-create-user.md) - + [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) - + [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) - + [`DELETE`](/sql-statements/sql-statement-delete.md) - + [`DESC`](/sql-statements/sql-statement-desc.md) - + [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - + [`DO`](/sql-statements/sql-statement-do.md) - + [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) - + [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - + [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - + [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - + [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) - + [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) - + [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) - + [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) - + [`DROP USER`](/sql-statements/sql-statement-drop-user.md) - + [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) - + [`EXECUTE`](/sql-statements/sql-statement-execute.md) - + [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) - + [`EXPLAIN`](/sql-statements/sql-statement-explain.md) - + [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) - + [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) - + [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) - + [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) - + [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) - + [`GRANT `](/sql-statements/sql-statement-grant-role.md) - + [`INSERT`](/sql-statements/sql-statement-insert.md) - + [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) - + [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) - + [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - + [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - + [`PREPARE`](/sql-statements/sql-statement-prepare.md) - + [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - + [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - + [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - + [`REPLACE`](/sql-statements/sql-statement-replace.md) - + [`RESTORE`](/sql-statements/sql-statement-restore.md) - + [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) - + [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) - + [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) - + [`SELECT`](/sql-statements/sql-statement-select.md) - + [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) - + [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) - + [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) - + [`SET ROLE`](/sql-statements/sql-statement-set-role.md) - + [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) - + [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) - + [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) - + [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) - + [`SHOW [GLOBAL|SESSION] BINDINGS`](/sql-statements/sql-statement-show-bindings.md) - + [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - + [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - + [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) - + [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - + [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) - + [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) - + [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) - + [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) - + [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) - + [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) - + [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) - + [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - + [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - + [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - + [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) - + [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) - + [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) - + [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) - + [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - + [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - + [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) - + [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - + [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) - + [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) - + [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) - + [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-histograms.md) - + [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) - + [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) - + [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - + [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) - + [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) - + [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) - + [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) - + [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) - + [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) - + [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) - + [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) - + [`TABLE`](/sql-statements/sql-statement-table.md) - + [`TRACE`](/sql-statements/sql-statement-trace.md) - + [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) - + [`UPDATE`](/sql-statements/sql-statement-update.md) - + [`USE`](/sql-statements/sql-statement-use.md) - + [`WITH`](/sql-statements/sql-statement-with.md) - + Data Types - + [Overview](/data-type-overview.md) - + [Default Values](/data-type-default-values.md) - + [Numeric Types](/data-type-numeric.md) - + [Date and Time Types](/data-type-date-and-time.md) - + [String Types](/data-type-string.md) - + [JSON Type](/data-type-json.md) - + Functions and Operators - + [Overview](/functions-and-operators/functions-and-operators-overview.md) - + [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) - + [Operators](/functions-and-operators/operators.md) - + [Control Flow Functions](/functions-and-operators/control-flow-functions.md) - + [String Functions](/functions-and-operators/string-functions.md) - + [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) - + [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) - + [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) - + [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) - + [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) - + [Information Functions](/functions-and-operators/information-functions.md) - + [JSON Functions](/functions-and-operators/json-functions.md) - + [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) - + [Window Functions](/functions-and-operators/window-functions.md) - + [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - + [Precision Math](/functions-and-operators/precision-math.md) - + [Set Operations](/functions-and-operators/set-operators.md) - + [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - + [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - + [Clustered Indexes](/clustered-indexes.md) - + [Constraints](/constraints.md) - + [Generated Columns](/generated-columns.md) - + [SQL Mode](/sql-mode.md) - + Transactions - + [Overview](/transaction-overview.md) - + [Isolation Levels](/transaction-isolation-levels.md) - + [Optimistic Transactions](/optimistic-transaction.md) - + [Pessimistic Transactions](/pessimistic-transaction.md) - + Garbage Collection (GC) - + [Overview](/garbage-collection-overview.md) - + [Configuration](/garbage-collection-configuration.md) - + [Views](/views.md) - + [Partitioning](/partitioned-table.md) - + [Character Set and Collation](/character-set-and-collation.md) - + System Tables - + [`mysql`](/mysql-schema.md) - + INFORMATION_SCHEMA - + [Overview](/information-schema/information-schema.md) - + [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) - + [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) - + [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) - + [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) - + [`CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) - + [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) - + [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-hardware.md) - + [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) - + [`CLUSTER_LOAD`](/information-schema/information-schema-cluster-load.md) - + [`CLUSTER_LOG`](/information-schema/information-schema-cluster-log.md) - + [`CLUSTER_SYSTEMINFO`](/information-schema/information-schema-cluster-systeminfo.md) - + [`COLLATIONS`](/information-schema/information-schema-collations.md) - + [`COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) - + [`COLUMNS`](/information-schema/information-schema-columns.md) - + [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) - + [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) - + [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) - + [`ENGINES`](/information-schema/information-schema-engines.md) - + [`INSPECTION_RESULT`](/information-schema/information-schema-inspection-result.md) - + [`INSPECTION_RULES`](/information-schema/information-schema-inspection-rules.md) - + [`INSPECTION_SUMMARY`](/information-schema/information-schema-inspection-summary.md) - + [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) - + [`METRICS_SUMMARY`](/information-schema/information-schema-metrics-summary.md) - + [`METRICS_TABLES`](/information-schema/information-schema-metrics-tables.md) - + [`PARTITIONS`](/information-schema/information-schema-partitions.md) - + [`PROCESSLIST`](/information-schema/information-schema-processlist.md) - + [`REFERENTIAL_CONSTRAINTS`](/information-schema/information-schema-referential-constraints.md) - + [`SCHEMATA`](/information-schema/information-schema-schemata.md) - + [`SEQUENCES`](/information-schema/information-schema-sequences.md) - + [`SESSION_VARIABLES`](/information-schema/information-schema-session-variables.md) - + [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) - + [`STATISTICS`](/information-schema/information-schema-statistics.md) - + [`TABLES`](/information-schema/information-schema-tables.md) - + [`TABLE_CONSTRAINTS`](/information-schema/information-schema-table-constraints.md) - + [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) - + [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) - + [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) - + [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) - + [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) - + [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) - + [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) - + [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) - + [`TIKV_STORE_STATUS`](/information-schema/information-schema-tikv-store-status.md) - + [`USER_PRIVILEGES`](/information-schema/information-schema-user-privileges.md) - + [`VIEWS`](/information-schema/information-schema-views.md) - + [`METRICS_SCHEMA`](/metrics-schema.md) - + UI - + TiDB Dashboard - + [Overview](/dashboard/dashboard-intro.md) - + Maintain - + [Deploy](/dashboard/dashboard-ops-deploy.md) - + [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) - + [Secure](/dashboard/dashboard-ops-security.md) - + [Access](/dashboard/dashboard-access.md) - + [Overview Page](/dashboard/dashboard-overview.md) - + [Cluster Info Page](/dashboard/dashboard-cluster-info.md) - + [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) - + [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) - + SQL Statements Analysis - + [SQL Statements Page](/dashboard/dashboard-statement-list.md) - + [SQL Details Page](/dashboard/dashboard-statement-details.md) - + [Slow Queries Page](/dashboard/dashboard-slow-query.md) - + Cluster Diagnostics - + [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) - + [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) - + [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) - + [Search Logs Page](/dashboard/dashboard-log-search.md) - + [Profile Instances Page](/dashboard/dashboard-profiling.md) - + Session Management and Configuration - + [Share Session](/dashboard/dashboard-session-share.md) - + [Configure SSO](/dashboard/dashboard-session-sso.md) - + [FAQ](/dashboard/dashboard-faq.md) - + CLI - + [tikv-ctl](/tikv-control.md) - + [pd-ctl](/pd-control.md) - + [tidb-ctl](/tidb-control.md) - + [pd-recover](/pd-recover.md) - + Command Line Flags - + [tidb-server](/command-line-flags-for-tidb-configuration.md) - + [tikv-server](/command-line-flags-for-tikv-configuration.md) - + [tiflash-server](/tiflash/tiflash-command-line-flags.md) - + [pd-server](/command-line-flags-for-pd-configuration.md) - + Configuration File Parameters - + [tidb-server](/tidb-configuration-file.md) - + [tikv-server](/tikv-configuration-file.md) - + [tiflash-server](/tiflash/tiflash-configuration.md) - + [pd-server](/pd-configuration-file.md) - + [System Variables](/system-variables.md) - + Storage Engines - + TiKV - + [TiKV Overview](/tikv-overview.md) - + [RocksDB Overview](/storage-engine/rocksdb-overview.md) - + [Titan Overview](/storage-engine/titan-overview.md) - + [Titan Configuration](/storage-engine/titan-configuration.md) - + TiFlash - + [Overview](/tiflash/tiflash-overview.md) - + [Use TiFlash](/tiflash/use-tiflash.md) - + TiUP - + [Documentation Map](/tiup/tiup-documentation-guide.md) - + [Overview](/tiup/tiup-overview.md) - + [Terminology and Concepts](/tiup/tiup-terminology-and-concepts.md) - + [Manage TiUP Components](/tiup/tiup-component-management.md) - + [FAQ](/tiup/tiup-faq.md) - + [Troubleshooting Guide](/tiup/tiup-troubleshooting-guide.md) - + [Reference Guide](/tiup/tiup-reference.md) - + [Cluster Topology Reference](/tiup/tiup-cluster-topology-reference.md) - + [Mirror Reference Guide](/tiup/tiup-mirror-reference.md) - + TiUP Components - + [tiup-playground](/tiup/tiup-playground.md) - + [tiup-cluster](/tiup/tiup-cluster.md) - + [tiup-mirror](/tiup/tiup-mirror.md) - + [tiup-bench](/tiup/tiup-bench.md) - + [Telemetry](/telemetry.md) - + [Errors Codes](/error-codes.md) - + [Table Filter](/table-filter.md) - + [Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) -+ FAQs - + [TiDB FAQs](/faq/tidb-faq.md) - + [SQL FAQs](/faq/sql-faq.md) - + [Deploy and Maintain FAQs](/faq/deploy-and-maintain-faq.md) - + [Upgrade FAQs](/faq/upgrade-faq.md) - + [High Availability FAQs](/faq/high-availability-faq.md) - + [High Reliability FAQs](/faq/high-reliability-faq.md) - + [Migration FAQs](/faq/migration-tidb-faq.md) -+ [Glossary](/glossary.md) -+ Release Notes - + [All Releases](/releases/release-notes.md) - + v5.2 - + [5.2.1](/releases/release-5.2.1.md) - + [5.2.0](/releases/release-5.2.0.md) - + v5.1 - + [5.1.2](/releases/release-5.1.2.md) - + [5.1.1](/releases/release-5.1.1.md) - + [5.1.0](/releases/release-5.1.0.md) - + v5.0 - + [5.0.4](/releases/release-5.0.4.md) - + [5.0.3](/releases/release-5.0.3.md) - + [5.0.2](/releases/release-5.0.2.md) - + [5.0.1](/releases/release-5.0.1.md) - + [5.0 GA](/releases/release-5.0.0.md) - + [5.0.0-rc](/releases/release-5.0.0-rc.md) - + v4.0 - + [4.0.15](/releases/release-4.0.15.md) - + [4.0.14](/releases/release-4.0.14.md) - + [4.0.13](/releases/release-4.0.13.md) - + [4.0.12](/releases/release-4.0.12.md) - + [4.0.11](/releases/release-4.0.11.md) - + [4.0.10](/releases/release-4.0.10.md) - + [4.0.9](/releases/release-4.0.9.md) - + [4.0.8](/releases/release-4.0.8.md) - + [4.0.7](/releases/release-4.0.7.md) - + [4.0.6](/releases/release-4.0.6.md) - + [4.0.5](/releases/release-4.0.5.md) - + [4.0.4](/releases/release-4.0.4.md) - + [4.0.3](/releases/release-4.0.3.md) - + [4.0.2](/releases/release-4.0.2.md) - + [4.0.1](/releases/release-4.0.1.md) - + [4.0 GA](/releases/release-4.0-ga.md) - + [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) - + [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) - + [4.0.0-rc](/releases/release-4.0.0-rc.md) - + [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) - + [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) - + [4.0.0-beta](/releases/release-4.0.0-beta.md) - + v3.1 - + [3.1.2](/releases/release-3.1.2.md) - + [3.1.1](/releases/release-3.1.1.md) - + [3.1.0 GA](/releases/release-3.1.0-ga.md) - + [3.1.0-rc](/releases/release-3.1.0-rc.md) - + [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) - + [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) - + [3.1.0-beta](/releases/release-3.1.0-beta.md) - + v3.0 - + [3.0.20](/releases/release-3.0.20.md) - + [3.0.19](/releases/release-3.0.19.md) - + [3.0.18](/releases/release-3.0.18.md) - + [3.0.17](/releases/release-3.0.17.md) - + [3.0.16](/releases/release-3.0.16.md) - + [3.0.15](/releases/release-3.0.15.md) - + [3.0.14](/releases/release-3.0.14.md) - + [3.0.13](/releases/release-3.0.13.md) - + [3.0.12](/releases/release-3.0.12.md) - + [3.0.11](/releases/release-3.0.11.md) - + [3.0.10](/releases/release-3.0.10.md) - + [3.0.9](/releases/release-3.0.9.md) - + [3.0.8](/releases/release-3.0.8.md) - + [3.0.7](/releases/release-3.0.7.md) - + [3.0.6](/releases/release-3.0.6.md) - + [3.0.5](/releases/release-3.0.5.md) - + [3.0.4](/releases/release-3.0.4.md) - + [3.0.3](/releases/release-3.0.3.md) - + [3.0.2](/releases/release-3.0.2.md) - + [3.0.1](/releases/release-3.0.1.md) - + [3.0 GA](/releases/release-3.0-ga.md) - + [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) - + [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) - + [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) - + [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) - + [3.0.0-beta](/releases/release-3.0-beta.md) - + v2.1 - + [2.1.19](/releases/release-2.1.19.md) - + [2.1.18](/releases/release-2.1.18.md) - + [2.1.17](/releases/release-2.1.17.md) - + [2.1.16](/releases/release-2.1.16.md) - + [2.1.15](/releases/release-2.1.15.md) - + [2.1.14](/releases/release-2.1.14.md) - + [2.1.13](/releases/release-2.1.13.md) - + [2.1.12](/releases/release-2.1.12.md) - + [2.1.11](/releases/release-2.1.11.md) - + [2.1.10](/releases/release-2.1.10.md) - + [2.1.9](/releases/release-2.1.9.md) - + [2.1.8](/releases/release-2.1.8.md) - + [2.1.7](/releases/release-2.1.7.md) - + [2.1.6](/releases/release-2.1.6.md) - + [2.1.5](/releases/release-2.1.5.md) - + [2.1.4](/releases/release-2.1.4.md) - + [2.1.3](/releases/release-2.1.3.md) - + [2.1.2](/releases/release-2.1.2.md) - + [2.1.1](/releases/release-2.1.1.md) - + [2.1 GA](/releases/release-2.1-ga.md) - + [2.1 RC5](/releases/release-2.1-rc.5.md) - + [2.1 RC4](/releases/release-2.1-rc.4.md) - + [2.1 RC3](/releases/release-2.1-rc.3.md) - + [2.1 RC2](/releases/release-2.1-rc.2.md) - + [2.1 RC1](/releases/release-2.1-rc.1.md) - + [2.1 Beta](/releases/release-2.1-beta.md) - + v2.0 - + [2.0.11](/releases/release-2.0.11.md) - + [2.0.10](/releases/release-2.0.10.md) - + [2.0.9](/releases/release-2.0.9.md) - + [2.0.8](/releases/release-2.0.8.md) - + [2.0.7](/releases/release-2.0.7.md) - + [2.0.6](/releases/release-2.0.6.md) - + [2.0.5](/releases/release-2.0.5.md) - + [2.0.4](/releases/release-2.0.4.md) - + [2.0.3](/releases/release-2.0.3.md) - + [2.0.2](/releases/release-2.0.2.md) - + [2.0.1](/releases/release-2.0.1.md) - + [2.0](/releases/release-2.0-ga.md) - + [2.0 RC5](/releases/release-2.0-rc.5.md) - + [2.0 RC4](/releases/release-2.0-rc.4.md) - + [2.0 RC3](/releases/release-2.0-rc.3.md) - + [2.0 RC1](/releases/release-2.0-rc.1.md) - + [1.1 Beta](/releases/release-1.1-beta.md) - + [1.1 Alpha](/releases/release-1.1-alpha.md) - + v1.0 - + [1.0.8](/releases/release-1.0.8.md) - + [1.0.7](/releases/release-1.0.7.md) - + [1.0.6](/releases/release-1.0.6.md) - + [1.0.5](/releases/release-1.0.5.md) - + [1.0.4](/releases/release-1.0.4.md) - + [1.0.3](/releases/release-1.0.3.md) - + [1.0.2](/releases/release-1.0.2.md) - + [1.0.1](/releases/release-1.0.1.md) - + [1.0](/releases/release-1.0-ga.md) - + [Pre-GA](/releases/release-pre-ga.md) - + [RC4](/releases/release-rc.4.md) - + [RC3](/releases/release-rc.3.md) - + [RC2](/releases/release-rc.2.md) - + [RC1](/releases/release-rc.1.md) +- About TiDB + - [TiDB Introduction](/overview.md) + - [TiDB 5.3 Release Notes](/releases/release-5.3.0.md) + - [Basic Features](/basic-features.md) + - [Experimental Features](/experimental-features.md) + - Benchmarks + - [v5.3 Sysbench Performance Test Report](/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md) + - [v5.3 TPC-C Performance Test Report](/benchmark/v5.3-performance-benchmarking-with-tpcc.md) + - [v5.3 MPP mode TPC-H 100GB Performance Test](/benchmark/v5.3-performance-benchmarking-with-tpch.md) + - [MySQL Compatibility](/mysql-compatibility.md) + - [TiDB Limitations](/tidb-limitations.md) + - [TiDB Adopters](/adopters.md) + - [Credits](/credits.md) +- Quick Start + - [Try Out TiDB](/quick-start-with-tidb.md) + - [Try Out HTAP](/quick-start-with-htap.md) + - [Learn TiDB SQL](/basic-sql-operations.md) + - [Learn HTAP](/explore-htap.md) + - [Import Example Database](/import-example-data.md) +- Deploy + - [Software and Hardware Requirements](/hardware-and-software-requirements.md) + - [Environment Configuration Checklist](/check-before-deployment.md) + - Plan Cluster Topology + - [Minimal Topology](/minimal-deployment-topology.md) + - [TiFlash Topology](/tiflash-deployment-topology.md) + - [TiCDC Topology](/ticdc-deployment-topology.md) + - [TiDB Binlog Topology](/tidb-binlog-deployment-topology.md) + - [TiSpark Topology](/tispark-deployment-topology.md) + - [Cross-DC Topology](/geo-distributed-deployment-topology.md) + - [Hybrid Topology](/hybrid-deployment-topology.md) + - Install and Start + - [Use TiUP (Recommended)](/production-deployment-using-tiup.md) + - [Deploy in Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable) + - [Verify Cluster Status](/post-installation-check.md) + - Test Cluster Performance + - [Test TiDB Using Sysbench](/benchmark/benchmark-tidb-using-sysbench.md) + - [Test TiDB Using TPC-C](/benchmark/benchmark-tidb-using-tpcc.md) +- Migrate + - [Overview](/migration-overview.md) + - Migrate from MySQL + - [Migrate from Amazon Aurora MySQL Using TiDB Lightning](/migrate-from-aurora-using-lightning.md) + - [Migrate from MySQL SQL Files Using TiDB Lightning](/migrate-from-mysql-dumpling-files.md) + - [Migrate from Amazon Aurora MySQL Using DM](/migrate-from-aurora-mysql-database.md) + - Migrate from CSV Files + - [Use TiDB Lightning](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) + - [Use `LOAD DATA` Statement](/sql-statements/sql-statement-load-data.md) + - [Migrate from SQL Files](/migrate-from-mysql-dumpling-files.md) + - [Replicate Incremental Data between TiDB Clusters in Real Time](/incremental-replication-between-clusters.md) +- Maintain + - Upgrade + - [Use TiUP (Recommended)](/upgrade-tidb-using-tiup.md) + - [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/upgrade-a-tidb-cluster) + - Scale + - [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) + - [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/v1.1/scale-a-tidb-cluster) + - Backup and Restore + - Use BR Tool (Recommended) + - [BR Tool Overview](/br/backup-and-restore-tool.md) + - [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) + - [BR Use Cases](/br/backup-and-restore-use-cases.md) + - [External Storages](/br/backup-and-restore-storages.md) + - [BR FAQ](/br/backup-and-restore-faq.md) + - [Configure Time Zone](/configure-time-zone.md) + - [Daily Checklist](/daily-check.md) + - [Maintain TiFlash](/tiflash/maintain-tiflash.md) + - [Maintain TiDB Using TiUP](/maintain-tidb-using-tiup.md) + - [Modify Configuration Online](/dynamic-config.md) + - [Online Unsafe Recovery](/online-unsafe-recovery.md) +- Monitor and Alert + - [Monitoring Framework Overview](/tidb-monitoring-framework.md) + - [Monitoring API](/tidb-monitoring-api.md) + - [Deploy Monitoring Services](/deploy-monitoring-services.md) + - [Export Grafana Snapshots](/exporting-grafana-snapshots.md) + - [TiDB Cluster Alert Rules](/alert-rules.md) + - [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) +- Troubleshoot + - [TiDB Troubleshooting Map](/tidb-troubleshooting-map.md) + - [Identify Slow Queries](/identify-slow-queries.md) + - [Analyze Slow Queries](/analyze-slow-queries.md) + - [SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) + - [Identify Expensive Queries](/identify-expensive-queries.md) + - [Statement Summary Tables](/statement-summary-tables.md) + - [Troubleshoot Hotspot Issues](/troubleshoot-hot-spot-issues.md) + - [Troubleshoot Increased Read and Write Latency](/troubleshoot-cpu-issues.md) + - [Save and Restore the On-Site Information of a Cluster](/sql-plan-replayer.md) + - [Troubleshoot Cluster Setup](/troubleshoot-tidb-cluster.md) + - [Troubleshoot High Disk I/O Usage](/troubleshoot-high-disk-io.md) + - [Troubleshoot Lock Conflicts](/troubleshoot-lock-conflicts.md) + - [Troubleshoot TiFlash](/tiflash/troubleshoot-tiflash.md) + - [Troubleshoot Write Conflicts in Optimistic Transactions](/troubleshoot-write-conflicts.md) +- Performance Tuning + - System Tuning + - [Operating System Tuning](/tune-operating-system.md) + - Software Tuning + - Configuration + - [Tune TiDB Memory](/configure-memory-usage.md) + - [Tune TiKV Threads](/tune-tikv-thread-performance.md) + - [Tune TiKV Memory](/tune-tikv-memory-performance.md) + - [TiKV Follower Read](/follower-read.md) + - [TiFlash Tuning](/tiflash/tune-tiflash-performance.md) + - [Coprocessor Cache](/coprocessor-cache.md) + - SQL Tuning + - [Overview](/sql-tuning-overview.md) + - Understanding the Query Execution Plan + - [Overview](/explain-overview.md) + - [`EXPLAIN` Walkthrough](/explain-walkthrough.md) + - [Indexes](/explain-indexes.md) + - [Joins](/explain-joins.md) + - [MPP Queries](/explain-mpp.md) + - [Subqueries](/explain-subqueries.md) + - [Aggregation](/explain-aggregation.md) + - [Views](/explain-views.md) + - [Partitions](/explain-partitions.md) + - SQL Optimization Process + - [Overview](/sql-optimization-concepts.md) + - Logic Optimization + - [Overview](/sql-logical-optimization.md) + - [Subquery Related Optimizations](/subquery-optimization.md) + - [Column Pruning](/column-pruning.md) + - [Decorrelation of Correlated Subquery](/correlated-subquery-optimization.md) + - [Eliminate Max/Min](/max-min-eliminate.md) + - [Predicates Push Down](/predicate-push-down.md) + - [Partition Pruning](/partition-pruning.md) + - [TopN and Limit Push Down](/topn-limit-push-down.md) + - [Join Reorder](/join-reorder.md) + - Physical Optimization + - [Overview](/sql-physical-optimization.md) + - [Index Selection](/choose-index.md) + - [Statistics](/statistics.md) + - [Wrong Index Solution](/wrong-index-solution.md) + - [Distinct Optimization](/agg-distinct-optimization.md) + - [Prepare Execution Plan Cache](/sql-prepare-plan-cache.md) + - Control Execution Plans + - [Overview](/control-execution-plan.md) + - [Optimizer Hints](/optimizer-hints.md) + - [SQL Plan Management](/sql-plan-management.md) + - [The Blocklist of Optimization Rules and Expression Pushdown](/blocklist-control-plan.md) +- Tutorials + - [Multiple Data Centers in One City Deployment](/multi-data-centers-in-one-city-deployment.md) + - [Three Data Centers in Two Cities Deployment](/three-data-centers-in-two-cities-deployment.md) + - [Two Data Centers in One City Deployment](/two-data-centers-in-one-city-deployment.md) + - Read Historical Data + - Use Stale Read (Recommended) + - [Usage Scenarios of Stale Read](/stale-read.md) + - [Perform Stale Read Using `As OF TIMESTAMP`](/as-of-timestamp.md) + - [Use the `tidb_snapshot` System Variable](/read-historical-data.md) + - Best Practices + - [Use TiDB](/best-practices/tidb-best-practices.md) + - [Java Application Development](/best-practices/java-app-best-practices.md) + - [Use HAProxy](/best-practices/haproxy-best-practices.md) + - [Highly Concurrent Write](/best-practices/high-concurrency-best-practices.md) + - [Grafana Monitoring](/best-practices/grafana-monitor-best-practices.md) + - [PD Scheduling](/best-practices/pd-scheduling-best-practices.md) + - [TiKV Performance Tuning with Massive Regions](/best-practices/massive-regions-best-practices.md) + - [Three-node Hybrid Deployment](/best-practices/three-nodes-hybrid-deployment.md) + - [Local Read Under Three Data Centers Deployment](/best-practices/three-dc-local-read.md) + - [Use Placement Rules](/configure-placement-rules.md) + - [Use Load Base Split](/configure-load-base-split.md) + - [Use Store Limit](/configure-store-limit.md) +- TiDB Ecosystem Tools + - [Overview](/ecosystem-tool-user-guide.md) + - [Use Cases](/ecosystem-tool-user-case.md) + - [Download](/download-ecosystem-tools.md) + - Backup & Restore (BR) + - [BR Tool Overview](/br/backup-and-restore-tool.md) + - [Use BR Command-line for Backup and Restoration](/br/use-br-command-line-tool.md) + - [BR Use Cases](/br/backup-and-restore-use-cases.md) + - [External Storages](/br/backup-and-restore-storages.md) + - [BR FAQ](/br/backup-and-restore-faq.md) + - TiDB Binlog + - [Overview](/tidb-binlog/tidb-binlog-overview.md) + - [Quick Start](/tidb-binlog/get-started-with-tidb-binlog.md) + - [Deploy](/tidb-binlog/deploy-tidb-binlog.md) + - [Maintain](/tidb-binlog/maintain-tidb-binlog-cluster.md) + - [Configure](/tidb-binlog/tidb-binlog-configuration-file.md) + - [Pump](/tidb-binlog/tidb-binlog-configuration-file.md#pump) + - [Drainer](/tidb-binlog/tidb-binlog-configuration-file.md#drainer) + - [Upgrade](/tidb-binlog/upgrade-tidb-binlog.md) + - [Monitor](/tidb-binlog/monitor-tidb-binlog-cluster.md) + - [Reparo](/tidb-binlog/tidb-binlog-reparo.md) + - [binlogctl](/tidb-binlog/binlog-control.md) + - [Binlog Consumer Client](/tidb-binlog/binlog-consumer-client.md) + - [TiDB Binlog Relay Log](/tidb-binlog/tidb-binlog-relay-log.md) + - [Bidirectional Replication Between TiDB Clusters](/tidb-binlog/bidirectional-replication-between-tidb-clusters.md) + - [Glossary](/tidb-binlog/tidb-binlog-glossary.md) + - Troubleshoot + - [Troubleshoot](/tidb-binlog/troubleshoot-tidb-binlog.md) + - [Handle Errors](/tidb-binlog/handle-tidb-binlog-errors.md) + - [FAQ](/tidb-binlog/tidb-binlog-faq.md) + - TiDB Lightning + - [Overview](/tidb-lightning/tidb-lightning-overview.md) + - [Tutorial](/get-started-with-tidb-lightning.md) + - [Deploy](/tidb-lightning/deploy-tidb-lightning.md) + - [Precheck](/tidb-lightning/tidb-lightning-prechecks.md) + - [Configure](/tidb-lightning/tidb-lightning-configuration.md) + - Key Features + - [Checkpoints](/tidb-lightning/tidb-lightning-checkpoints.md) + - [Table Filter](/table-filter.md) + - [CSV Support](/tidb-lightning/migrate-from-csv-using-tidb-lightning.md) + - [Backends](/tidb-lightning/tidb-lightning-backends.md) + - [Import Data in Parallel](/tidb-lightning/tidb-lightning-distributed-import.md) + - [Web Interface](/tidb-lightning/tidb-lightning-web-interface.md) + - [Monitor](/tidb-lightning/monitor-tidb-lightning.md) + - [FAQ](/tidb-lightning/tidb-lightning-faq.md) + - [Glossary](/tidb-lightning/tidb-lightning-glossary.md) + - TiCDC + - [Overview](/ticdc/ticdc-overview.md) + - [Deploy](/ticdc/deploy-ticdc.md) + - [Maintain](/ticdc/manage-ticdc.md) + - [Troubleshoot](/ticdc/troubleshoot-ticdc.md) + - [Monitor](/ticdc/monitor-ticdc.md) + - [Alert Rules](/ticdc/ticdc-alert-rules.md) + - [TiCDC OpenAPI](/ticdc/ticdc-open-api.md) + - [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) + - [Integrate TiDB with Confluent Platform](/ticdc/integrate-confluent-using-ticdc.md) + - [Glossary](/ticdc/ticdc-glossary.md) + - [Dumpling](/dumpling-overview.md) + - sync-diff-inspector + - [Overview](/sync-diff-inspector/sync-diff-inspector-overview.md) + - [Data Check for Tables with Different Schema/Table Names](/sync-diff-inspector/route-diff.md) + - [Data Check in the Sharding Scenario](/sync-diff-inspector/shard-diff.md) + - [Data Check for TiDB Upstream/Downstream Clusters](/sync-diff-inspector/upstream-downstream-diff.md) + - [Data Check in the DM Replication Scenario](/sync-diff-inspector/dm-diff.md) + - TiSpark + - [Quick Start](/get-started-with-tispark.md) + - [User Guide](/tispark-overview.md) +- Reference + - Cluster Architecture + - [Overview](/tidb-architecture.md) + - [Storage](/tidb-storage.md) + - [Computing](/tidb-computing.md) + - [Scheduling](/tidb-scheduling.md) + - Key Monitoring Metrics + - [Overview](/grafana-overview-dashboard.md) + - [TiDB](/grafana-tidb-dashboard.md) + - [PD](/grafana-pd-dashboard.md) + - [TiKV](/grafana-tikv-dashboard.md) + - [TiFlash](/tiflash/monitor-tiflash.md) + - [TiCDC](/ticdc/monitor-ticdc.md) + - Secure + - [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) + - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) + - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) + - [Encryption at Rest](/encryption-at-rest.md) + - [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) + - [Log Redaction](/log-redaction.md) + - Privileges + - [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) + - [Privilege Management](/privilege-management.md) + - [User Account Management](/user-account-management.md) + - [Role-Based Access Control](/role-based-access-control.md) + - [Certificate-Based Authentication](/certificate-authentication.md) + - SQL + - SQL Language Structure and Syntax + - Attributes + - [AUTO_INCREMENT](/auto-increment.md) + - [AUTO_RANDOM](/auto-random.md) + - [SHARD_ROW_ID_BITS](/shard-row-id-bits.md) + - [Literal Values](/literal-values.md) + - [Schema Object Names](/schema-object-names.md) + - [Keywords and Reserved Words](/keywords.md) + - [User-Defined Variables](/user-defined-variables.md) + - [Expression Syntax](/expression-syntax.md) + - [Comment Syntax](/comment-syntax.md) + - SQL Statements + - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [`ADMIN`](/sql-statements/sql-statement-admin.md) + - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) + - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) + - [`ADMIN CHECK [TABLE|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) + - [`ADMIN SHOW DDL [JOBS|QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) + - [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) + - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) + - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) + - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) + - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) + - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) + - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) + - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) + - [`BACKUP`](/sql-statements/sql-statement-backup.md) + - [`BEGIN`](/sql-statements/sql-statement-begin.md) + - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`COMMIT`](/sql-statements/sql-statement-commit.md) + - [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) + - [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) + - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) + - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) + - [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) + - [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) + - [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) + - [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) + - [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) + - [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) + - [`CREATE USER`](/sql-statements/sql-statement-create-user.md) + - [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) + - [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) + - [`DELETE`](/sql-statements/sql-statement-delete.md) + - [`DESC`](/sql-statements/sql-statement-desc.md) + - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) + - [`DO`](/sql-statements/sql-statement-do.md) + - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) + - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) + - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) + - [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) + - [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) + - [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) + - [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) + - [`DROP USER`](/sql-statements/sql-statement-drop-user.md) + - [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) + - [`EXECUTE`](/sql-statements/sql-statement-execute.md) + - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) + - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) + - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) + - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) + - [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) + - [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) + - [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + - [`GRANT `](/sql-statements/sql-statement-grant-role.md) + - [`INSERT`](/sql-statements/sql-statement-insert.md) + - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) + - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) + - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) + - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`PREPARE`](/sql-statements/sql-statement-prepare.md) + - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) + - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) + - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) + - [`REPLACE`](/sql-statements/sql-statement-replace.md) + - [`RESTORE`](/sql-statements/sql-statement-restore.md) + - [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) + - [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) + - [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) + - [`SELECT`](/sql-statements/sql-statement-select.md) + - [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) + - [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) + - [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) + - [`SET ROLE`](/sql-statements/sql-statement-set-role.md) + - [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) + - [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) + - [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) + - [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) + - [`SHOW [GLOBAL|SESSION] BINDINGS`](/sql-statements/sql-statement-show-bindings.md) + - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) + - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) + - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) + - [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) + - [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) + - [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) + - [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) + - [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) + - [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) + - [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) + - [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) + - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) + - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) + - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) + - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) + - [`SHOW INDEX [FROM|IN]`](/sql-statements/sql-statement-show-index.md) + - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) + - [`SHOW KEYS [FROM|IN]`](/sql-statements/sql-statement-show-keys.md) + - [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) + - [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) + - [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) + - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) + - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) + - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) + - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) + - [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) + - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) + - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-histograms.md) + - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) + - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) + - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) + - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) + - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) + - [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) + - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) + - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) + - [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) + - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) + - [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) + - [`TABLE`](/sql-statements/sql-statement-table.md) + - [`TRACE`](/sql-statements/sql-statement-trace.md) + - [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) + - [`UPDATE`](/sql-statements/sql-statement-update.md) + - [`USE`](/sql-statements/sql-statement-use.md) + - [`WITH`](/sql-statements/sql-statement-with.md) + - Data Types + - [Overview](/data-type-overview.md) + - [Default Values](/data-type-default-values.md) + - [Numeric Types](/data-type-numeric.md) + - [Date and Time Types](/data-type-date-and-time.md) + - [String Types](/data-type-string.md) + - [JSON Type](/data-type-json.md) + - Functions and Operators + - [Overview](/functions-and-operators/functions-and-operators-overview.md) + - [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) + - [Operators](/functions-and-operators/operators.md) + - [Control Flow Functions](/functions-and-operators/control-flow-functions.md) + - [String Functions](/functions-and-operators/string-functions.md) + - [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) + - [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) + - [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) + - [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) + - [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) + - [Information Functions](/functions-and-operators/information-functions.md) + - [JSON Functions](/functions-and-operators/json-functions.md) + - [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) + - [Window Functions](/functions-and-operators/window-functions.md) + - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) + - [Precision Math](/functions-and-operators/precision-math.md) + - [Set Operations](/functions-and-operators/set-operators.md) + - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) + - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) + - [Clustered Indexes](/clustered-indexes.md) + - [Constraints](/constraints.md) + - [Generated Columns](/generated-columns.md) + - [SQL Mode](/sql-mode.md) + - [Table Attributes](/table-attributes.md) + - Transactions + - [Overview](/transaction-overview.md) + - [Isolation Levels](/transaction-isolation-levels.md) + - [Optimistic Transactions](/optimistic-transaction.md) + - [Pessimistic Transactions](/pessimistic-transaction.md) + - Garbage Collection (GC) + - [Overview](/garbage-collection-overview.md) + - [Configuration](/garbage-collection-configuration.md) + - [Views](/views.md) + - [Partitioning](/partitioned-table.md) + - [Temporary Tables](/temporary-tables.md) + - [Character Set and Collation](/character-set-and-collation.md) + - [Placement Rules in SQL](/placement-rules-in-sql.md) + - System Tables + - [`mysql`](/mysql-schema.md) + - INFORMATION_SCHEMA + - [Overview](/information-schema/information-schema.md) + - [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) + - [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) + - [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) + - [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) + - [`CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) + - [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) + - [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-hardware.md) + - [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) + - [`CLUSTER_LOAD`](/information-schema/information-schema-cluster-load.md) + - [`CLUSTER_LOG`](/information-schema/information-schema-cluster-log.md) + - [`CLUSTER_SYSTEMINFO`](/information-schema/information-schema-cluster-systeminfo.md) + - [`COLLATIONS`](/information-schema/information-schema-collations.md) + - [`COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) + - [`COLUMNS`](/information-schema/information-schema-columns.md) + - [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) + - [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) + - [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) + - [`ENGINES`](/information-schema/information-schema-engines.md) + - [`INSPECTION_RESULT`](/information-schema/information-schema-inspection-result.md) + - [`INSPECTION_RULES`](/information-schema/information-schema-inspection-rules.md) + - [`INSPECTION_SUMMARY`](/information-schema/information-schema-inspection-summary.md) + - [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) + - [`METRICS_SUMMARY`](/information-schema/information-schema-metrics-summary.md) + - [`METRICS_TABLES`](/information-schema/information-schema-metrics-tables.md) + - [`PARTITIONS`](/information-schema/information-schema-partitions.md) + - [`PLACEMENT_RULES`](/information-schema/information-schema-placement-rules.md) + - [`PROCESSLIST`](/information-schema/information-schema-processlist.md) + - [`REFERENTIAL_CONSTRAINTS`](/information-schema/information-schema-referential-constraints.md) + - [`SCHEMATA`](/information-schema/information-schema-schemata.md) + - [`SEQUENCES`](/information-schema/information-schema-sequences.md) + - [`SESSION_VARIABLES`](/information-schema/information-schema-session-variables.md) + - [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) + - [`STATISTICS`](/information-schema/information-schema-statistics.md) + - [`TABLES`](/information-schema/information-schema-tables.md) + - [`TABLE_CONSTRAINTS`](/information-schema/information-schema-table-constraints.md) + - [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) + - [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) + - [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) + - [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) + - [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) + - [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) + - [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) + - [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) + - [`TIKV_STORE_STATUS`](/information-schema/information-schema-tikv-store-status.md) + - [`USER_PRIVILEGES`](/information-schema/information-schema-user-privileges.md) + - [`VIEWS`](/information-schema/information-schema-views.md) + - [`METRICS_SCHEMA`](/metrics-schema.md) + - UI + - TiDB Dashboard + - [Overview](/dashboard/dashboard-intro.md) + - Maintain + - [Deploy](/dashboard/dashboard-ops-deploy.md) + - [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) + - [User Management](/dashboard/dashboard-user.md) + - [Secure](/dashboard/dashboard-ops-security.md) + - [Access](/dashboard/dashboard-access.md) + - [Overview Page](/dashboard/dashboard-overview.md) + - [Cluster Info Page](/dashboard/dashboard-cluster-info.md) + - [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) + - [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) + - SQL Statements Analysis + - [SQL Statements Page](/dashboard/dashboard-statement-list.md) + - [SQL Details Page](/dashboard/dashboard-statement-details.md) + - [Slow Queries Page](/dashboard/dashboard-slow-query.md) + - Cluster Diagnostics + - [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) + - [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) + - [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) + - [Search Logs Page](/dashboard/dashboard-log-search.md) + - [Instance Profiling] + - [Manual Profiling](/dashboard/dashboard-profiling.md) + - [Continuous Profiling](/dashboard/continuous-profiling.md) + - Session Management and Configuration + - [Share Session](/dashboard/dashboard-session-share.md) + - [Configure SSO](/dashboard/dashboard-session-sso.md) + - [FAQ](/dashboard/dashboard-faq.md) + - CLI + - [tikv-ctl](/tikv-control.md) + - [pd-ctl](/pd-control.md) + - [tidb-ctl](/tidb-control.md) + - [pd-recover](/pd-recover.md) + - Command Line Flags + - [tidb-server](/command-line-flags-for-tidb-configuration.md) + - [tikv-server](/command-line-flags-for-tikv-configuration.md) + - [tiflash-server](/tiflash/tiflash-command-line-flags.md) + - [pd-server](/command-line-flags-for-pd-configuration.md) + - Configuration File Parameters + - [tidb-server](/tidb-configuration-file.md) + - [tikv-server](/tikv-configuration-file.md) + - [tiflash-server](/tiflash/tiflash-configuration.md) + - [pd-server](/pd-configuration-file.md) + - [System Variables](/system-variables.md) + - Storage Engines + - TiKV + - [TiKV Overview](/tikv-overview.md) + - [RocksDB Overview](/storage-engine/rocksdb-overview.md) + - [Titan Overview](/storage-engine/titan-overview.md) + - [Titan Configuration](/storage-engine/titan-configuration.md) + - TiFlash + - [Overview](/tiflash/tiflash-overview.md) + - [Use TiFlash](/tiflash/use-tiflash.md) + - TiUP + - [Documentation Map](/tiup/tiup-documentation-guide.md) + - [Overview](/tiup/tiup-overview.md) + - [Terminology and Concepts](/tiup/tiup-terminology-and-concepts.md) + - [Manage TiUP Components](/tiup/tiup-component-management.md) + - [FAQ](/tiup/tiup-faq.md) + - [Troubleshooting Guide](/tiup/tiup-troubleshooting-guide.md) + - [Reference Guide](/tiup/tiup-reference.md) + - [Cluster Topology Reference](/tiup/tiup-cluster-topology-reference.md) + - [Mirror Reference Guide](/tiup/tiup-mirror-reference.md) + - TiUP Components + - [tiup-playground](/tiup/tiup-playground.md) + - [tiup-cluster](/tiup/tiup-cluster.md) + - [tiup-mirror](/tiup/tiup-mirror.md) + - [tiup-bench](/tiup/tiup-bench.md) + - [Telemetry](/telemetry.md) + - [Errors Codes](/error-codes.md) + - [Table Filter](/table-filter.md) + - [Schedule Replicas by Topology Labels](/schedule-replicas-by-topology-labels.md) +- FAQs + - [TiDB FAQs](/faq/tidb-faq.md) + - [SQL FAQs](/faq/sql-faq.md) + - [Deploy and Maintain FAQs](/faq/deploy-and-maintain-faq.md) + - [Upgrade FAQs](/faq/upgrade-faq.md) + - [High Availability FAQs](/faq/high-availability-faq.md) + - [High Reliability FAQs](/faq/high-reliability-faq.md) + - [Migration FAQs](/faq/migration-tidb-faq.md) +- [Glossary](/glossary.md) +- Release Notes + - [All Releases](/releases/release-notes.md) + - [Release Timeline](/releases/release-timeline.md) + - v5.3 + - [5.3.0](/releases/release-5.3.0.md) + - v5.2 + - [5.2.3](/releases/release-5.2.3.md) + - [5.2.2](/releases/release-5.2.2.md) + - [5.2.1](/releases/release-5.2.1.md) + - [5.2.0](/releases/release-5.2.0.md) + - v5.1 + - [5.1.2](/releases/release-5.1.2.md) + - [5.1.1](/releases/release-5.1.1.md) + - [5.1.0](/releases/release-5.1.0.md) + - v5.0 + - [5.0.4](/releases/release-5.0.4.md) + - [5.0.3](/releases/release-5.0.3.md) + - [5.0.2](/releases/release-5.0.2.md) + - [5.0.1](/releases/release-5.0.1.md) + - [5.0 GA](/releases/release-5.0.0.md) + - [5.0.0-rc](/releases/release-5.0.0-rc.md) + - v4.0 + - [4.0.15](/releases/release-4.0.15.md) + - [4.0.14](/releases/release-4.0.14.md) + - [4.0.13](/releases/release-4.0.13.md) + - [4.0.12](/releases/release-4.0.12.md) + - [4.0.11](/releases/release-4.0.11.md) + - [4.0.10](/releases/release-4.0.10.md) + - [4.0.9](/releases/release-4.0.9.md) + - [4.0.8](/releases/release-4.0.8.md) + - [4.0.7](/releases/release-4.0.7.md) + - [4.0.6](/releases/release-4.0.6.md) + - [4.0.5](/releases/release-4.0.5.md) + - [4.0.4](/releases/release-4.0.4.md) + - [4.0.3](/releases/release-4.0.3.md) + - [4.0.2](/releases/release-4.0.2.md) + - [4.0.1](/releases/release-4.0.1.md) + - [4.0 GA](/releases/release-4.0-ga.md) + - [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) + - [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) + - [4.0.0-rc](/releases/release-4.0.0-rc.md) + - [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) + - [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) + - [4.0.0-beta](/releases/release-4.0.0-beta.md) + - v3.1 + - [3.1.2](/releases/release-3.1.2.md) + - [3.1.1](/releases/release-3.1.1.md) + - [3.1.0 GA](/releases/release-3.1.0-ga.md) + - [3.1.0-rc](/releases/release-3.1.0-rc.md) + - [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) + - [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) + - [3.1.0-beta](/releases/release-3.1.0-beta.md) + - v3.0 + - [3.0.20](/releases/release-3.0.20.md) + - [3.0.19](/releases/release-3.0.19.md) + - [3.0.18](/releases/release-3.0.18.md) + - [3.0.17](/releases/release-3.0.17.md) + - [3.0.16](/releases/release-3.0.16.md) + - [3.0.15](/releases/release-3.0.15.md) + - [3.0.14](/releases/release-3.0.14.md) + - [3.0.13](/releases/release-3.0.13.md) + - [3.0.12](/releases/release-3.0.12.md) + - [3.0.11](/releases/release-3.0.11.md) + - [3.0.10](/releases/release-3.0.10.md) + - [3.0.9](/releases/release-3.0.9.md) + - [3.0.8](/releases/release-3.0.8.md) + - [3.0.7](/releases/release-3.0.7.md) + - [3.0.6](/releases/release-3.0.6.md) + - [3.0.5](/releases/release-3.0.5.md) + - [3.0.4](/releases/release-3.0.4.md) + - [3.0.3](/releases/release-3.0.3.md) + - [3.0.2](/releases/release-3.0.2.md) + - [3.0.1](/releases/release-3.0.1.md) + - [3.0 GA](/releases/release-3.0-ga.md) + - [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) + - [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) + - [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) + - [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) + - [3.0.0-beta](/releases/release-3.0-beta.md) + - v2.1 + - [2.1.19](/releases/release-2.1.19.md) + - [2.1.18](/releases/release-2.1.18.md) + - [2.1.17](/releases/release-2.1.17.md) + - [2.1.16](/releases/release-2.1.16.md) + - [2.1.15](/releases/release-2.1.15.md) + - [2.1.14](/releases/release-2.1.14.md) + - [2.1.13](/releases/release-2.1.13.md) + - [2.1.12](/releases/release-2.1.12.md) + - [2.1.11](/releases/release-2.1.11.md) + - [2.1.10](/releases/release-2.1.10.md) + - [2.1.9](/releases/release-2.1.9.md) + - [2.1.8](/releases/release-2.1.8.md) + - [2.1.7](/releases/release-2.1.7.md) + - [2.1.6](/releases/release-2.1.6.md) + - [2.1.5](/releases/release-2.1.5.md) + - [2.1.4](/releases/release-2.1.4.md) + - [2.1.3](/releases/release-2.1.3.md) + - [2.1.2](/releases/release-2.1.2.md) + - [2.1.1](/releases/release-2.1.1.md) + - [2.1 GA](/releases/release-2.1-ga.md) + - [2.1 RC5](/releases/release-2.1-rc.5.md) + - [2.1 RC4](/releases/release-2.1-rc.4.md) + - [2.1 RC3](/releases/release-2.1-rc.3.md) + - [2.1 RC2](/releases/release-2.1-rc.2.md) + - [2.1 RC1](/releases/release-2.1-rc.1.md) + - [2.1 Beta](/releases/release-2.1-beta.md) + - v2.0 + - [2.0.11](/releases/release-2.0.11.md) + - [2.0.10](/releases/release-2.0.10.md) + - [2.0.9](/releases/release-2.0.9.md) + - [2.0.8](/releases/release-2.0.8.md) + - [2.0.7](/releases/release-2.0.7.md) + - [2.0.6](/releases/release-2.0.6.md) + - [2.0.5](/releases/release-2.0.5.md) + - [2.0.4](/releases/release-2.0.4.md) + - [2.0.3](/releases/release-2.0.3.md) + - [2.0.2](/releases/release-2.0.2.md) + - [2.0.1](/releases/release-2.0.1.md) + - [2.0](/releases/release-2.0-ga.md) + - [2.0 RC5](/releases/release-2.0-rc.5.md) + - [2.0 RC4](/releases/release-2.0-rc.4.md) + - [2.0 RC3](/releases/release-2.0-rc.3.md) + - [2.0 RC1](/releases/release-2.0-rc.1.md) + - [1.1 Beta](/releases/release-1.1-beta.md) + - [1.1 Alpha](/releases/release-1.1-alpha.md) + - v1.0 + - [1.0.8](/releases/release-1.0.8.md) + - [1.0.7](/releases/release-1.0.7.md) + - [1.0.6](/releases/release-1.0.6.md) + - [1.0.5](/releases/release-1.0.5.md) + - [1.0.4](/releases/release-1.0.4.md) + - [1.0.3](/releases/release-1.0.3.md) + - [1.0.2](/releases/release-1.0.2.md) + - [1.0.1](/releases/release-1.0.1.md) + - [1.0](/releases/release-1.0-ga.md) + - [Pre-GA](/releases/release-pre-ga.md) + - [RC4](/releases/release-rc.4.md) + - [RC3](/releases/release-rc.3.md) + - [RC2](/releases/release-rc.2.md) + - [RC1](/releases/release-rc.1.md) diff --git a/_index.md b/_index.md index 15a43923d1025..896dc05ce32a9 100644 --- a/_index.md +++ b/_index.md @@ -16,7 +16,8 @@ Designed for the cloud, TiDB provides flexible scalability, reliability and secu - [TiDB Introduction](/overview.md) - [Basic Features](/basic-features.md) -- [TiDB 5.2 Release Notes](/releases/release-5.2.0.md) +- [TiDB 5.3 Release Notes](/releases/release-5.3.0.md) +- [TiDB Release Timeline](/releases/release-timeline.md) - [Compatibility with MySQL](/mysql-compatibility.md) - [Usage Limitations](/tidb-limitations.md) - [TiDB Adopters](/adopters.md) diff --git a/adopters.md b/adopters.md index 94fc942c9ccbc..f48d460ee8d6c 100644 --- a/adopters.md +++ b/adopters.md @@ -10,6 +10,7 @@ This is a list of TiDB adopters in various industries. | Company | Industry | Case studies | | :--- | :--- | :--- | +|[SHAREit Group](https://www.crunchbase.com/organization/shareit-2)|Internet|[English](https://pingcap.com/case-studies/why-shareit-selects-tikv-for-data-storage-for-its-2.4-billion-user-business); [Chinese](https://pingcap.com/zh/blog/devcon-2021-shareit)| |[China Telecom Bestpay](https://www.crunchbase.com/organization/orange-finance)|Mobile Payment|[English](https://pingcap.com/case-studies/how-we-process-data-five-times-more-efficiently-using-a-scale-out-mysql-alternative/); [Chinese](https://pingcap.com/cases-cn/user-case-bestpay/)| |[VNG](https://en.wikipedia.org/wiki/VNG_Corporation)|Mobile Payment|English [#1](https://pingcap.com/case-studies/tidb-at-zalopay-infrastructure-lesson-learned/), [#2](https://pingcap.com/case-studies/zalopay-using-a-scale-out-mysql-alternative-to-serve-millions-of-users)| |[Ping++](https://www.crunchbase.com/organization/ping-5)|Mobile Payment|| @@ -25,7 +26,6 @@ This is a list of TiDB adopters in various industries. |[Meituan](https://www.crunchbase.com/organization/meituan)|E-commerce|English [#1](https://pingcap.com/case-studies/migrating-from-mysql-to-a-scale-out-database-to-serve-our-290-million-monthly-users), [#2](https://pingcap.com/case-studies/how-we-use-a-mysql-alternative-to-avoid-sharding-and-provide-strong-consistency); [Chinese](https://pingcap.com/zh/case/user-case-meituandianping)| |[Happigo.com](https://www.crunchbase.com/organization/happigo-com)|E-commerce|| |[Yimutian](https://www.crunchbase.com/organization/yimutian)|E-commerce|| -|[Youju Tech](https://nn.yjyz.com/)|E-commerce|| |[Maizuo](https://www.crunchbase.com/organization/maizhuo)|E-commerce|| |[Mogujie](https://www.crunchbase.com/organization/mogujie)|E-commerce|| |[Zhihu](https://en.wikipedia.org/wiki/Zhihu)|Knowledge Sharing|English [#1](https://pingcap.com/case-studies/lesson-learned-from-queries-over-1.3-trillion-rows-of-data-within-milliseconds-of-response-time-at-zhihu/), [#2](https://pingcap.com/case-studies/horizontally-scaling-hive-metastore-database-by-migrating-from-mysql-to-tidb), [#3](https://pingcap.com/case-studies/boosting-big-data-performance-by-combining-tidb-with-hive-and-hdfs)| @@ -53,6 +53,7 @@ This is a list of TiDB adopters in various industries. |[Mobikok](https://www.linkedin.com/company/shenzhen-keke-network-technology-co.-ltd./)|AdTech|| |[Ninja Van](https://www.crunchbase.com/organization/ninja-van-2)| Logistics|[English](https://pingcap.com/case-studies/choose-a-mysql-alternative-over-vitess-and-crdb-to-scale-out-our-databases-on-k8s)| |[ZTO Express](https://www.crunchbase.com/organization/zto-express)| Logistics|[English](https://pingcap.com/case-studies/why-we-migrated-from-exadata-to-a-scale-out-htap-database-for-near-real-time-analytics); [Chinese](https://pingcap.com/cases-cn/user-case-zto-express/)| +|[JD.com](https://en.wikipedia.org/wiki/JD.com)|Logistics|[English](https://pingcap.com/case-studies/8x-system-performance-boost-why-we-migrated-from-mysql-to-newsql-database); [Chinese](https://pingcap.com/zh/case/user-case-jdl)| |[G7 Networks](https://www.crunchbase.com/organization/g7)| Logistics|| |[Hive-Box](http://www.fcbox.com/en/pc/index.html#/)|Logistics|| |[NetEase Games](https://www.linkedin.com/company/netease-games)|Gaming|[English](https://pingcap.com/case-studies/why-we-chose-tidb-over-other-mysql-based-and-newsql-storage-solutions); [Chinese](https://pingcap.com/cases-cn/user-case-wangyihuyu/)| diff --git a/alert-rules.md b/alert-rules.md index b6d482271238e..91ef1464cb2e8 100644 --- a/alert-rules.md +++ b/alert-rules.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/alert-rules/','/docs/dev/reference/alert-rules/'] # TiDB Cluster Alert Rules -This document describes the alert rules for different components in a TiDB cluster, including the rule descriptions and solutions of the alert items in TiDB, TiKV, PD, TiFlash, TiDB Binlog, Node_exporter and Blackbox_exporter. +This document describes the alert rules for different components in a TiDB cluster, including the rule descriptions and solutions of the alert items in TiDB, TiKV, PD, TiFlash, TiDB Binlog, TiCDC, Node_exporter and Blackbox_exporter. According to the severity level, alert rules are divided into three categories (from high to low): emergency-level, critical-level, and warning-level. This division of severity levels applies to all alert items of each component below. @@ -345,7 +345,7 @@ This section gives the alert rules for the PD component. * Alert rule: - `count(changes(pd_server_tso{type="save"}[10m]) > 0) >= 2` + `count(changes(pd_tso_events{type="save"}[10m]) > 0) >= 2` * Description: @@ -376,7 +376,7 @@ This section gives the alert rules for the PD component. * Alert rule: - `changes(pd_server_tso{type="system_time_slow"}[10m]) >= 1` + `changes(pd_tso_events{type="system_time_slow"}[10m]) >= 1` * Description: @@ -789,6 +789,10 @@ For the detailed descriptions of TiFlash alert rules, see [TiFlash Alert Rules]( For the detailed descriptions of TiDB Binlog alert rules, see [TiDB Binlog monitoring document](/tidb-binlog/monitor-tidb-binlog-cluster.md#alert-rules). +## TiCDC Alert rules + +For the detailed descriptions of TiCDC alert rules, see [TiCDC Alert Rules](/ticdc/ticdc-alert-rules.md). + ## Node_exporter host alert rules This section gives the alert rules for the Node_exporter host. @@ -924,7 +928,7 @@ This section gives the alert rules for the Node_exporter host. * Alert rule: - `((rate(node_disk_write_time_seconds_total{device=~".+"}[5m]) / rate(node_disk_writes_completed_total{device=~".+"}[5m])) or (irate(node_disk_write_time_seconds_total{device=~".+"}[5m]) / irate(node_disk_writes_completed_total{device=~".+"}[5m]))) * 1000 > 16` + `((rate(node_disk_write_time_seconds_total{device=~".+"}[5m]) / rate(node_disk_writes_completed_total{device=~".+"}[5m])) or (irate(node_disk_write_time_seconds_total{device=~".+"}[5m]) / irate(node_disk_writes_completed_total{device=~".+"}[5m])))> 16` * Description: diff --git a/basic-features.md b/basic-features.md index fc1a7b2738f88..8de3482699269 100644 --- a/basic-features.md +++ b/basic-features.md @@ -6,129 +6,176 @@ aliases: ['/docs/dev/basic-features/'] # TiDB Features -The following table provides an overview of the feature development history of TiDB. Note that features under active development might change before the final release. - -| Data types, functions, and operators | 5.2 | 5.1 | 5.0 | 4.0 | -|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:| -| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | -| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | -| [String types](/data-type-string.md) | Y | Y | Y | Y | -| [JSON type](/data-type-json.md) | Experimental | Experimental | Experimental | Experimental | -| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | -| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | -| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | -| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | -| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | -| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | -| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | -| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | -| [JSON functions](/functions-and-operators/json-functions.md) | Experimental | Experimental | Experimental | Experimental | -| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | -| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | -| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | -| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | -| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | -| **Indexing and constraints** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) | Experimental | Experimental | Experimental | Experimental | -| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | -| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | -| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | -| [Invisible indexes](/sql-statements/sql-statement-add-index.md) | Y | Y | Y | N | -| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | -| [Unique indexes](/constraints.md) | Y | Y | Y | Y | -| [Clustered index on integer `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | -| [Clustered index on composite or non-integer key](/constraints.md) | Y | Y | Y | N | -| **SQL statements** [^2] | **5.2** | **5.1** | **5.0** | **4.0** | -| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | -| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | -| `LOAD DATA INFILE` | Y | Y | Y | Y | -| `SELECT INTO OUTFILE` | Y | Y | Y | Y | -| `INNER JOIN`, `LEFT\|RIGHT [OUTER] JOIN` | Y | Y | Y | Y | -| `UNION`, `UNION ALL` | Y | Y | Y | Y | -| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | N | -| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | -| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | -| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | N | N | -| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | -| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | -| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | -| [User-defined variables](/user-defined-variables.md) | Experimental | Experimental | Experimental | Experimental | -| **Advanced SQL Features** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Prepared statement cache](/sql-prepare-plan-cache.md) | Experimental | Experimental | Experimental | Experimental | -| [SQL plan management (SPM)](/sql-plan-management.md) | Y | Y | Y | Y | -| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Experimental | -| [Stale Read](/stale-read.md) | Y | Y | N | N | -| [Follower reads](/follower-read.md) | Y | Y | Y | Y | -| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | -| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | -| [MPP Exection Engine](/explain-mpp.md) | Y | Y | Y | N | -| [Index Merge Join](/explain-index-merge.md) | Experimental | Experimental | Experimental | Experimental | -| **Data definition language (DDL)** | **5.2** | **5.1** | **5.0** | **4.0** | -| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | -| [Generated columns](/generated-columns.md) | Experimental | Experimental | Experimental | Experimental | -| [Views](/views.md) | Y | Y | Y | Y | -| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | -| [Auto increment](/auto-increment.md) | Y | Y | Y | Y | -| [Auto random](/auto-random.md) | Y | Y | Y | Y | -| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | -| Multi schema change: add column(s) | Y | Y | Y | N | -| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | N | N | -| **Transactions** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | N | -| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | N | -| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | -| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | -| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | -| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | -| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | -| **Partitioning** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Range partitioning](/partitioned-table.md) | Y | Y | Y | Y | -| [Hash partitioning](/partitioned-table.md) | Y | Y | Y | Y | -| [List partitioning](/partitioned-table.md) | Experimental | Experimental | Experimental | N | -| [List COLUMNS partitioning](/partitioned-table.md) | Experimental | Experimental | Experimental | N | -| [`EXCHANGE PARTITION`](/partitioned-table.md) | Experimental | Experimental | Experimental | N | -| [Dynamic Pruning](/partitioned-table.md#dynamic-pruning-mode) | Experimental | Experimental | N | N | -| **Statistics** | **5.2** | **5.1** | **5.0** | **4.0** | -| [CMSketch](/statistics.md) | Deprecated | Deprecated | Deprecated | Y | -| [Histograms](/statistics.md) | Y | Y | Y | Y | -| [Extended statistics (multiple columns)](/statistics.md) | Experimental | Experimental | Experimental | N | -| [Statistics Feedback](/statistics.md#automatic-update) | Experimental | Experimental | Experimental | Experimental | -| [Fast Analyze](/system-variables.md#tidb_enable_fast_analyze) | Experimental | Experimental | Experimental | Experimental | -| **Security** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | -| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | -| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | -| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | -| `caching_sha2_password` authentication | Y | N | N | N | -| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | -| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | N | N | -| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | N | N | -| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | N | -| **Data import and export** | **5.2** | **5.1** | **5.0** | **4.0** | -| [Fast Importer (TiDB Lightning)](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | -| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | -| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | -| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) | Y | Y | Y | N | -| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | -| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) | Deprecated | Deprecated | Deprecated | Deprecated | -| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | -| **Management, observability and tools** | **5.2** | **5.1** | **5.0** | **4.0** | -| [TiDB Dashboard](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | -| [SQL diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Experimental | Experimental | Experimental | Experimental | -| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | -| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | -| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | -| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | -| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | -| Ansible deployment | N | N | N | Deprecated | -| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | -| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | -| Top SQL | Y | N | N | N | -| [Global Kill](/sql-statements/sql-statement-kill.md) | Experimental | Experimental | Experimental | Experimental | -| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Experimental | Experimental | Experimental | -| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Experimental | Experimental | Experimental | Experimental | -| [`SET CONFIG`](/dynamic-config.md) | Experimental | Experimental | Experimental | Experimental | +This document lists the features supported in each TiDB version. Note that supports for experimental features might change before the final release. + +## Data types, functions, and operators + +| Data types, functions, and operators | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | +| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | +| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | +| [JSON type](/data-type-json.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | +| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | +| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | +| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | +| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | +| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | +| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | +| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | +| [JSON functions](/functions-and-operators/json-functions.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | +| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | +| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | +| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | +| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | + +## Indexing and constraints + +| Indexing and constraints | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | +| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | +| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | +| [Invisible indexes](/sql-statements/sql-statement-add-index.md) | Y | Y | Y | Y | N | +| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | +| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | +| [Clustered index on integer `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | +| [Clustered index on composite or non-integer key](/constraints.md) | Y | Y | Y | Y | N | + +## SQL statements + +| **SQL statements** [^2] | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | +| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | +| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | +| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | +| `INNER JOIN`, `LEFT\|RIGHT [OUTER] JOIN` | Y | Y | Y | Y | Y | +| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | +| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | N | +| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | +| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | +| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | N | N | +| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | +| [User-defined variables](/user-defined-variables.md) | Experimental | Experimental | Experimental | Experimental | Experimental | + +## Advanced SQL features + +| **Advanced SQL features** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Prepared statement cache](/sql-prepare-plan-cache.md) | Y | Experimental | Experimental | Experimental | Experimental | +| [SQL plan management (SPM)](/sql-plan-management.md) | Y | Y | Y | Y | Y | +| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Experimental | +| [Stale Read](/stale-read.md) | Y | Y | Y | N | N | +| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | +| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | +| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | +| [MPP Exection Engine](/explain-mpp.md) | Y | Y | Y | Y | N | +| [Index Merge Join](/explain-index-merge.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Placement Rules in SQL](/placement-rules-in-sql.md) | Experimental | N | N | N | N | + +## Data definition language (DDL) + +| **Data definition language (DDL)** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | +| [Generated columns](/generated-columns.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Views](/views.md) | Y | Y | Y | Y | Y | +| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | +| [Auto increment](/auto-increment.md) | Y | Y | Y | Y | Y | +| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | +| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | +| Multi schema change: add column(s) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | N | N | +| [Temporary tables](/temporary-tables.md) | Y | N | N | N | N | + +## Transactions + +| **Transactions** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | N | +| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | N | +| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | +| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | +| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | +| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | +| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | + +## Partitioning + +| **Partitioning** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Range partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | +| [Hash partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | +| [List partitioning](/partitioned-table.md) | Experimental | Experimental | Experimental | Experimental | N | +| [List COLUMNS partitioning](/partitioned-table.md) | Experimental | Experimental | Experimental | Experimental | N | +| [`EXCHANGE PARTITION`](/partitioned-table.md) | Experimental | Experimental | Experimental | Experimental | N | +| [Dynamic Pruning](/partitioned-table.md#dynamic-pruning-mode) | Experimental | Experimental | Experimental | N | N | + +## Statistics + +| **Statistics** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [CMSketch](/statistics.md) | Deprecated | Deprecated | Deprecated | Deprecated | Y | +| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | +| [Extended statistics (multiple columns)](/statistics.md) | Experimental | Experimental | Experimental | Experimental | N | +| [Statistics Feedback](/statistics.md#automatic-update) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Fast Analyze](/system-variables.md#tidb_enable_fast_analyze) | Experimental | Experimental | Experimental | Experimental | Experimental | + +## Security + +| **Security** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | +| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | +| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | +| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | +| `caching_sha2_password` authentication | Y | Y | N | N | N | +| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | +| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | N | N | +| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | N | N | +| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | N | + +## Data import and export + +| **Data import and export** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [Fast Importer (TiDB Lightning)](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | +| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | +| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | +| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) | Y | Y | Y | Y | N [^3] | +| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | +| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) | Y | Y | Y | Y | Y | +| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | + +## Management, observability, and tools + +| **Management, observability, and tools** | **5.3** | **5.2** | **5.1** | **5.0** | **4.0** | +|----------------------------------------------------------------------------------------------------------|:------------:|:------------:|:------------:|:------------:|:------------:| +| [TiDB Dashboard](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | +| [SQL diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | +| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | +| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | +| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | +| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | +| Ansible deployment | N | N | N | N | Deprecated | +| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | +| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | +| Top SQL | Y | Y | N | N | N | +| [Global Kill](/sql-statements/sql-statement-kill.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Experimental | Experimental | Experimental | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [`SET CONFIG`](/dynamic-config.md) | Experimental | Experimental | Experimental | Experimental | Experimental | +| [Continuous Profiling](/dashboard/continuous-profiling.md) | Experimental | N | N | N | N | [^1]: TiDB incorrectly treats latin1 as a subset of utf8. See [TiDB #18955](https://github.com/pingcap/tidb/issues/18955) for more details. [^2]: See [Statement Reference](/sql-statements/sql-statement-select.md) for a full list of SQL statements supported. + +[^3]: For TiDB v4.0, the `LOAD DATA` transaction does not guarantee atomicity. diff --git a/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md b/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md new file mode 100644 index 0000000000000..2a5e44374dcdc --- /dev/null +++ b/benchmark/benchmark-sysbench-v5.3.0-vs-v5.2.2.md @@ -0,0 +1,202 @@ +--- +title: TiDB Sysbench Performance Test Report -- v5.3.0 vs. v5.2.2 +--- + +# TiDB Sysbench Performance Test Report -- v5.3.0 vs. v5.2.2 + +## Test overview + +This test aims at comparing the Sysbench performance of TiDB v5.3.0 and TiDB v5.2.2 in the Online Transactional Processing (OLTP) scenario. The results show that the performance of v5.3.0 is nearly the same as that of v5.2.2. + +## Test environment (AWS EC2) + +### Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| Sysbench | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | v5.2.2 and v5.3.0 | +| TiDB | v5.2.2 and v5.3.0 | +| TiKV | v5.2.2 and v5.3.0 | +| Sysbench | 1.1.0-ead2689 | + +### Parameter configuration + +TiDB v5.3.0 and TiDB v5.2.2 use the same configuration. + +#### TiDB parameter configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV parameter configuration + +{{< copyable "" >}} + +```yaml +storage.scheduler-worker-pool-size: 5 +raftstore.store-pool-size: 3 +raftstore.apply-pool-size: 3 +rocksdb.max-background-jobs: 8 +raftdb.max-background-jobs: 4 +raftdb.allow-concurrent-memtable-write: true +server.grpc-concurrency: 6 +readpool.unified.min-thread-count: 5 +readpool.unified.max-thread-count: 20 +readpool.storage.normal-concurrency: 10 +pessimistic-txn.pipelined: true +``` + +#### TiDB global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; +``` + +#### HAProxy configuration - haproxy.cfg + +For more details about how to use HAProxy on TiDB, see [Best Practices for Using HAProxy in TiDB](/best-practices/haproxy-best-practices.md). + +```yaml +global # Global configuration. + chroot /var/lib/haproxy # Changes the current directory and sets superuser privileges for the startup process to improve security. + pidfile /var/run/haproxy.pid # Writes the PIDs of HAProxy processes into this file. + maxconn 4000 # The maximum number of concurrent connections for a single HAProxy process. + user haproxy # Same with the UID parameter. + group haproxy # Same with the GID parameter. A dedicated user group is recommended. + nbproc 64 # The number of processes created when going daemon. When starting multiple processes to forward requests, ensure that the value is large enough so that HAProxy does not block processes. + daemon # Makes the process fork into background. It is equivalent to the command line "-D" argument. It can be disabled by the command line "-db" argument. + +defaults # Default configuration. + log global # Inherits the settings of the global configuration. + retries 2 # The maximum number of retries to connect to an upstream server. If the number of connection attempts exceeds the value, the backend server is considered unavailable. + timeout connect 2s # The maximum time to wait for a connection attempt to a backend server to succeed. It should be set to a shorter time if the server is located on the same LAN as HAProxy. + timeout client 30000s # The maximum inactivity time on the client side. + timeout server 30000s # The maximum inactivity time on the server side. + +listen tidb-cluster # Database load balancing. + bind 0.0.0.0:3390 # The Floating IP address and listening port. + mode tcp # HAProxy uses layer 4, the transport layer. + balance roundrobin # The server with the fewest connections receives the connection. "leastconn" is recommended where long sessions are expected, such as LDAP, SQL and TSE, rather than protocols using short sessions, such as HTTP. The algorithm is dynamic, which means that server weights might be adjusted on the fly for slow starts for instance. + server tidb-1 10.9.18.229:4000 check inter 2000 rise 2 fall 3 # Detects port 4000 at a frequency of once every 2000 milliseconds. If it is detected as successful twice, the server is considered available; if it is detected as failed three times, the server is considered unavailable. + server tidb-2 10.9.39.208:4000 check inter 2000 rise 2 fall 3 + server tidb-3 10.9.64.166:4000 check inter 2000 rise 2 fall 3 +``` + +## Test plan + +1. Deploy TiDB v5.3.0 and v5.2.2 using TiUP. +2. Use Sysbench to import 16 tables, each table with 10 million rows of data. +3. Execute the `analyze table` statement on each table. +4. Back up the data used for restore before different concurrency tests, which ensures data consistency for each test. +5. Start the Sysbench client to perform the `point_select`, `read_write`, `update_index`, and `update_non_index` tests. Perform stress tests on TiDB via HAProxy. For each concurrency under each workload, the test takes 20 minutes. +6. After each type of test is completed, stop the cluster, overwrite the cluster with the backup data in step 4, and restart the cluster. + +### Prepare test data + +Run the following command to prepare the test data: + +{{< copyable "shell-regular" >}} + +```bash +sysbench oltp_common \ + --threads=16 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + --mysql-user=root \ + --mysql-password=password \ + prepare --tables=16 --table-size=10000000 +``` + +### Perform the test + +Run the following command to perform the test: + +{{< copyable "shell-regular" >}} + +```bash +sysbench $testname \ + --threads=$threads \ + --time=1200 \ + --report-interval=1 \ + --rand-type=uniform \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=$aws_nlb_host \ + --mysql-port=$aws_nlb_port \ + run --tables=16 --table-size=10000000 +``` + +## Test results + +### Point Select performance + +| Threads | v5.2.2 TPS | v5.3.0 TPS | v5.2.2 95% latency (ms) | v5.3.0 95% latency (ms) | TPS improvement (%) | +|:----------|:----------|:----------|:----------|:----------|:----------| +|300|267673.17|267516.77|1.76|1.67|-0.06| +|600|369820.29|361672.56|2.91|2.97|-2.20| +|900|417143.31|416479.47|4.1|4.18|-0.16| + +Compared with v5.2.2, the Point Select performance of v5.3.0 is reduced slightly by 0.81%. + +![Point Select](/media/sysbench_v522vsv530_point_select.png) + +### Update Non-index performance + +| Threads | v5.2.2 TPS | v5.3.0 TPS | v5.2.2 95% latency (ms) | v5.3.0 95% latency (ms) | TPS improvement (%) | +|:----------|:----------|:----------|:----------|:----------|:----------| +|300|39715.31|40041.03|11.87|12.08|0.82| +|600|50239.42|51110.04|20.74|20.37|1.73| +|900|57073.97|57252.74|28.16|27.66|0.31| + +Compared with v5.2.2, the Update Non-index performance of v5.3.0 is improved slightly by 0.95%. + +![Update Non-index](/media/sysbench_v522vsv530_update_non_index.png) + +### Update Index performance + +| Threads | v5.2.2 TPS | v5.3.0 TPS | v5.2.2 95% latency (ms) | v5.3.0 95% latency (ms) | TPS improvement (%) | +|:----------|:----------|:----------|:----------|:----------|:----------| +|300|17634.03|17821.1|25.74|25.74|1.06| +|600|20998.59|21534.13|46.63|45.79|2.55| +|900|23420.75|23859.64|64.47|62.19|1.87| + +Compared with v5.2.2, the Update Index performance of v5.3.0 is improved slightly by 1.83%. + +![Update Index](/media/sysbench_v522vsv530_update_index.png) + +### Read Write performance + +| Threads | v5.2.2 TPS | v5.3.0 TPS | v5.2.2 95% latency (ms) | v5.3.0 95% latency (ms) | TPS improvement (%) | +|:----------|:----------|:----------|:----------|:----------|:----------| +|300|3872.01|3848.63|106.75|106.75|-0.60| +|600|4514.17|4471.77|200.47|196.89|-0.94| +|900|4877.05|4861.45|287.38|282.25|-0.32| + +Compared with v5.2.2, the Read Write performance of v5.3.0 is reduced slightly by 0.62%. + +![Read Write](/media/sysbench_v522vsv530_read_write.png) \ No newline at end of file diff --git a/benchmark/v5.2-performance-benchmarking-with-tpcc.md b/benchmark/v5.2-performance-benchmarking-with-tpcc.md index 495b709cdcf81..69cc8412d6b53 100644 --- a/benchmark/v5.2-performance-benchmarking-with-tpcc.md +++ b/benchmark/v5.2-performance-benchmarking-with-tpcc.md @@ -6,7 +6,7 @@ title: TiDB TPC-C Performance Test Report -- v5.2.0 vs. v5.1.1 ## Test overview -This test aims to compare the TPC-H performance of TiDB v5.2.0 and TiDB v5.1.1 in the online analytical processing (OLAP) scenario. The results show that compared with v5.1.1, the TPC-C performance of v5.2.0 is reduced by 4.22%. +This test aims to compare the TPC-C performance of TiDB v5.2.0 and TiDB v5.1.1 in the online transactional processing (OLTP) scenario. The results show that compared with v5.1.1, the TPC-C performance of v5.2.0 is reduced by 4.22%. ## Test environment (AWS EC2) diff --git a/benchmark/v5.3-performance-benchmarking-with-tpcc.md b/benchmark/v5.3-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..e9263c59585de --- /dev/null +++ b/benchmark/v5.3-performance-benchmarking-with-tpcc.md @@ -0,0 +1,128 @@ +--- +title: TiDB TPC-C Performance Test Report -- v5.3.0 vs. v5.2.2 +--- + +# TiDB TPC-C Performance Test Report -- v5.3.0 vs. v5.2.2 + +## Test overview + +This test aims at comparing the TPC-C performance of TiDB v5.3.0 and TiDB v5.2.2 in the online transactional processing (OLTP) scenario. The result shows that compared with v5.2.2, the TPC-C performance of v5.3.0 is reduced by 2.99%. + +## Test environment (AWS EC2) + +## Hardware configuration + +| Service type | EC2 type | Instance count | +|:----------|:----------|:----------| +| PD | m5.xlarge | 3 | +| TiKV | i3.4xlarge| 3 | +| TiDB | c5.4xlarge| 3 | +| TPC-C | c5.9xlarge| 1 | + +### Software version + +| Service type | Software version | +|:----------|:-----------| +| PD | v5.2.2 and v5.3.0 | +| TiDB | v5.2.2 and v5.3.0 | +| TiKV | v5.2.2 and v5.3.0 | +| TiUP | 1.5.1 | + +### Parameter configuration + +TiDB v5.3.0 and TiDB v5.2.2 use the same configuration. + +#### TiDB parameter configuration + +{{< copyable "" >}} + +```yaml +log.level: "error" +performance.max-procs: 20 +prepared-plan-cache.enabled: true +tikv-client.max-batch-wait-time: 2000000 +``` + +#### TiKV parameter configuration + +{{< copyable "" >}} + +```yaml +pessimistic-txn.pipelined: true +raftdb.allow-concurrent-memtable-write: true +raftdb.max-background-jobs: 4 +raftstore.apply-max-batch-size: 2048 +raftstore.apply-pool-size: 3 +raftstore.store-max-batch-size: 2048 +raftstore.store-pool-size: 3 +readpool.storage.normal-concurrency: 10 +readpool.unified.max-thread-count: 20 +readpool.unified.min-thread-count: 5 +rocksdb.max-background-jobs: 8 +server.grpc-concurrency: 6 +storage.scheduler-worker-pool-size: 20 +``` + +#### TiDB global variable configuration + +{{< copyable "sql" >}} + +```sql +set global tidb_hashagg_final_concurrency=1; +set global tidb_hashagg_partial_concurrency=1; +set global tidb_enable_async_commit = 1; +set global tidb_enable_1pc = 1; +set global tidb_guarantee_linearizability = 0; +set global tidb_enable_clustered_index = 1; +``` + +#### HAProxy configuration - haproxy.cfg + +For more details about how to use HAProxy on TiDB, see [Best Practices for Using HAProxy in TiDB](/best-practices/haproxy-best-practices.md). + +```yaml +global # Global configuration. + chroot /var/lib/haproxy # Changes the current directory and sets superuser privileges for the startup process to improve security. + pidfile /var/run/haproxy.pid # Writes the PIDs of HAProxy processes into this file. + maxconn 4000 # The maximum number of concurrent connections for a single HAProxy process. + user haproxy # Same with the UID parameter. + group haproxy # Same with the GID parameter. A dedicated user group is recommended. + nbproc 64 # The number of processes created when going daemon. When starting multiple processes to forward requests, ensure that the value is large enough so that HAProxy does not block processes. + daemon # Makes the process fork into background. It is equivalent to the command line "-D" argument. It can be disabled by the command line "-db" argument. + +defaults # Default configuration. + log global # Inherits the settings of the global configuration. + retries 2 # The maximum number of retries to connect to an upstream server. If the number of connection attempts exceeds the value, the backend server is considered unavailable. + timeout connect 2s # The maximum time to wait for a connection attempt to a backend server to succeed. It should be set to a shorter time if the server is located on the same LAN as HAProxy. + timeout client 30000s # The maximum inactivity time on the client side. + timeout server 30000s # The maximum inactivity time on the server side. + +listen tidb-cluster # Database load balancing. + bind 0.0.0.0:3390 # The Floating IP address and listening port. + mode tcp # HAProxy uses layer 4, the transport layer. + balance roundrobin # The server with the fewest connections receives the connection. "leastconn" is recommended where long sessions are expected, such as LDAP, SQL and TSE, rather than protocols using short sessions, such as HTTP. The algorithm is dynamic, which means that server weights might be adjusted on the fly for slow starts for instance. + server tidb-1 10.9.18.229:4000 check inter 2000 rise 2 fall 3 # Detects port 4000 at a frequency of once every 2000 milliseconds. If it is detected as successful twice, the server is considered available; if it is detected as failed three times, the server is considered unavailable. + server tidb-2 10.9.39.208:4000 check inter 2000 rise 2 fall 3 + server tidb-3 10.9.64.166:4000 check inter 2000 rise 2 fall 3 +``` + +## Test plan + +1. Deploy TiDB v5.3.0 and v5.2.2 using TiUP. +2. Create a database named `tpcc`: `create database tpcc;`. +3. Use BenchmarkSQL to import the TPC-C 5000 Warehouse data: `tiup bench tpcc prepare --warehouse 5000 --db tpcc -H 127.0.0.1 -p 4000`. +4. Run the `tiup bench tpcc run -U root --db tpcc --host 127.0.0.1 --port 4000 --time 1800s --warehouses 5000 --threads {{thread}}` command to perform stress tests on TiDB via HAProxy. For each concurrency, the test takes 30 minutes. +5. Extract the tpmC data of New Order from the result. + +## Test result + +Compared with v5.2.2, the TPC-C performance of v5.3.0 is **reduced slightly by 2.99%**. + +| Threads | v5.2.2 tpmC | v5.3.0 tpmC | tpmC improvement (%) | +|:----------|:----------|:----------|:----------| +|50|42228.8|41580|-1.54| +|100|49400|48248.2|-2.33| +|200|54436.6|52809.4|-2.99| +|400|57026.7|54117.1|-5.10| + +![TPC-C](/media/tpcc_v522_vs_v530.png) \ No newline at end of file diff --git a/benchmark/v5.2-performance-benchmarking-with-tpch.md b/benchmark/v5.3-performance-benchmarking-with-tpch.md similarity index 82% rename from benchmark/v5.2-performance-benchmarking-with-tpch.md rename to benchmark/v5.3-performance-benchmarking-with-tpch.md index 3cefbfe0e55a3..d8a204146b7e8 100644 --- a/benchmark/v5.2-performance-benchmarking-with-tpch.md +++ b/benchmark/v5.3-performance-benchmarking-with-tpch.md @@ -1,17 +1,17 @@ --- -title: TiDB TPC-H 100GB Performance Test Report -- v5.2 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 -summary: Compare the TPC-H 100GB performance of TiDB 5.2 MPP mode, Greenplum 6.15.0, and Apache Spark 3.1.1. +title: TiDB TPC-H 100GB Performance Test Report -- v5.3 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 +summary: Compare the TPC-H 100GB performance of TiDB 5.3 MPP mode, Greenplum 6.15.0, and Apache Spark 3.1.1. --- -# TiDB TPC-H 100GB Performance Test Report -- TiDB v5.2 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 +# TiDB TPC-H 100GB Performance Test Report -- TiDB v5.3 MPP mode vs. Greenplum 6.15.0 and Apache Spark 3.1.1 ## Test overview -This test aims at comparing the TPC-H 100GB performance of TiDB v5.2 in the MPP mode with that of Greenplum and Apache Spark, two mainstream analytics engines, in their latest versions. The test result shows that the performance of TiDB v5.2 in the MPP mode is two to three times faster than that of the other two solutions under TPC-H workload. +This test aims at comparing the TPC-H 100GB performance of TiDB v5.3 in the MPP mode with that of Greenplum and Apache Spark, two mainstream analytics engines, in their latest versions. The test result shows that the performance of TiDB v5.3 in the MPP mode is two to three times faster than that of the other two solutions under TPC-H workload. -In v5.0, TiDB introduces the MPP mode for [TiFlash](/tiflash/tiflash-overview.md). In v5.1 and v5.2, the MPP mode has been further improved, which significantly enhances TiDB's Hybrid Transactional and Analytical Processing (HTAP) capabilities. Test objects in this report are as follows: +In v5.0, TiDB introduces the MPP mode for [TiFlash](/tiflash/tiflash-overview.md), which significantly enhances TiDB's Hybrid Transactional and Analytical Processing (HTAP) capabilities. Test objects in this report are as follows: -+ TiDB v5.2 columnar storage in the MPP mode ++ TiDB v5.3 columnar storage in the MPP mode + Greenplum 6.15.0 + Apache Spark 3.1.1 + Parquet @@ -28,15 +28,15 @@ In v5.0, TiDB introduces the MPP mode for [TiFlash](/tiflash/tiflash-overview.md | Service type | Software version | |:----------|:-----------| -| TiDB | 5.2 | +| TiDB | 5.3 | | Greenplum | 6.15.0 | | Apache Spark | 3.1.1 | ### Parameter configuration -#### TiDB v5.2 +#### TiDB v5.3 -For the v5.2 cluster, TiDB uses the default parameter configuration except for the following configuration items. +For the v5.3 cluster, TiDB uses the default parameter configuration except for the following configuration items. In the configuration file `users.toml` of TiFlash, configure `max_memory_usage` as follows: @@ -87,7 +87,7 @@ The test of Apache Spark uses Apache Parquet as the storage format and stores th > > The following test results are the average data of three tests. All numbers are in seconds. -| Query ID | TiDB v5.2 | Greenplum 6.15.0 | Apache Spark 3.1.1 + Parquet | +| Query ID | TiDB v5.3 | Greenplum 6.15.0 | Apache Spark 3.1.1 + Parquet | | :-------- | :----------- | :------------ | :-------------- | | 1 | 8.08 | 64.1307 | 52.64 | | 2 | 2.53 | 4.76612 | 11.83 | @@ -112,11 +112,11 @@ The test of Apache Spark uses Apache Parquet as the storage format and stores th | 21 | 24.44 | 39.08594 | 34.83 | | 22 | 1.23 | 7.67476 | 4.59 | -![TPC-H](/media/tidb-v5.2-tpch-100-vs-gp-spark.png) +![TPC-H](/media/tidb-v5.3-tpch-100-vs-gp-spark.png) In the performance diagram above: -- Blue lines represent TiDB v5.2; +- Blue lines represent TiDB v5.3; - Red lines represent Greenplum 6.15.0; - Yellow lines represent Apache Spark 3.1.1. - The y-axis represents the execution time of the query. The less the time is, the better the performance is. diff --git a/best-practices/haproxy-best-practices.md b/best-practices/haproxy-best-practices.md index 41b8c11fdf87a..0073b8beed819 100644 --- a/best-practices/haproxy-best-practices.md +++ b/best-practices/haproxy-best-practices.md @@ -14,7 +14,7 @@ This document describes best practices for configuration and usage of [HAProxy]( HAProxy is free, open-source software written in C language that provides a high availability load balancer and proxy server for TCP and HTTP-based applications. Because of its fast and efficient use of CPU and memory, HAProxy is now widely used by many well-known websites such as GitHub, Bitbucket, Stack Overflow, Reddit, Tumblr, Twitter, Tuenti, and AWS (Amazon Web Services). -HAProxy is written in the year 2000 by Willy Tarreau, the core contributor to the Linux kernel, who is still responsible for the maintenance of the project and provides free software updates in the open-source community. The latest stable version 2.0.0 was released on August 16, 2019, bringing more [excellent features](https://www.haproxy.com/blog/haproxy-2-0-and-beyond/). +HAProxy is written in the year 2000 by Willy Tarreau, the core contributor to the Linux kernel, who is still responsible for the maintenance of the project and provides free software updates in the open-source community. In this guide, HAProxy [2.5.0](https://www.haproxy.com/blog/announcing-haproxy-2-5/) is used. It is recommended to use the latest stable version. See [the released version of HAProxy](http://www.haproxy.org/) for details. ## Basic features @@ -72,28 +72,51 @@ yum -y install epel-release gcc systemd-devel ## Deploy HAProxy -You can easily use HAProxy to configure and set up a load-balanced database environment. This section shows general deployment operations. You can customize the [configuration file](http://cbonte.github.io/haproxy-dconv/1.9/configuration.html) based on your actual scenario. +You can easily use HAProxy to configure and set up a load-balanced database environment. This section shows general deployment operations. You can customize the [configuration file](http://cbonte.github.io/haproxy-dconv/2.5/configuration.html) based on your actual scenario. ### Install HAProxy -1. Use yum to install HAProxy: +1. Download the package of the HAProxy 2.5.0 source code: {{< copyable "shell-regular" >}} ```bash - yum -y install haproxy + wget https://github.com/haproxy/haproxy/archive/refs/tags/v2.5.0.zip ``` -2. Check whether the installation is successful: +2. Unzip the package: {{< copyable "shell-regular" >}} ```bash - which haproxy + unzip v2.5.0.zip ``` +3. Compile the application from the source code: + + {{< copyable "shell-regular" >}} + + ```bash + cd haproxy-2.5.0 + make clean + make -j 8 TARGET=linux-glibc USE_THREAD=1 + make PREFIX=${/app/haproxy} SBINDIR=${/app/haproxy/bin} install # Replace `${/app/haproxy}` and `${/app/haproxy/bin}` with your custom directories. ``` - /usr/sbin/haproxy + +4. Reconfigure the profile: + + {{< copyable "shell-regular" >}} + + ```bash + echo 'export PATH=/app/haproxy/bin:$PATH' >> /etc/profile + ``` + +5. Check whether the installation is successful: + + {{< copyable "shell-regular" >}} + + ```bash + which haproxy ``` #### HAProxy commands @@ -146,10 +169,10 @@ global # Global configuration. log 127.0.0.1 local2 # Global syslog servers (up to two). chroot /var/lib/haproxy # Changes the current directory and sets superuser privileges for the startup process to improve security. pidfile /var/run/haproxy.pid # Writes the PIDs of HAProxy processes into this file. - maxconn 4000 # The maximum number of concurrent connections for a single HAProxy process. + maxconn 256 # The maximum number of concurrent connections per HAProxy thread. + nbthread 48 # The maximum number of threads. (The upper limit is equal to the number of CPUs) user haproxy # Same with the UID parameter. group haproxy # Same with the GID parameter. A dedicated user group is recommended. - nbproc 40 # The number of processes created when going daemon. When starting multiple processes to forward requests, make sure the value is large enough so that HAProxy does not block processes. daemon # Makes the process fork into background. It is equivalent to the command line "-D" argument. It can be disabled by the command line "-db" argument. stats socket /var/lib/haproxy/stats # The directory where statistics output is saved. @@ -183,9 +206,7 @@ listen tidb-cluster # Database load balancing. ### Start HAProxy -There are two methods to start HAProxy. - -Method 1: Execute `haproxy`. `/etc/haproxy/haproxy.cfg` is read by default (recommended). +To start HAProxy, run `haproxy`. `/etc/haproxy/haproxy.cfg` is read by default (recommended). {{< copyable "shell-regular" >}} @@ -193,19 +214,9 @@ Method 1: Execute `haproxy`. `/etc/haproxy/haproxy.cfg` is read by default (reco haproxy -f /etc/haproxy/haproxy.cfg ``` -Method 2: Use `systemd` to start HAProxy. - -{{< copyable "shell-regular" >}} - -```bash -systemctl start haproxy.service -``` - ### Stop HAProxy -There are two methods to stop HAProxy. - -Method 1: Use `kill -9`. +To stop HAProxy, use the `kill -9` command. 1. Run the following command: @@ -222,11 +233,3 @@ Method 1: Use `kill -9`. ```bash kill -9 ${haproxy.pid} ``` - -Method 2: Use `systemd`. - -{{< copyable "shell-regular" >}} - -```bash -systemctl stop haproxy.service -``` diff --git a/best-practices/pd-scheduling-best-practices.md b/best-practices/pd-scheduling-best-practices.md index fe84fa2d54b7b..76d700c627c9b 100644 --- a/best-practices/pd-scheduling-best-practices.md +++ b/best-practices/pd-scheduling-best-practices.md @@ -260,7 +260,7 @@ Hot regions scheduling issues generally fall into the following categories: Similar to slow scheduling, the speed of region merge is most likely limited by the configurations of `merge-schedule-limit` and `region-schedule-limit`, or the region merge scheduler is competing with other schedulers. Specifically, the solutions are: -- If it is known from metrics that there are a large number of empty regions in the system, you can adjust `max-merge-region-size` and `max-merge-region-keys` to smaller values to speed up the merge. This is because the merge process involves replica migration, so the smaller the region to be merged, the faster the merge is. If the merge operators are already generated rapidly, to further speed up the process, you can set `patrol-region-interval` to `10ms`. This makes region scanning faster at the cost of more CPU consumption. +- If it is known from metrics that there are a large number of empty regions in the system, you can adjust `max-merge-region-size` and `max-merge-region-keys` to smaller values to speed up the merge. This is because the merge process involves replica migration, so the smaller the region to be merged, the faster the merge is. If the merge operators are already generated rapidly, to further speed up the process, you can set `patrol-region-interval` to `10ms` (the default value of this configuration item is `10ms` in v5.3.0 and later versions of TiDB). This makes region scanning faster at the cost of more CPU consumption. - A lot of tables have been created and then emptied (including truncated tables). These empty Regions cannot be merged if the split table attribute is enabled. You can disable this attribute by adjusting the following parameters: diff --git a/best-practices/tidb-best-practices.md b/best-practices/tidb-best-practices.md index dc5d4d322fd7a..2fd3ddd72a605 100644 --- a/best-practices/tidb-best-practices.md +++ b/best-practices/tidb-best-practices.md @@ -160,14 +160,14 @@ As mentioned before, TiDB limits the size of a single transaction in the Key-Val It is recommended to split statements into batches or add a limit to the statements, whether they are `INSERT`, `UPDATE` or `DELETE` statements. -When deleting a large amount of data, it is recommended to use `Delete * from t where xx limit 5000;`. It deletes through the loop and use `Affected Rows == 0` as a condition to end the loop. +When deleting a large amount of data, it is recommended to use `Delete from t where xx limit 5000;`. It deletes through the loop and use `Affected Rows == 0` as a condition to end the loop. If the amount of data that needs to be deleted at a time is large, this loop method gets slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all is cleared by Garbage Collection) and affect the following `DELETE` statement. If possible, it is recommended to refine the `WHERE` condition. Assume that you need to delete all data on `2017-05-26`, you can use the following statements: ```sql for i from 0 to 23: while affected_rows > 0: - delete * from t where insert_time >= i:00:00 and insert_time < (i+1):00:00 limit 5000; + delete from t where insert_time >= i:00:00 and insert_time < (i+1):00:00 limit 5000; affected_rows = select affected_rows() ``` diff --git a/br/backup-and-restore-tool.md b/br/backup-and-restore-tool.md index cd71e8a527395..66d646663664b 100644 --- a/br/backup-and-restore-tool.md +++ b/br/backup-and-restore-tool.md @@ -8,7 +8,9 @@ aliases: ['/docs/dev/br/backup-and-restore-tool/','/docs/dev/reference/tools/br/ [BR](http://github.com/pingcap/br) (Backup & Restore) is a command-line tool for distributed backup and restoration of the TiDB cluster data. -Compared with [`dumpling`](/backup-and-restore-using-dumpling-lightning.md), BR is more suitable for scenarios of huge data volume. +Compared with [`dumpling`](/backup-and-restore-using-dumpling-lightning.md), BR is more suitable for scenarios involved huge data volumes. + +In addition to regular backup and restoration, you can also use BR for large-scale data migration as long as compatibility is ensured. This document describes BR's implementation principles, recommended deployment configuration, usage restrictions and several methods to use BR. @@ -136,6 +138,7 @@ These features are as follows: | Clustered index | [#565](https://github.com/pingcap/br/issues/565) | Make sure that the value of the `tidb_enable_clustered_index` global variable during restore is consistent with that during backup. Otherwise, data inconsistency might occur, such as `default not found` and inconsistent data index. | | New collation | [#352](https://github.com/pingcap/br/issues/352) | Make sure that the value of the `new_collations_enabled_on_first_bootstrap` variable is consistent with that during backup. Otherwise, inconsistent data index might occur and checksum might fail to pass. | | TiCDC enabled on the restore cluster | [#364](https://github.com/pingcap/br/issues/364#issuecomment-646813965) | Currently, TiKV cannot push down the BR-ingested SST files to TiCDC. Therefore, you need to disable TiCDC when using BR to restore data. | +| Global temporary tables | | Make sure that you are using BR v5.3.0 or a later version to back up and restore data. Otherwise, an error occurs in the definition of the backed global temporary tables. | However, even after you have ensured that the above features are consistently enabled or disabled during backup and restore, compatibility issues might still occur due to the inconsistent internal versions or inconsistent interfaces between BR and TiKV/TiDB/PD. To avoid such cases, BR has the built-in version check. @@ -187,6 +190,7 @@ The following are some recommended operations for using BR for backup and restor - It is recommended that you execute multiple restore operations serially. Running different restore operations in parallel increases Region conflicts and also reduces restore performance. - It is recommended that you mount a shared storage (for example, NFS) on the backup path specified by `-s`, to make it easier to collect and manage backup files. - It is recommended that you use a storage hardware with high throughput, because the throughput of a storage hardware limits the backup and restoration speed. +- It is recommended that you disable the checksum feature (`--checksum = false`) during backup operation and only enable it during the restore operation to reduce migration time. This is because BR by default respectively performs checksum calculation after backup and restore operations to compare the stored data with the corresponding cluster data to ensure accuracy. ### How to use BR diff --git a/br/use-br-command-line-tool.md b/br/use-br-command-line-tool.md index 835cf1842be23..f7bf357daa95b 100644 --- a/br/use-br-command-line-tool.md +++ b/br/use-br-command-line-tool.md @@ -246,6 +246,32 @@ LAST_BACKUP_TS=`br validate decode --field="end-version" -s local:///home/tidb/b In the above example, for the incremental backup data, BR records the data changes and the DDL operations during `(LAST_BACKUP_TS, current PD timestamp]`. When restoring data, BR first restores DDL operations and then the data. +### Encrypt data during backup (experimental feature) + +Since TiDB v5.3.0, TiDB supports backup encryption. You can configure the following parameters to encrypt data during backup: + +* `--crypter.method`: Encryption algorithm. Supports three algorithms `aes128-ctr/aes192-ctr/aes256-ctr`. The default value is `plaintext` and indicates no encryption. +* `--crypter.key`: Encryption key in hexadecimal string format. `aes128-ctr` means 128 bit (16 bytes) key length, `aes192-ctr` means 24 bytes and `aes256-ctr` means 32 bytes. +* `--crypter.key-file`: The key file. You can directly pass in the file path where the key is stored as a parameter without passing in "crypter.key" + +> **Warning:** +> +> - This is still an experimental feature. It is **NOT** recommended that you use it in the production environment. +> - If the key is lost, the backup data cannot be restored to the cluster. +> - The encryption feature needs to be used on BR tools and TiDB clusters v5.3.0 or later versions, and the encrypted backup data cannot be restored on clusters ealier than v5.3.0. + +The configuration example for backup encryption is as follows: + +{{< copyable "shell-regular" >}} + +```shell +br backup full\ + --pd ${PDIP}:2379 \ + -s local:///home/tidb/backupdata/incr \ + --crypter.method aes128-ctr \ + --crypter.key 0123456789abcdef0123456789abcdef +``` + ### Point-in-time recovery (experimental feature) Point-in-time recovery (PITR) allows you to restore data to a point in time of your choice. @@ -310,7 +336,7 @@ To restore the cluster data, use the `br restore` command. You can add the `full > - Where each peer is scattered to during restore is random. We don't know in advance which node will read which file. > > These can be avoided using shared storage, for example mounting an NFS on the local path, or using S3. With network storage, every node can automatically read every SST file, so these caveats no longer apply. -> +> > Also, note that you can only run one restore operation for a single cluster at the same time. Otherwise, unexpected behaviors might occur. For details, see [FAQ](/br/backup-and-restore-faq.md#can-i-use-multiple-br-processes-at-the-same-time-to-restore-the-data-of-a-single-cluster). ### Restore all the backup data @@ -481,9 +507,29 @@ br restore full -f 'mysql.usertable' -s $external_storage_url --ratelimit 128 > - the system variable tables (`mysql.tidb`,`mysql.global_variables`) cannot be restored. > - the user information tables (such as `mysql.user` and `mysql.columns_priv`) cannot be restored. > - GC data cannot be restored. -> +> > Restoring system tables might cause more compatibility issues. To avoid unexpected issues, **DO NOT** restore system tables in the production environment. +### Decrypt data during restore (experimental feature) + +> **Warning:** +> +> This is still an experimental feature. It is **NOT** recommended that you use it in the production environment. + +After encrypting the backup data, you need to pass in the corresponding decryption parameters to restore the data. You need to ensure that the decryption parameters and encryption parameters are consistent. If the decryption algorithm or key is incorrect, the data cannot be restored. + +The following is an example of decrypting the backup data: + +{{< copyable "shell-regular" >}} + +```shell +br restore full\ + --pd ${PDIP}:2379 \ + -s local:///home/tidb/backupdata/incr \ + --crypter.method aes128-ctr \ + --crypter.key 0123456789abcdef0123456789abcdef +``` + ### Restore Raw KV (experimental feature) > **Warning:** diff --git a/configure-memory-usage.md b/configure-memory-usage.md index c754c221b270b..cab9f79a53a10 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -100,7 +100,7 @@ The following example constructs a memory-intensive SQL statement that triggers 2. Execute `CREATE TABLE t(a int);` and insert 1000 rows of data. -3. Execute `select * from t t1 join t t1 join t t3 order by t1.a`. This SQL statement outputs one billion records, which consumes a large amount of memory and therefore triggers the alarm. +3. Execute `select * from t t1 join t t2 join t t3 order by t1.a`. This SQL statement outputs one billion records, which consumes a large amount of memory and therefore triggers the alarm. 4. Check the `tidb.log` file which records the total system memory, current system memory usage, memory usage of the tidb-server instance, and the directory of status files. diff --git a/configure-placement-rules.md b/configure-placement-rules.md index b28a4526cf270..79b9134e6ba51 100644 --- a/configure-placement-rules.md +++ b/configure-placement-rules.md @@ -10,6 +10,10 @@ aliases: ['/docs/dev/configure-placement-rules/','/docs/dev/how-to/configure/pla > > In the scenario of using TiFlash, the Placement Rules feature has been extensively tested and can be used in the production environment. Except for the scenario where TiFlash is used, using Placement Rules alone has not been extensively tested, so it is not recommended to enable this feature separately in the production environment. +> **Note:** +> +> TiDB v5.3.0 introduces an experimental support for [Placement Rules in SQL](/placement-rules-in-sql.md). This offers a more convenient way to configure the placement of tables and partitions. Placement Rules in SQL might replace placement configuration with PD in future releases. + Placement Rules is an experimental feature of the Placement Driver (PD) introduced in v4.0. It is a replica rule system that guides PD to generate corresponding schedules for different types of data. By combining different scheduling rules, you can finely control the attributes of any continuous data range, such as the number of replicas, the storage location, the host type, whether to participate in Raft election, and whether to act as the Raft leader. ## Rule system diff --git a/connectors-and-apis.md b/connectors-and-apis.md index 444af731e8eb0..337d0b60521ac 100644 --- a/connectors-and-apis.md +++ b/connectors-and-apis.md @@ -74,7 +74,7 @@ The third-party APIs are not developed by Oracle. The following table lists the | Ruby | MySQL/Ruby | `libmysqlclient` | Uses `libmysqlclient`. See [Section 27.12.1, “The MySQL/Ruby API”](https://dev.mysql.com/doc/refman/5.7/en/apis-ruby-mysqlruby.html) | | | Ruby/MySQL | Native Driver | See [Section 27.12.2, “The Ruby/MySQL API”](https://dev.mysql.com/doc/refman/5.7/en/apis-ruby-rubymysql.html) | | Scheme | `Myscsh` | `libmysqlclient` | See [`Myscsh`](https://github.com/aehrisch/myscsh) | -| SPL | `sql_mysql` | `libmysqlclient` | See [`sql_mysql` for SPL](http://www.clifford.at/spl/spldoc/sql_mysql.html) | +| SPL | `sql_mysql` | `libmysqlclient` | See `sql_mysql` for SPL | | Tcl | MySQLtcl | `libmysqlclient` | See [Section 27.13, “MySQL Tcl API”](https://dev.mysql.com/doc/refman/5.7/en/apis-tcl.html) | ## Connector versions supported by TiDB diff --git a/dashboard/continuous-profiling.md b/dashboard/continuous-profiling.md new file mode 100644 index 0000000000000..7e7d6958bc4f6 --- /dev/null +++ b/dashboard/continuous-profiling.md @@ -0,0 +1,159 @@ +--- +title: TiDB Dashboard Continuous Profiling +summary: Learn how to enable Continuous Profiling and observe system conditions by using this feature. +--- + +# TiDB Dashboard Instance Profiling - Continuous Profiling + +> **Warning:** +> +> Continuous Profiling is currently an experimental feature and is not recommended for use in production environments. +> + +Introduced in TiDB 5.3.0, Continuous Profiling is a way to observe resource overhead at the system call level. With the support of Continuous Profiling, TiDB provides performance insight as clear as directly looking into the database source code, and helps R&D and operation and maintenance personnel to locate the root cause of performance problems using a flame graph. + +With less than 0.5% performance loss, this feature takes continuous snapshots (similar to CT scan) of the database internal operations, turning the database from a "black box" into a "white box" that is more observable. This feature runs automatically after being enabled by one click and keeps storage results generated within the retention period. Storage results beyond the retention period are recycled to release the storage space. + +## Restrictions + +Before enabling the Continuous Profiling feature, pay attention to the following restrictions: + +- Under the x86 architecture, this feature supports TiDB, TiKV, and PD, but does not support TiFlash. This feature is not fully compatible with the ARM architecture and cannot be enabled under this architecture. + +- Currently, this feature is only available for clusters deployed or upgraded using TiUP, and is unavailable for clusters deployed or upgraded by using TiDB Operator or binary packages. + +## Profiling content + +With Continuous Profiling, you can collect continuous performance data of TiDB, TiKV, and PD instances, and have the nodes monitored day and night without restarting any of them. The data collected can be displayed on a flame graph or a directed acyclic graph. The graph visually shows what internal operations are performed on the instances during the performance summary period and the corresponding proportions. With this graph, you can quickly learn the CPU resource consumption of these instances. + +Currently, Continuous Profiling can display the following performance data: + +- TiDB/PD: CPU profile, Heap, Mutex, Goroutine (debug=2) +- TiKV: CPU profile + +## Enable Continuous Profiling + +To enable Continuous Profiling, you need to first check the version information and then configure related settings on the control machine and TiDB Dashboard. + +### Check versions + +Before enabling this feature, check the version of the TiUP cluster and ensure that it is 1.7.0 or above. If it is earlier than 1.7.0, upgrade it first. + +1. Check the TiUP version: + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster --version + ``` + + The command output shows the TiUP version, as shown below: + + ```shell + tiup version 1.7.0 tiup + Go Version: go1.17.2 + Git Ref: v1.7.0 + ``` + + - If `tiup version` is earlier than 1.7.0, upgrade the TiUP cluster by referring to the next step. + - If `tiup version` is 1.7.0 or above, you can directly reload Prometheus. + +2. Upgrade TiUP and TiUP cluster to the latest version. + + - Upgrade TiUP: + + {{< copyable "shell-regular" >}} + + ```shell + tiup update --self + ``` + + - Upgrade the TiUP cluster: + + {{< copyable "shell-regular" >}} + + ```shell + tiup update cluster + ``` + +After TiUP and the TiUP cluster are upgraded to the latest version, version check is completed. + +### Configure the control machine and TiDB Dashboard + +1. On the control machine, add the `ng_port` configuration item by using TiUP. + + 1. Open the configuration file of the cluster in the editing mode: + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster edit-config ${cluster-name} + ``` + + 2. Set parameters: add `ng_port:${port}` under [monitoring_servers](/tiup/tiup-cluster-topology-reference.md#monitoring_servers). + + ```shell + monitoring_servers: + - host: 172.16.6.6 + ng_port: ${port} + ``` + + 3. Reload Prometheus: + + {{< copyable "shell-regular" >}} + + ```shell + tiup cluster reload ${cluster-name} --role prometheus + ``` + + After reloading Prometheus, you have finished operations on the control machine. + +2. Enable the Continuous Profiling feature. + + 1. On TiDB Dashboard, click **Advanced Debugging** > **Profile Instances** > **Continuous Profile**. + + 2. In the displayed window, switch on the button under **Enable Feature** on the right. Modify the value of **Retention Duration** as required or retain the default value. + + 3. Click **Save** to enable this feature. + + ![Enable the feature](/media/dashboard/dashboard-conprof-start.png) + +## Access the page + +You can access the instance profiling page using either of the following methods: + +- After logging into TiDB Dashboard, click **Advanced Debugging** > **Profile Instances** > **Continuous Profile** on the left navigation bar. + + ![Access](/media/dashboard/dashboard-conprof-access.png) + +- Visit via your browser. Replace `127.0.0.1:2379` with the actual PD instance address and port. + +## View profiling history + +After starting continuous profiling, you can view the profiling result on the instance profiling page. + +![Profiling history](/media/dashboard/dashboard-conprof-history.png) + +Performance profiling runs in the background. Refreshing or exiting the current page will not terminate a running performance profiling task. + +## Download profiling result + +On the profiling result page, you can click **Download Profiling Result** in the upper-right corner to download all profiling results. + +![Download profiling result](/media/dashboard/dashboard-conprof-download.png) + +You can also click an individual instance to view its profiling result: + +![View the profiling result of an instance](/media/dashboard/dashboard-conprof-single.png) + +## Disable Continuous Profiling + +1. On TiDB Dashboard, click **Advanced Debugging** > **Profile Instances** > **Continuous Profile** on the left navigation bar. Click **Settings**. + +2. In the popped-up window, switch off the button under **Enable Feature**. + +3. In the dialog box of **Disable Continuous Profiling Feature**, click **Disable**. + +4. Click **Save**. + + ![Disable the feature](/media/dashboard/dashboard-conprof-stop.png) \ No newline at end of file diff --git a/dashboard/dashboard-access.md b/dashboard/dashboard-access.md index 0b9061da2f93a..10053101effda 100644 --- a/dashboard/dashboard-access.md +++ b/dashboard/dashboard-access.md @@ -28,7 +28,10 @@ You can use TiDB Dashboard in the following common desktop browsers of a relativ ## Sign in -For the first-time access, TiDB Dashboard displays the user sign in interface, as shown in the image below. You can sign in using the TiDB `root` account. By default, the `root` password is empty. +After accessing TiDB Dashboard, you will be directed to the user login interface, as shown in the image below. + +- You can sign in with TiDB Dashboard using the TiDB `root` account. By default, the `root` password is empty. +- If you have created a [User-defined SQL User](/dashboard/dashboard-user.md), you can sign in using this account and the corresponding password. ![Login interface](/media/dashboard/dashboard-access-login.png) diff --git a/dashboard/dashboard-ops-security.md b/dashboard/dashboard-ops-security.md index 7f334f784f829..c19bab9b94015 100644 --- a/dashboard/dashboard-ops-security.md +++ b/dashboard/dashboard-ops-security.md @@ -8,11 +8,19 @@ aliases: ['/docs/dev/dashboard/dashboard-ops-security/'] Although you need to sign into TiDB Dashboard before accessing it, TiDB Dashboard is designed to be accessed by trusted user entities by default. When you want to provide TiDB Dashboard to external network users or untrusted users for access, take the following measures to avoid security vulnerabilities. -## Set a strong password for TiDB `root` user +## Enhance security of TiDB users + +### Set a strong password for the TiDB `root` user The account system of TiDB Dashboard is consistent with that of TiDB SQL users. By default, TiDB's `root` user has no password, so accessing TiDB Dashboard does not require password authentication, which will give the malicious visitor high privileges, including executing privileged SQL statements. -It is recommended that you set a strong password for TiDB `root` user. See [TiDB User Account Management](/user-account-management.md) for details. +It is recommended that you set a strong password for the TiDB `root` user. See [TiDB User Account Management](/user-account-management.md) for details. Alternatively, you can disable the TiDB `root` user. + +### Create a least-privileged user for TiDB Dashboard + +The account system of TiDB Dashboard is consistent with that of TiDB SQL. Users accessing TiDB Dashboard are authenticated and authorized based on TiDB SQL user's privileges. Therefore, TiDB Dashboard requires limited privileges, or merely the read-only privilege. You can configure users to access TiDB Dashboard based on the principle of least privilege, thus avoiding access of high-privileged users. + +It is recommended that you create a least-privileged SQL user to access and sign in with TiDB Dashboard. This avoids access of high-privileged users and improves security. See [TiDB Dashboard User Management](/dashboard/dashboard-user.md) for details. ## Use a firewall to block untrusted access diff --git a/dashboard/dashboard-profiling.md b/dashboard/dashboard-profiling.md index d477b06e0e0c7..04f0c5658e7fa 100644 --- a/dashboard/dashboard-profiling.md +++ b/dashboard/dashboard-profiling.md @@ -1,20 +1,18 @@ --- -title: Instance Profiling Page -summary: Learn the instance profiling page of TiDB Dashboard. +title: Manual Profiling +summary: Learn the manual instance profiling of TiDB Dashboard. aliases: ['/docs/dev/dashboard/dashboard-profiling/'] --- -# Instance Profiling Page +# Manual Instance Profiling Page -The instance profiling page summarizes the internal performance data of each TiDB instance, TiKV instance, and PD instance. With this page, you can view the instance profiling without restarting these instances. - -The summarized performance data can be displayed as a flame graph or a directed acyclic graph, which visually shows the internal operations and proportions performed on the instances during the performance summary period. With this graph, you can quickly learn the main CPU resource consumption of these instances. +On the manual instance profiling page, you can collect continuous performance data of TiDB, TiKV, PD, and TiFlash instances without restarting any of them. The data collected can be displayed on a flame graph or a directed acyclic graph. The graph visually shows what internal operations are performed on the instances during the performance summary period and the corresponding proportions. With this graph, you can quickly learn the CPU resource consumption of these instances. ## Access the page -You can access the instance profiling page using one of the following methods: +You can access the instance profiling page using either of the following methods: -- After logging into TiDB Dashboard, click **Advanced Debugging** → **Profile Instances** on the left navigation bar. +- After logging into TiDB Dashboard, click **Advanced Debugging** > **Profile Instances** > **Manually Profile** on the left navigation bar. ![Access instance profiling page](/media/dashboard/dashboard-profiling-access.png) diff --git a/dashboard/dashboard-session-sso.md b/dashboard/dashboard-session-sso.md index 597e3e8d5ade3..e06400d0675c2 100644 --- a/dashboard/dashboard-session-sso.md +++ b/dashboard/dashboard-session-sso.md @@ -88,7 +88,7 @@ Once SSO is configured for TiDB Dashboard, you can sign in via SSO by taking fol 3. You are redirected back to TiDB Dashboard to finish the sign-in. -## Example: Use Okta for TiDB Dashboard SSO sign-in +## Example 1: Use Okta for TiDB Dashboard SSO sign-in [Okta](https://www.okta.com/) is an OIDC SSO identity service, which is compatible with the SSO feature of TiDB Dashboard. The steps below demonstrate how to configure Okta and TiDB Dashboard so that Okta can be used as the TiDB Dashboard SSO provider. @@ -100,7 +100,7 @@ First, create an Okta Application Integration to integrate SSO. 2. Navigate from the left sidebar **Applications** > **Applications**. -3. Click **Create App Integration**。 +3. Click **Create App Integration**. ![Sample Step](/media/dashboard/dashboard-session-sso-okta-1.png) @@ -149,3 +149,53 @@ First, create an Okta Application Integration to integrate SSO. ![Sample Step 3](/media/dashboard/dashboard-session-sso-okta-info-3.png) Now TiDB Dashboard has been configured to use Okta SSO for sign-in. + +## Example 2: Use Auth0 for TiDB Dashboard SSO sign-in + +Similar to Okta, [Auth0](https://auth0.com/) also provides OIDC SSO identity service. The following steps describe how to configure Auth0 and TiDB Dashboard so that Auth0 can be used as the TiDB Dashboard SSO provider. + +### Step 1: Configure Auth0 + +1. Access the Auth0 administration site. + +2. Navigate on the left sidebar **Applications** > **Applications**. + +3. Click **Create App Integration**. + + ![Create Application](/media/dashboard/dashboard-session-sso-auth0-create-app.png) + + In the popped-up dialog, fill **Name**, for example, "TiDB Dashboard". Choose **Single Page Web Applications** in **Choose an application type**. Click **Create**. + +4. Click **Settings**. + + ![Settings](/media/dashboard/dashboard-session-sso-auth0-settings-1.png) + +5. Fill **Allowed Callback URLs** as follows: + + ``` + http://DASHBOARD_IP:PORT/dashboard/?sso_callback=1 + ``` + + Replace `DASHBOARD_IP:PORT` with the actual domain (or IP address) and port that you use to access the TiDB Dashboard in your browser. + +6. Fill **Allowed Logout URLs** as follows: + + ``` + http://DASHBOARD_IP:PORT/dashboard/ + ``` + + Similarly, replace `DASHBOARD_IP:PORT` with the actual domain (or IP address) and port. + + ![Settings](/media/dashboard/dashboard-session-sso-auth0-settings-2.png) + +7. Keep the default values for other settings and click **Save Changes**. + +### Step 2: Obtain OIDC information and fill in TiDB Dashboard + +1. Fill **OIDC Client ID** of TiDB Dashboard with **Client ID** in **Basic Information** under the **Settings** tab of Auth0. + +2. Fill **OIDC Discovery URL** with the **Domain** field value prefixed with `https://` and suffixed with `/`, for example, `https://example.us.auth0.com/`. Complete authorization and save the configuration. + + ![Settings](/media/dashboard/dashboard-session-sso-auth0-settings-3.png) + +Now TiDB Dashboard has been configured to use Auth0 SSO for sign-in. diff --git a/dashboard/dashboard-user.md b/dashboard/dashboard-user.md new file mode 100644 index 0000000000000..d7c3224a9511c --- /dev/null +++ b/dashboard/dashboard-user.md @@ -0,0 +1,81 @@ +--- +title: TiDB Dashboard User Management +summary: Learn how to create SQL users to access TiDB Dashboard. +aliases: ['/docs/dev/dashboard/dashboard-user/'] +--- + +# TiDB Dashboard User Management + +TiDB Dashboard uses the same user privilege system and sign-in authentication as TiDB. You can control and manage TiDB SQL users to limit their access to TiDB Dashboard. This document describes the least privileges required for TiDB SQL users to access TiDB Dashboard and exemplifies how to create a least-privileged SQL user. + +For details about how to control and manage TiDB SQL users, see [TiDB User Account Management](/user-account-management.md). + +## Required privileges + +- To access TiDB Dashboard when [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security) is not enabled on the connected TiDB server, the SQL user should have **all** the following privileges: + + - PROCESS + - SHOW DATABASES + - CONFIG + - DASHBOARD_CLIENT + +- To access TiDB Dashboard when [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security) is enabled on the connected TiDB server, the SQL user should have **all** the following privileges: + + - PROCESS + - SHOW DATABASES + - CONFIG + - DASHBOARD_CLIENT + - RESTRICTED_TABLES_ADMIN + - RESTRICTED_STATUS_ADMIN + - RESTRICTED_VARIABLES_ADMIN + +- To modify the configurations on the interface after signing in with TiDB Dashboard, the SQL user must also have the following privilege: + + - SYSTEM_VARIABLES_ADMIN + +> **Note:** +> +> Users with high privileges such as `ALL PRIVILEGES` or `SUPER` can sign in with TiDB Dashboard as well. Therefore, to comply with the least privilege principle, it is highly recommended that you create users with the required privileges only to prevent unintended operations. See [Privilege Management](/privilege-management.md) for more information on these privileges. + +If an SQL user does not meet the preceding privilege requirements, the user fails to sign in with TiDB Dashboard, as shown below. + +![insufficient-privileges](/media/dashboard/dashboard-user-insufficient-privileges.png) + +## Example: Create a least-privileged SQL user to access TiDB Dashboard + +- When [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security) is not enabled on the connected TiDB server, to create an SQL user `dashboardAdmin` that can sign in with TiDB Dashboard, execute the following SQL statements: + + ```sql + CREATE USER 'dashboardAdmin'@'%' IDENTIFIED BY ''; + GRANT PROCESS, CONFIG ON *.* TO 'dashboardAdmin'@'%'; + GRANT SHOW DATABASES ON *.* TO 'dashboardAdmin'@'%'; + GRANT DASHBOARD_CLIENT ON *.* TO 'dashboardAdmin'@'%'; + ``` + + -- To modify the configuration items on the interface after signing in with TiDB Dashboard, the user-defined SQL user must be granted with the following privilege: + + ```sql + GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; + ``` + +- When [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security) is enabled on the connected TiDB server, disable SEM first and execute the following SQL statements to create an SQL user `dashboardAdmin` that can sign in with TiDB Dashboard. After creating the user, enable SEM again: + + ```sql + CREATE USER 'dashboardAdmin'@'%' IDENTIFIED BY ''; + GRANT PROCESS, CONFIG ON *.* TO 'dashboardAdmin'@'%'; + GRANT SHOW DATABASES ON *.* TO 'dashboardAdmin'@'%'; + GRANT DASHBOARD_CLIENT ON *.* TO 'dashboardAdmin'@'%'; + GRANT RESTRICTED_STATUS_ADMIN ON *.* TO 'dashboardAdmin'@'%'; + GRANT RESTRICTED_TABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; + GRANT RESTRICTED_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; + ``` + + -- To modify the configuration items on the interface after signing in with TiDB Dashboard, the user-defined SQL user must be granted with the following privilege: + + ```sql + GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'dashboardAdmin'@'%'; + ``` + +## Sign in with TiDB Dashboard + +After creating an SQL user that meets the privilege requirements of TiDB Dashboard, you can use this user to [Sign in](/dashboard/dashboard-access.md#Sign in) with TiDB Dashboard. \ No newline at end of file diff --git a/deploy-monitoring-services.md b/deploy-monitoring-services.md index c7a079e40fd47..7436d27b6e8fb 100644 --- a/deploy-monitoring-services.md +++ b/deploy-monitoring-services.md @@ -29,7 +29,7 @@ Assume that the TiDB cluster topology is as follows: ```bash # Downloads the package. -wget https://download.pingcap.org/prometheus-2.8.1.linux-amd64.tar.gz +wget https://download.pingcap.org/prometheus-2.27.1.linux-amd64.tar.gz wget https://download.pingcap.org/node_exporter-0.17.0.linux-amd64.tar.gz wget https://download.pingcap.org/grafana-6.1.6.linux-amd64.tar.gz ``` @@ -38,7 +38,7 @@ wget https://download.pingcap.org/grafana-6.1.6.linux-amd64.tar.gz ```bash # Extracts the package. -tar -xzf prometheus-2.8.1.linux-amd64.tar.gz +tar -xzf prometheus-2.27.1.linux-amd64.tar.gz tar -xzf node_exporter-0.17.0.linux-amd64.tar.gz tar -xzf grafana-6.1.6.linux-amd64.tar.gz ``` @@ -62,7 +62,7 @@ Edit the Prometheus configuration file: {{< copyable "shell-regular" >}} ```bash -cd prometheus-2.8.1.linux-amd64 && +cd prometheus-2.27.1.linux-amd64 && vi prometheus.yml ``` diff --git a/download-ecosystem-tools.md b/download-ecosystem-tools.md index f347a4b7ed20a..9b24a916a310b 100644 --- a/download-ecosystem-tools.md +++ b/download-ecosystem-tools.md @@ -18,7 +18,7 @@ If you want to download the latest version of [TiDB Binlog](/tidb-binlog/tidb-bi > **Note:** > -> `{version}` in the above download link indicates the version number of TiDB. For example, the download link for `v5.2.1` is `https://download.pingcap.org/tidb-v5.2.1-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of TiDB. For example, the download link for `v5.3.0` is `https://download.pingcap.org/tidb-v5.3.0-linux-amd64.tar.gz`. ## TiDB Lightning @@ -30,7 +30,7 @@ Download [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) by using t > **Note:** > -> `{version}` in the above download link indicates the version number of TiDB Lightning. For example, the download link for `v5.2.1` is `https://download.pingcap.org/tidb-toolkit-v5.2.1-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of TiDB Lightning. For example, the download link for `v5.3.0` is `https://download.pingcap.org/tidb-toolkit-v5.3.0-linux-amd64.tar.gz`. ## BR (backup and restore) @@ -42,7 +42,7 @@ Download [BR](/br/backup-and-restore-tool.md) by using the download link in the > **Note:** > -> `{version}` in the above download link indicates the version number of BR. For example, the download link for `v5.2.1` is `https://download.pingcap.org/tidb-toolkit-v5.2.1-linux-amd64.tar.gz`. +> `{version}` in the above download link indicates the version number of BR. For example, the download link for `v5.3.0` is `https://download.pingcap.org/tidb-toolkit-v5.3.0-linux-amd64.tar.gz`. ## TiDB DM (Data Migration) @@ -54,7 +54,7 @@ Download [DM](https://docs.pingcap.com/tidb-data-migration/stable/overview) by u > **Note:** > -> `{version}` in the above download link indicates the version number of DM. For example, the download link for `v2.0.6` is `https://download.pingcap.org/dm-v2.0.6-linux-amd64.tar.gz`. You can check the published DM versions in the [DM Release](https://github.com/pingcap/dm/releases) page. +> `{version}` in the above download link indicates the version number of DM. For example, the download link for `v5.3.0` is `https://download.pingcap.org/dm-v5.3.0-linux-amd64.tar.gz`. You can check the published DM versions in the [DM Release](https://github.com/pingcap/dm/releases) page. ## Dumpling @@ -66,7 +66,7 @@ Download [Dumpling](/dumpling-overview.md) from the links below: > **Note:** > -> The `{version}` in the download link is the version number of Dumpling. For example, the link for downloading the `v5.2.1` version of Dumpling is `https://download.pingcap.org/tidb-toolkit-v5.2.1-linux-amd64.tar.gz`. You can view the currently released versions in [Dumpling Releases](https://github.com/pingcap/dumpling/releases). +> The `{version}` in the download link is the version number of Dumpling. For example, the link for downloading the `v5.3.0` version of Dumpling is `https://download.pingcap.org/tidb-toolkit-v5.3.0-linux-amd64.tar.gz`. You can view the currently released versions in [TiDB Releases](https://github.com/pingcap/tidb/releases). > > Dumpling supports arm64 linux. You can replace `amd64` in the download link with `arm64`, which means the `arm64` version of Dumpling. diff --git a/dumpling-overview.md b/dumpling-overview.md index 0238881d6d644..165151c96b760 100644 --- a/dumpling-overview.md +++ b/dumpling-overview.md @@ -61,7 +61,7 @@ dumpling \ -P 4000 \ -h 127.0.0.1 \ --filetype sql \ - --t 8 \ + -t 8 \ -o /tmp/test \ -r 200000 \ -F 256MiB @@ -71,7 +71,7 @@ In the command above: + The `-h`, `-p`, and `-u` option respectively mean the address, the port, and the user. If a password is required for authentication, you can use `-p $YOUR_SECRET_PASSWORD` to pass the password to Dumpling. + The `-o` option specifies the export directory of the storage, which supports a local file path or a [URL of an external storage](/br/backup-and-restore-storages.md). -+ The `t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. ++ The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. Usually, it's less than 64. + The `-r` option specifies the maximum number of rows in a single file. With this option specified, Dumpling enables the in-table concurrency to speed up the export and reduce the memory usage. + The `-F` option is used to specify the maximum size of a single file (the unit here is `MiB`; inputs like `5GiB` or `8KB` are also acceptable). It is recommended to keep its value to 256 MiB or less if you plan to use TiDB Lightning to load this file into a TiDB instance. @@ -252,8 +252,9 @@ Examples: The exported file is stored in the `./export-` directory by default. Commonly used options are as follows: -- The `t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. +- The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. - The `-r` option specifies the maximum number of records (or the number of rows in the database) for a single file. When it is enabled, Dumpling enables concurrency in the table to improve the speed of exporting large tables. +- The `--compress gzip` option can be used to compress the dump. This can help to speed up dumping of data if storage is the bottleneck or if storage capacity is a concern. The drawback of this is an increase in CPU usage. Each file is compressed individually. With the above options specified, Dumpling can have a quicker speed of data export. @@ -315,7 +316,7 @@ When Dumpling is exporting a large single table from TiDB, Out of Memory (OOM) m ### TiDB GC settings when exporting a large volume of data -When exporting data from TiDB, if the TiDB version is greater than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +When exporting data from TiDB, if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. In other scenarios, if the data size is very large, to avoid export failure due to GC during the export process, you can extend the GC time in advance: @@ -370,7 +371,7 @@ Finally, all the exported data can be imported back to TiDB using [TiDB Lightnin | `--cert` | The address of the client certificate file for TLS connection | | `--key` | The address of the client private key file for TLS connection | | `--csv-delimiter` | Delimiter of character type variables in CSV files | '"' | -| `--csv-separator` | Separator of each value in CSV files | ',' | +| `--csv-separator` | Separator of each value in CSV files. It is not recommended to use the default ‘,’. It is recommended to use ‘\|+\|’ or other uncommon character combinations| ',' | ',' | | `--csv-null-value` | Representation of null values in CSV files | "\\N" | | `--escape-backslash` | Use backslash (`\`) to escape special characters in the export file | true | | `--output-filename-template` | The filename templates represented in the format of [golang template](https://golang.org/pkg/text/template/#hdr-Arguments)
Support the `{{.DB}}`, `{{.Table}}`, and `{{.Index}}` arguments
The three arguments represent the database name, table name, and chunk ID of the data file | '{{.DB}}.{{.Table}}.{{.Index}}' | diff --git a/dynamic-config.md b/dynamic-config.md index 6ade8ff606461..a0a53a7ad9db7 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -132,7 +132,6 @@ The following TiKV configuration items can be modified online: | `raftstore.raft-log-gc-count-limit` | The hard limit on the allowable number of residual Raft logs | | `raftstore.raft-log-gc-size-limit` | The hard limit on the allowable size of residual Raft logs | | `raftstore.raft-entry-cache-life-time` | The maximum remaining time allowed for the log cache in memory | -| `raftstore.raft-reject-transfer-leader-duration` | Determines the smallest duration that a Leader is transferred to a newly added node | | `raftstore.split-region-check-tick-interval` | The time interval at which to check whether the Region split is needed | | `raftstore.region-split-check-diff` | The maximum value by which the Region data is allowed to exceed before Region split | | `raftstore.region-compact-check-interval` | The time interval at which to check whether it is necessary to manually trigger RocksDB compaction | @@ -198,6 +197,11 @@ The following TiKV configuration items can be modified online: | `split.byte-threshold` | The threshold to execute `load-base-split` on a Region. If the traffic of read requests for a Region exceeds the `byte-threshold` for a consecutive period of time, this Region should be split. | | `split.split-balance-score` | The parameter of `load-base-split`, which ensures the load of the two split Regions is as balanced as possible. The smaller the value is, the more balanced the load is. But setting it too small might cause split failure. | | `split.split-contained-score` | The parameter of `load-base-split`. The smaller the value, the fewer cross-Region visits after Region split. | +| `cdc.min-ts-interval` | The time interval at which Resolved TS is forwarded | +| `cdc.old-value-cache-memory-quota` | The upper limit of memory occupied by the TiCDC Old Value entries | +| `cdc.sink-memory-quota` | The upper limit of memory occupied by TiCDC data change events | +| `cdc.incremental-scan-speed-limit` | The upper limit on the speed of incremental scanning for historical data | +| `cdc.incremental-scan-concurrency` | The maximum number of concurrent incremental scanning tasks for historical data | In the table above, parameters with the `{db-name}` or `{db-name}.{cf-name}` prefix are configurations related to RocksDB. The optional values of `db-name` are `rocksdb` and `raftdb`. diff --git a/enable-tls-between-clients-and-servers.md b/enable-tls-between-clients-and-servers.md index c1d3278274cca..a8f62d607fc8b 100644 --- a/enable-tls-between-clients-and-servers.md +++ b/enable-tls-between-clients-and-servers.md @@ -35,14 +35,14 @@ Similar to MySQL, TiDB allows TLS and non-TLS connections on the same TCP port. ## Configure TiDB server to use secure connections -See the following desrciptions about the related parameters to enable secure connections: +See the following descriptions about the related parameters to enable secure connections: - [`auto-tls`](/tidb-configuration-file.md#auto-tls): enables automatic certificate generation (since v5.2.0) - [`ssl-cert`](/tidb-configuration-file.md#ssl-cert): specifies the file path of the SSL certificate - [`ssl-key`](/tidb-configuration-file.md#ssl-key): specifies the private key that matches the certificate - [`ssl-ca`](/tidb-configuration-file.md#ssl-ca): (optional) specifies the file path of the trusted CA certificate -`auto-tls` allows secure connections but does not provide client certificate validation. For certificate validation, and to control how certificates are generated, see the advice on configuring the `ssl-cert`, `ssl-key` and `ssl-ca` variables below. +`auto-tls` allows secure connections but does not provide client certificate validation. For certificate validation, and to control how certificates are generated, see the advice on configuring the `ssl-cert`, `ssl-key` and `ssl-ca` variables below. To enable secure connections with your own certificates in the TiDB server, you must specify both of the `ssl-cert` and `ssl-key` parameters in the configuration file when you start the TiDB server. You can also specify the `ssl-ca` parameter for client authentication (see [Enable authentication](#enable-authentication)). @@ -50,7 +50,7 @@ All the files specified by the parameters are in PEM (Privacy Enhanced Mail) for If the certificate parameters are correct, TiDB outputs `secure connection is enabled` when started; otherwise, it outputs `secure connection is NOT ENABLED`. -For TiDB versions earlier than v5.2.0, you can use `mysql_ssl_rsa_setup --datadir=./certs` to generate certficates. The `mysql_ssal_rsa_setup` tool is a part of MySQL Server. +For TiDB versions earlier than v5.2.0, you can use `mysql_ssl_rsa_setup --datadir=./certs` to generate certficates. The `mysql_ssl_rsa_setup` tool is a part of MySQL Server. ## Configure the MySQL client to use encrypted connections @@ -164,7 +164,7 @@ To replace the certificate, the key or CA, first replace the corresponding files The newly loaded certificate, key, and CA take effect on the connection that is established after the statement is successfully executed. The connection established before the statement execution is not affected. -## Montoring +## Monitoring Since TiDB v5.2.0, you can use the `Ssl_server_not_after` and `Ssl_server_not_before` status variables to monitor the start and end dates of the validity of the certificate. diff --git a/enable-tls-between-components.md b/enable-tls-between-components.md index 292f60cfd1cfb..5d540f80de4fd 100644 --- a/enable-tls-between-components.md +++ b/enable-tls-between-components.md @@ -125,7 +125,7 @@ Currently, it is not supported to only enable encrypted transmission of some spe {{< copyable "shell-regular" >}} ```bash - ./pd-ctl -u https://127.0.0.1:2379 --cacert /path/to/ca.pem --cert /path/to/client.pem --key /path/to/client-key.pem + tiup ctl pd -u https://127.0.0.1:2379 --cacert /path/to/ca.pem --cert /path/to/client.pem --key /path/to/client-key.pem ``` {{< copyable "shell-regular" >}} diff --git a/experimental-features.md b/experimental-features.md index aa55505635c1c..961e1bdae8e50 100644 --- a/experimental-features.md +++ b/experimental-features.md @@ -22,14 +22,15 @@ This document introduces the experimental features of TiDB in different versions ## SQL ++ [Use SQL interface to set placement rules for data] (/placement-rules-in-sql.md) (Introduced in v5.3) + List Partition (Introduced in v5.0) + List COLUMNS Partition (Introduced in v5.0) + [Dynamic Pruning Mode for Partitioned Tables](/partitioned-table.md#dynamic-pruning-mode). (Introduced in v5.1) + The expression index feature. The expression index is also called the function-based index. When you create an index, the index fields do not have to be a specific column but can be an expression calculated from one or more columns. This feature is useful for quickly accessing the calculation-based tables. See [Expression index](/sql-statements/sql-statement-create-index.md) for details. (Introduced in v4.0) -+ [Generated Columns](/generated-columns.md). -+ [User-Defined Variables](/user-defined-variables.md). -+ [JSON data type](/data-type-json.md) and [JSON functions](/functions-and-operators/json-functions.md). -+ [View](/information-schema/information-schema-views.md). ++ [Generated Columns](/generated-columns.md) (Introduced in v2.1) ++ [User-Defined Variables](/user-defined-variables.md) (Introduced in v2.1) ++ [JSON data type](/data-type-json.md) and [JSON functions](/functions-and-operators/json-functions.md) (Introduced in v2.1) ++ [View](/information-schema/information-schema-views.md) (Introduced in v2.1) ## Configuration management @@ -39,23 +40,28 @@ This document introduces the experimental features of TiDB in different versions ## Data sharing and subscription + [Integrate TiCDC with Kafka Connect (Confluent Platform)](/ticdc/integrate-confluent-using-ticdc.md) (Introduced in v5.0) -+ [The cyclic replication feature of TiCDC](/ticdc/manage-ticdc.md#cyclic-replication) (Introduced in v5.0) ## Storage -+ [Disable Titan](/storage-engine/titan-configuration.md#disable-titan-experimental). -+ [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental). ++ [Disable Titan](/storage-engine/titan-configuration.md#disable-titan-experimental) (Introduced in v4.0) ++ [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) (Introduced in v4.0) + TiFlash supports distributing the new data of the storage engine on multiple hard drives to share the I/O pressure. (Introduced in v4.0) +## Data migration + ++ [DM OpenAPI](https://docs.pingcap.com/tidb-data-migration/stable/open-api) (Introduced in v5.3) + ## Backup and restoration -+ [Back up Raw KV](/br/use-br-command-line-tool.md#back-up-raw-kv-experimental-feature). ++ [Back up Raw KV](/br/use-br-command-line-tool.md#back-up-raw-kv-experimental-feature) (Introduced in v3.1) ## Garbage collection -+ [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50). ++ [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) (Introduced in v5.0) ## Diagnostics -+ [SQL diagnostics](/information-schema/information-schema-sql-diagnostics.md). -+ [Cluster diagnostics](/dashboard/dashboard-diagnostics-access.md). ++ [SQL diagnostics](/information-schema/information-schema-sql-diagnostics.md) (Introduced in v4.0) ++ [Cluster diagnostics](/dashboard/dashboard-diagnostics-access.md) (Introduced in v4.0) ++ [Continuous profiling](/dashboard/continuous-profiling.md) (Introduced in v5.3) ++ [Online Unsafe Recovery](/online-unsafe-recovery.md) (Introduced in v5.3) \ No newline at end of file diff --git a/explain-indexes.md b/explain-indexes.md index 75c0e22c10c29..b5a13ea8ea26c 100644 --- a/explain-indexes.md +++ b/explain-indexes.md @@ -40,7 +40,6 @@ TiDB uses the `IndexLookup` operator when retrieving data from a secondary index EXPLAIN SELECT * FROM t1 WHERE intkey = 123; EXPLAIN SELECT * FROM t1 WHERE intkey < 10; EXPLAIN SELECT * FROM t1 WHERE intkey BETWEEN 300 AND 310; -EXPLAIN SELECT * FROM t1 WHERE intkey BETWEEN 300 AND 310; EXPLAIN SELECT * FROM t1 WHERE intkey IN (123,29,98); EXPLAIN SELECT * FROM t1 WHERE intkey >= 99 AND intkey <= 103; ``` diff --git a/faq/migration-tidb-faq.md b/faq/migration-tidb-faq.md index 7d65592647351..eb88e252fe356 100644 --- a/faq/migration-tidb-faq.md +++ b/faq/migration-tidb-faq.md @@ -174,7 +174,7 @@ Deleting a large amount of data leaves a lot of useless keys, affecting the quer ### What is the most efficient way of deleting data? -When deleting a large amount of data, it is recommended to use `Delete * from t where xx limit 5000;`. It deletes through the loop and uses `Affected Rows == 0` as a condition to end the loop, so as not to exceed the limit of transaction size. With the prerequisite of meeting business filtering logic, it is recommended to add a strong filter index column or directly use the primary key to select the range, such as `id >= 5000*n+m and id < 5000*(n+1)+m`. +When deleting a large amount of data, it is recommended to use `Delete from t where xx limit 5000;`. It deletes through the loop and uses `Affected Rows == 0` as a condition to end the loop, so as not to exceed the limit of transaction size. With the prerequisite of meeting business filtering logic, it is recommended to add a strong filter index column or directly use the primary key to select the range, such as `id >= 5000*n+m and id < 5000*(n+1)+m`. If the amount of data that needs to be deleted at a time is very large, this loop method will get slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all will be processed by Garbage Collection) and influence the following Delete statement. If possible, it is recommended to refine the Where condition. See [details in TiDB Best Practices](https://pingcap.com/blog/2017-07-24-tidbbestpractice/#write). diff --git a/faq/sql-faq.md b/faq/sql-faq.md index dd1749b87df40..1547061b8fa3e 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -122,7 +122,7 @@ Two solutions: ## Does TiDB release space immediately after deleting data? -None of the `DELETE`, `TRUNCATE` and `DROP` operations release data immediately. For the `TRUNCATE` and `DROP` operations, after the TiDB GC (Garbage Collection) time (10 minutes by default), the data is deleted and the space is released. For the `DELETE` operation, the data is deleted but the space is not released according to TiDB GC. When subsequent data is written into RocksDB and executes `COMPACT`, the space is reused. +None of the `DELETE`, `TRUNCATE` and `DROP` operations release data immediately. For the `TRUNCATE` and `DROP` operations, after the TiDB GC (Garbage Collection) time (10 minutes by default), the data is deleted and the space is released. For the `DELETE` operations, the data is deleted but the space is not immediately released until the compaction is performed. ## Does TiDB support the `REPLACE INTO` syntax? diff --git a/generated-columns.md b/generated-columns.md index 9921404e1dae9..721910077c99a 100644 --- a/generated-columns.md +++ b/generated-columns.md @@ -142,7 +142,7 @@ desc select a+1 from t where a+1=3; > > In the above example, the column type of `a` is int and the column type of `a+1` is bigint. If the type of the generated column is set to int, the replacement will not occur. > -> For type conversion rules, see [Type Conversion of Expression Evaluation] (/functions-and-operators/type-conversion-in-expression-evaluation.md). +> For type conversion rules, see [Type Conversion of Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md). ## Limitations diff --git a/get-started-with-tidb-lightning.md b/get-started-with-tidb-lightning.md index a87108f108633..32d27a71c41f2 100644 --- a/get-started-with-tidb-lightning.md +++ b/get-started-with-tidb-lightning.md @@ -64,7 +64,7 @@ Download the TiDB Lightning installation package from the following link: ```toml [lightning] - # logging + # Logging level = "info" file = "tidb-lightning.log" diff --git a/hardware-and-software-requirements.md b/hardware-and-software-requirements.md index 094722a101c21..03f9767cc2d00 100644 --- a/hardware-and-software-requirements.md +++ b/hardware-and-software-requirements.md @@ -91,11 +91,11 @@ You can deploy and run TiDB on the 64-bit generic hardware server platform in th Before you deploy TiFlash, note the following items: - TiFlash can be [deployed on multiple disks](/tiflash/tiflash-configuration.md#multi-disk-deployment). -- It is recommended to use a high-performance SSD as the first disk of the TiFlash data directory to buffer the real-time replication of TiKV data. The performance of this disk should not be lower than that of TiKV, such as PCI-E SSD. The disk capacity should be no less than 10% of the total capacity; otherwise, it might become the bottleneck of this node. You can deploy ordinary SSDs for other disks, but note that a better PCI-E SSD brings better performance. +- It is recommended to use a high-performance SSD as the first disk of the TiFlash data directory to buffer the real-time replication of TiKV data. The performance of this disk should not be lower than that of TiKV, such as PCI-E SSD. The disk capacity should be no less than 10% of the total capacity; otherwise, it might become the bottleneck of this node. You can deploy ordinary SSDs for other disks, but note that a better PCI-E SSD brings better performance. - It is recommended to deploy TiFlash on different nodes from TiKV. If you must deploy TiFlash and TiKV on the same node, increase the number of CPU cores and memory, and try to deploy TiFlash and TiKV on different disks to avoid interfering each other. - The total capacity of the TiFlash disks is calculated in this way: `the data volume of the entire TiKV cluster to be replicated / the number of TiKV replicas * the number of TiFlash replicas`. For example, if the overall planned capacity of TiKV is 1 TB, the number of TiKV replicas is 3, and the number of TiFlash replicas is 2, then the recommended total capacity of TiFlash is `1024 GB / 3 * 2`. You can replicate only the data of some tables. In such case, determine the TiFlash capacity according to the data volume of the tables to be replicated. -Before you deploy TiCDC, note that it is recommended to deploy TiCDC on PCIe-SSD disks larger than 200 GB. +Before you deploy TiCDC, note that it is recommended to deploy TiCDC on PCIe-SSD disks larger than 1 TB. ## Network requirements diff --git a/information-schema/information-schema-placement-rules.md b/information-schema/information-schema-placement-rules.md new file mode 100644 index 0000000000000..34a51747de883 --- /dev/null +++ b/information-schema/information-schema-placement-rules.md @@ -0,0 +1,80 @@ +--- +title: PLACEMENT_RULES +summary: Learn the `PLACEMENT_RULES` information_schema table. +--- + +# PLACEMENT_RULES + +The `PLACEMENT_RULES` table provides information on all explicitly configured [Placement Rules in SQL](/placement-rules-in-sql.md). The information includes both placement policies and directly attached rules. + +{{< copyable "sql" >}} + +```sql +USE information_schema; +DESC placement_rules; +``` + +```sql ++----------------------+--------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------------------+--------------+------+------+---------+-------+ +| POLICY_ID | bigint(64) | NO | | NULL | | +| CATALOG_NAME | varchar(512) | NO | | NULL | | +| POLICY_NAME | varchar(5) | YES | | NULL | | +| SCHEMA_NAME | varchar(5) | YES | | NULL | | +| TABLE_NAME | varchar(5) | YES | | NULL | | +| PARTITION_NAME | varchar(5) | YES | | NULL | | +| PRIMARY_REGION | varchar(5) | NO | | NULL | | +| REGIONS | varchar(5) | NO | | NULL | | +| CONSTRAINTS | varchar(5) | NO | | NULL | | +| LEADER_CONSTRAINTS | varchar(5) | NO | | NULL | | +| FOLLOWER_CONSTRAINTS | varchar(5) | NO | | NULL | | +| LEARNER_CONSTRAINTS | varchar(5) | NO | | NULL | | +| SCHEDULE | varchar(20) | NO | | NULL | | +| FOLLOWERS | bigint(64) | NO | | NULL | | +| LEARNERS | bigint(64) | NO | | NULL | | ++----------------------+--------------+------+------+---------+-------+ +15 rows in set (0.00 sec) +``` + +## Examples + +The `PLACEMENT_RULES` table only shows explicitly configured rules. To see the canonical version of placement rules (including placement policies attached to tables), use the statement `SHOW PLACEMENT` instead: + +{{< copyable "sql" >}} + +```sql +CREATE TABLE t1 (a INT); +CREATE TABLE t2 (a INT) primary_region="us-east-1" regions="us-east-1"; +CREATE PLACEMENT POLICY p1 primary_region="us-east-1" regions="us-east-1"; +CREATE TABLE t3 (a INT) PLACEMENT POLICY=p1; +SHOW PLACEMENT; -- Includes t3. +SELECT * FROM information_schema.placement_rules; -- Does not include t3. +``` + +```sql +Query OK, 0 rows affected (0.09 sec) + +Query OK, 0 rows affected (0.11 sec) + +Query OK, 0 rows affected (0.08 sec) + +Query OK, 0 rows affected (0.11 sec) + ++---------------+------------------------------------------------+ +| Target | Placement | ++---------------+------------------------------------------------+ +| POLICY p1 | PRIMARY_REGION="us-east-1" REGIONS="us-east-1" | +| TABLE test.t2 | PRIMARY_REGION="us-east-1" REGIONS="us-east-1" | +| TABLE test.t3 | PRIMARY_REGION="us-east-1" REGIONS="us-east-1" | ++---------------+------------------------------------------------+ +3 rows in set (0.00 sec) + ++-----------+--------------+-------------+-------------+------------+----------------+----------------+-----------+-------------+--------------------+----------------------+---------------------+----------+-----------+----------+ +| POLICY_ID | CATALOG_NAME | POLICY_NAME | SCHEMA_NAME | TABLE_NAME | PARTITION_NAME | PRIMARY_REGION | REGIONS | CONSTRAINTS | LEADER_CONSTRAINTS | FOLLOWER_CONSTRAINTS | LEARNER_CONSTRAINTS | SCHEDULE | FOLLOWERS | LEARNERS | ++-----------+--------------+-------------+-------------+------------+----------------+----------------+-----------+-------------+--------------------+----------------------+---------------------+----------+-----------+----------+ +| 3 | def | p1 | NULL | NULL | NULL | us-east-1 | us-east-1 | | | | | | 0 | 0 | +| NULL | def | NULL | test | t2 | NULL | us-east-1 | us-east-1 | | | | | | 0 | 0 | ++-----------+--------------+-------------+-------------+------------+----------------+----------------+-----------+-------------+--------------------+----------------------+---------------------+----------+-----------+----------+ +2 rows in set (0.00 sec) +``` diff --git a/information-schema/information-schema-statistics.md b/information-schema/information-schema-statistics.md index b113b21e40094..2b02a35794522 100644 --- a/information-schema/information-schema-statistics.md +++ b/information-schema/information-schema-statistics.md @@ -51,7 +51,7 @@ Fields in the `STATISTICS` table are described as follows: * `SEQ_IN_INDEX`: The column number in the index, starting from `1`. * `COLUMN_NAME`: The column name. See the description of the `Expression` column. * `COLLATION`: The sorting method of the columns in the index. The value can be `A` (ascending order), `D` (descending order) or `NULL` (unsorted). -* `CARDINALITY`: The estimated number of unique values ​​in the index. To update this value, execute `ANALYZE TABLE`. +* `CARDINALITY`: TiDB does not use this field. The field value is always `0`. * `SUB_PART`: The prefix of the index. If only part of the prefix of the column is indexed, the value is the number of indexed characters; if the entire column is indexed, the value is `NULL`. * `PACKED`: TiDB does not use this field. This value is always `NULL`. * `NULLABLE`: If the column might contain a `NULL` value, the value is `YES`; if not, the value is `''`. diff --git a/information-schema/information-schema.md b/information-schema/information-schema.md index c06b656376e69..5ac50eca3b3a8 100644 --- a/information-schema/information-schema.md +++ b/information-schema/information-schema.md @@ -76,6 +76,7 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefi | [`METRICS_SUMMARY`](/information-schema/information-schema-metrics-summary.md) | A summary of metrics extracted from Prometheus. | | `METRICS_SUMMARY_BY_LABEL` | See `METRICS_SUMMARY` table. | | [`METRICS_TABLES`](/information-schema/information-schema-metrics-tables.md) | Provides the PromQL definitions for tables in `METRICS_SCHEMA`. | +| [`PLACEMENT_RULES`](/information-schema/information-schema-placement-rules.md) | Provides information on all objects that have explicit placement rules assigned. | | [`SEQUENCES`](/information-schema/information-schema-sequences.md) | The TiDB implementation of sequences is based on MariaDB. | | [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. | | [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. | diff --git a/media/cdc-cyclic-replication.png b/media/cdc-cyclic-replication.png deleted file mode 100644 index 90bd33baf35e1..0000000000000 Binary files a/media/cdc-cyclic-replication.png and /dev/null differ diff --git a/media/dashboard/dashboard-conprof-access.png b/media/dashboard/dashboard-conprof-access.png new file mode 100644 index 0000000000000..4d40ec391a944 Binary files /dev/null and b/media/dashboard/dashboard-conprof-access.png differ diff --git a/media/dashboard/dashboard-conprof-download.png b/media/dashboard/dashboard-conprof-download.png new file mode 100644 index 0000000000000..eb70428e8ca38 Binary files /dev/null and b/media/dashboard/dashboard-conprof-download.png differ diff --git a/media/dashboard/dashboard-conprof-history.png b/media/dashboard/dashboard-conprof-history.png new file mode 100644 index 0000000000000..e599bc3bd5c91 Binary files /dev/null and b/media/dashboard/dashboard-conprof-history.png differ diff --git a/media/dashboard/dashboard-conprof-single.png b/media/dashboard/dashboard-conprof-single.png new file mode 100644 index 0000000000000..a1f937c229016 Binary files /dev/null and b/media/dashboard/dashboard-conprof-single.png differ diff --git a/media/dashboard/dashboard-conprof-start.png b/media/dashboard/dashboard-conprof-start.png new file mode 100644 index 0000000000000..d85a2e3f8b1c2 Binary files /dev/null and b/media/dashboard/dashboard-conprof-start.png differ diff --git a/media/dashboard/dashboard-conprof-stop.png b/media/dashboard/dashboard-conprof-stop.png new file mode 100644 index 0000000000000..8054473dc235b Binary files /dev/null and b/media/dashboard/dashboard-conprof-stop.png differ diff --git a/media/dashboard/dashboard-profiling-access.png b/media/dashboard/dashboard-profiling-access.png index 107d30b5781ef..0364bfc3947dd 100644 Binary files a/media/dashboard/dashboard-profiling-access.png and b/media/dashboard/dashboard-profiling-access.png differ diff --git a/media/dashboard/dashboard-session-sso-auth0-create-app.png b/media/dashboard/dashboard-session-sso-auth0-create-app.png new file mode 100644 index 0000000000000..74fc2fe50470e Binary files /dev/null and b/media/dashboard/dashboard-session-sso-auth0-create-app.png differ diff --git a/media/dashboard/dashboard-session-sso-auth0-settings-1.png b/media/dashboard/dashboard-session-sso-auth0-settings-1.png new file mode 100644 index 0000000000000..4ee9b4fcef17c Binary files /dev/null and b/media/dashboard/dashboard-session-sso-auth0-settings-1.png differ diff --git a/media/dashboard/dashboard-session-sso-auth0-settings-2.png b/media/dashboard/dashboard-session-sso-auth0-settings-2.png new file mode 100644 index 0000000000000..5a3fe11915db8 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-auth0-settings-2.png differ diff --git a/media/dashboard/dashboard-session-sso-auth0-settings-3.png b/media/dashboard/dashboard-session-sso-auth0-settings-3.png new file mode 100644 index 0000000000000..c340088308815 Binary files /dev/null and b/media/dashboard/dashboard-session-sso-auth0-settings-3.png differ diff --git a/media/dashboard/dashboard-user-insufficient-privileges.png b/media/dashboard/dashboard-user-insufficient-privileges.png new file mode 100644 index 0000000000000..eccb4412d172d Binary files /dev/null and b/media/dashboard/dashboard-user-insufficient-privileges.png differ diff --git a/media/parallel-import-shard-tables-en.png b/media/parallel-import-shard-tables-en.png new file mode 100644 index 0000000000000..f2badd517c81d Binary files /dev/null and b/media/parallel-import-shard-tables-en.png differ diff --git a/media/parallel-import-single-tables-en.png b/media/parallel-import-single-tables-en.png new file mode 100644 index 0000000000000..e03c895400f5c Binary files /dev/null and b/media/parallel-import-single-tables-en.png differ diff --git a/media/shard-table-replica-1.png b/media/shard-table-replica-1.png index 925c6993a16a1..87d6a7415bec6 100644 Binary files a/media/shard-table-replica-1.png and b/media/shard-table-replica-1.png differ diff --git a/media/sqlgram/SelectLockOpt.png b/media/sqlgram/SelectLockOpt.png deleted file mode 100644 index c36ae018a0536..0000000000000 Binary files a/media/sqlgram/SelectLockOpt.png and /dev/null differ diff --git a/media/sysbench_v522vsv530_point_select.png b/media/sysbench_v522vsv530_point_select.png new file mode 100644 index 0000000000000..4e9db64a36037 Binary files /dev/null and b/media/sysbench_v522vsv530_point_select.png differ diff --git a/media/sysbench_v522vsv530_read_write.png b/media/sysbench_v522vsv530_read_write.png new file mode 100644 index 0000000000000..94772c1006a89 Binary files /dev/null and b/media/sysbench_v522vsv530_read_write.png differ diff --git a/media/sysbench_v522vsv530_update_index.png b/media/sysbench_v522vsv530_update_index.png new file mode 100644 index 0000000000000..9975c720e1638 Binary files /dev/null and b/media/sysbench_v522vsv530_update_index.png differ diff --git a/media/sysbench_v522vsv530_update_non_index.png b/media/sysbench_v522vsv530_update_non_index.png new file mode 100644 index 0000000000000..cdb9000c92957 Binary files /dev/null and b/media/sysbench_v522vsv530_update_non_index.png differ diff --git a/media/tidb-v5-tpch-100-vs-gp-spark.png b/media/tidb-v5-tpch-100-vs-gp-spark.png new file mode 100644 index 0000000000000..250e80dac7bd0 Binary files /dev/null and b/media/tidb-v5-tpch-100-vs-gp-spark.png differ diff --git a/media/tidb-v5.3-tpch-100-vs-gp-spark.png b/media/tidb-v5.3-tpch-100-vs-gp-spark.png new file mode 100644 index 0000000000000..cbe2e3b230476 Binary files /dev/null and b/media/tidb-v5.3-tpch-100-vs-gp-spark.png differ diff --git a/media/tpcc_v522_vs_v530.png b/media/tpcc_v522_vs_v530.png new file mode 100644 index 0000000000000..4a37373c83f2d Binary files /dev/null and b/media/tpcc_v522_vs_v530.png differ diff --git a/mysql-compatibility.md b/mysql-compatibility.md index 3058b1c8a303e..d21e34cdbcf12 100644 --- a/mysql-compatibility.md +++ b/mysql-compatibility.md @@ -25,7 +25,6 @@ However, some features of MySQL are not supported. This could be because there i + Events + User-defined functions + `FOREIGN KEY` constraints [#18209](https://github.com/pingcap/tidb/issues/18209) -+ Temporary tables [#1248](https://github.com/pingcap/tidb/issues/1248) + `FULLTEXT`/`SPATIAL` functions and indexes [#1793](https://github.com/pingcap/tidb/issues/1793) + Character sets other than `utf8`, `utf8mb4`, `ascii`, `latin1` and `binary` + SYS schema @@ -121,6 +120,10 @@ These differences are documented further in [`ANALYZE TABLE`](/sql-statements/sq Views in TiDB are not updatable. They do not support write operations such as `UPDATE`, `INSERT`, and `DELETE`. +### Temporary tables + +For details, see [Compatibility between TiDB local temporary tables and MySQL temporary tables](/temporary-tables.md#compatibility-with-mysql-temporary-tables). + ### Storage engines For compatibility reasons, TiDB supports the syntax to create tables with alternative storage engines. In implementation, TiDB describes the metadata as the InnoDB storage engine. diff --git a/online-unsafe-recovery.md b/online-unsafe-recovery.md new file mode 100644 index 0000000000000..7d3fd9d6d14bf --- /dev/null +++ b/online-unsafe-recovery.md @@ -0,0 +1,84 @@ +--- +title: Online Unsafe Recovery +summary: Learn how to use Online Unsafe Recovery. +--- + +# Online Unsafe Recovery + +> **Warning:** +> +> - Online Unsafe Recovery is a type of lossy recovery. If you use this feature, the integrity of data and data indexes cannot be guaranteed. +> - Online Unsafe Recovery is an experimental feature, and it is **NOT** recommended to use it in the production environment. The interface, strategy, and internal implementation of this feature might change when it becomes generally available (GA). Although this feature has been tested in some scenarios, it is not thoroughly validated and might cause system unavailability. +> - It is recommended to perform the feature-related operations with the support from the TiDB team. If any misoperation is performed, it might be hard to recover the cluster. + +When permanently damaged replicas cause part of data on TiKV to be unreadable and unwritable, you can use the Online Unsafe Recovery feature to perform a lossy recovery operation. + +## Feature description + +In TiDB, the same data might be stored in multiple stores at the same time according to the replica rules defined by users. This guarantees that data is still readable and writable even if a single or a few stores are temporarily offline or damaged. However, when most or all replicas of a Region go offline during a short period of time, the Region becomes temporarily unavailable, by design, to ensure data integrity. + +Suppose that multiple replicas of a data range encounter issues like permanent damage (such as disk damage), and these issues cause the stores to stay offline. In this case, this data range is temporarily unavailable. If you want the cluster back in use and also accept data rewind or data loss, in theory, you can re-form the majority of replicas by manually removing the failed replicas from the group. This allows application-layer services to read and write this data range (might be stale or empty) again. + +In this case, if some stores with loss-tolerant data are permanently damaged, you can perform a lossy recovery operation by using Online Unsafe Recovery. Using this feature, PD, under its global perspective, collects the metadata of data shards from all stores and generates a real-time and complete recovery plan. Then, PD distributes the plan to all surviving stores to make them perform data recovery tasks. In addition, once the data recovery plan is distributed, PD periodically monitors the recovery progress and re-send the plan when necessary. + +## User scenarios + +The Online Unsafe Recovery feature is suitable for the following scenarios: + +* The data for application services is unreadable and unwritable, because permanently damaged stores cause the stores to fail to restart. +* You can accept data loss and want the affected data to be readable and writable. +* You want to perform a one-stop online data recovery operation. + +## Usage + +### Prerequisites + +Before using Online Unsafe Recovery, make sure that the following requirements are met: + +* The offline stores indeed cause some pieces of data to be unavailable. +* The offline stores cannot be automatically recovered or restarted. + +### Step 1. Disable all types of scheduling + +You need to temporarily disable all types of internal scheduling, such as load balancing. After disabling them, it is recommended to wait for about 10 minutes so that the triggered scheduling can have sufficient time to complete the scheduled tasks. + +> **Note:** +> +> After the scheduling is disabled, the system cannot resolve data hotspot issues. Therefore, you need to enable the scheduling as soon as possible after the recovery is completed. + +1. Use PD Control to get the current configuration by running the [`config show`](/pd-control.md#config-show--set-option-value--placement-rules) command. +2. Use PD Control to disable all types of scheduling. For example: + + * [`config set region-schedule-limit 0`](/pd-control.md#config-show--set-option-value--placement-rules) + * [`config set replica-schedule-limit 0`](/pd-control.md#config-show--set-option-value--placement-rules) + * [`config set merge-schedule-limit 0`](/pd-control.md#config-show--set-option-value--placement-rules) + +### Step 2. Remove the stores that cannot be automatically recovered + +Use PD Control to remove the stores that cannot be automatically recovered by running the [`unsafe remove-failed-stores [,,...]`](/pd-control.md#unsafe-remove-failed-stores-store-ids--show--history) command. + +> **Note:** +> +> The returned result of this command only indicates that the request is accepted, not that the recovery is completed successfully. The stores are actually recovered in the background. + +### Step 3. Check the progress + +When the above store removal command runs successfully, you can use PD Control to check the removal progress by running the [`unsafe remove-failed-stores show`](/pd-control.md#config-show--set-option-value--placement-rules) command. When the command result shows "Last recovery has finished", the system recovery is completed. + +### Step 4. Test read and write tasks + +After the progress command shows that the recovery task is completed, you can try to execute some simple SQL queries like the following example or perform write tasks to ensure that the data is readable and writable. + +```sql +select count(*) from table_that_suffered_from_group_majority_failure; +``` + +> **Note:** +> +> The situation that data can be read and written does not indicate there is no data loss. + +### Step 5. Restart the scheduling + +To restart the scheduling, you need to adjust the `0` value of `config set region-schedule-limit 0`, `config set replica-schedule-limit 0`, and `config set merge-schedule-limit 0` modified in step 1 to the initial values. + +Then, the whole process is finished. \ No newline at end of file diff --git a/optimizer-hints.md b/optimizer-hints.md index aa9cb41eb8e0a..5422385d3ffaf 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -187,6 +187,16 @@ The `AGG_TO_COP()` hint tells the optimizer to push down the aggregate operation select /*+ AGG_TO_COP() */ sum(t1.a) from t t1; ``` +### LIMIT_TO_COP() + +The `LIMIT_TO_COP()` hint tells the optimizer to push down the `Limit` and `TopN` operators in the specified query block to the coprocessor. If the optimizer does not perform such an operation, it is recommended to use this hint. For example: + +{{< copyable "sql" >}} + +```sql +SELECT /*+ LIMIT_TO_COP() */ * FROM t WHERE a = 1 AND b > 10 ORDER BY c LIMIT 1; +``` + ### READ_FROM_STORAGE(TIFLASH[t1_name [, tl_name ...]], TIKV[t2_name [, tl_name ...]]) The `READ_FROM_STORAGE(TIFLASH[t1_name [, tl_name ...]], TIKV[t2_name [, tl_name ...]])` hint tells the optimizer to read specific table(s) from specific storage engine(s). Currently, this hint supports two storage engine parameters - `TIKV` and `TIFLASH`. If a table has an alias, use the alias as the parameter of `READ_FROM_STORAGE()`; if the table does not has an alias, use the table's original name as the parameter. For example: diff --git a/pd-configuration-file.md b/pd-configuration-file.md index 01f011044fa29..82873189ea228 100644 --- a/pd-configuration-file.md +++ b/pd-configuration-file.md @@ -156,14 +156,12 @@ Configuration items related to the log file ### `max-days` + The maximum number of days in which a log is kept -+ Default value: `28` -+ Minimum value: `1` ++ Default value: `0` ### `max-backups` + The maximum number of log files to keep -+ Default value: `7` -+ Minimum value: `1` ++ Default value: `0` ## `metric` @@ -191,7 +189,7 @@ Configuration items related to scheduling ### `patrol-region-interval` + Controls the running frequency at which `replicaChecker` checks the health state of a Region. The smaller this value is, the faster `replicaChecker` runs. Normally, you do not need to adjust this parameter. -+ Default value: `100ms` ++ Default value: `10ms` ### `split-merge-interval` @@ -201,12 +199,12 @@ Configuration items related to scheduling ### `max-snapshot-count` + Controls the maximum number of snapshots that a single store receives or sends at the same time. PD schedulers depend on this configuration to prevent the resources used for normal traffic from being preempted. -+ Default value value: `3` ++ Default value value: `64` ### `max-pending-peer-count` + Controls the maximum number of pending peers in a single store. PD schedulers depend on this configuration to prevent too many Regions with outdated logs from being generated on some nodes. -+ Default value: `16` ++ Default value: `64` ### `max-store-down-time` diff --git a/pd-control.md b/pd-control.md index 0007849bec7b0..9ad5be5bf9183 100644 --- a/pd-control.md +++ b/pd-control.md @@ -28,7 +28,7 @@ If you want to download the latest version of `pd-ctl`, directly download the Ti > **Note:** > -> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.2.1`, the package download link is `https://download.pingcap.org/tidb-v5.2.1-linux-amd64.tar.gz`. +> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.3.0`, the package download link is `https://download.pingcap.org/tidb-v5.3.0-linux-amd64.tar.gz`. ### Compile from source code @@ -40,26 +40,26 @@ If you want to download the latest version of `pd-ctl`, directly download the Ti Single-command mode: ```bash -./pd-ctl store -u http://127.0.0.1:2379 +tiup ctl pd store -u http://127.0.0.1:2379 ``` Interactive mode: ```bash -./pd-ctl -i -u http://127.0.0.1:2379 +tiup ctl pd -i -u http://127.0.0.1:2379 ``` Use environment variables: ```bash export PD_ADDR=http://127.0.0.1:2379 -./pd-ctl +tiup ctl pd ``` Use TLS to encrypt: ```bash -./pd-ctl -u https://127.0.0.1:2379 --cacert="path/to/ca" --cert="path/to/cert" --key="path/to/key" +tiup ctl pd -u https://127.0.0.1:2379 --cacert="path/to/ca" --cert="path/to/cert" --key="path/to/key" ``` ## Command line flags @@ -131,7 +131,7 @@ Usage: >> config show // Display the config information of the scheduling { "replication": { - "enable-placement-rules": "false", + "enable-placement-rules": "true", "isolation-level": "", "location-labels": "", "max-replicas": 3, @@ -139,13 +139,6 @@ Usage: }, "schedule": { "enable-cross-table-merge": "true", - "enable-debug-metrics": "false", - "enable-location-replacement": "true", - "enable-make-up-replica": "true", - "enable-one-way-merge": "false", - "enable-remove-down-replica": "true", - "enable-remove-extra-replica": "true", - "enable-replace-offline-replica": "true", "high-space-ratio": 0.7, "hot-region-cache-hits-threshold": 3, "hot-region-schedule-limit": 4, @@ -154,17 +147,16 @@ Usage: "low-space-ratio": 0.8, "max-merge-region-keys": 200000, "max-merge-region-size": 20, - "max-pending-peer-count": 16, - "max-snapshot-count": 3, + "max-pending-peer-count": 64, + "max-snapshot-count": 64, "max-store-down-time": "30m0s", "merge-schedule-limit": 8, - "patrol-region-interval": "100ms", + "patrol-region-interval": "10ms", "region-schedule-limit": 2048, "region-score-formula-version": "v2", "replica-schedule-limit": 64, "scheduler-max-waiting-operator": 5, "split-merge-interval": "1h0m0s", - "store-limit-mode": "manual", "tolerant-size-ratio": 0 } } @@ -175,17 +167,17 @@ Usage: "location-labels": "", "isolation-level": "", "strictly-match-label": "false", - "enable-placement-rules": "false" + "enable-placement-rules": "true" } >> config show cluster-version // Display the current version of the cluster, which is the current minimum version of TiKV nodes in the cluster and does not correspond to the binary version. -"5.2.1" +"5.2.2" ``` - `max-snapshot-count` controls the maximum number of snapshots that a single store receives or sends out at the same time. The scheduler is restricted by this configuration to avoid taking up normal application resources. When you need to improve the speed of adding replicas or balancing, increase this value. ```bash - >> config set max-snapshot-count 16 // Set the maximum number of snapshots to 16 + >> config set max-snapshot-count 64 // Set the maximum number of snapshots to 64 ``` - `max-pending-peer-count` controls the maximum number of pending peers in a single store. The scheduler is restricted by this configuration to avoid producing a large number of Regions without the latest log in some nodes. When you need to improve the speed of adding replicas or balancing, increase this value. Setting it to 0 indicates no limit. @@ -243,7 +235,7 @@ Usage: - `patrol-region-interval` controls the execution frequency that `replicaChecker` checks the health status of Regions. A shorter interval indicates a higher execution frequency. Generally, you do not need to adjust it. ```bash - >> config set patrol-region-interval 50ms // Set the execution frequency of replicaChecker to 50ms + >> config set patrol-region-interval 10ms // Set the execution frequency of replicaChecker to 10ms ``` - `max-store-down-time` controls the time that PD decides the disconnected store cannot be restored if exceeded. If PD does not receive heartbeats from a store within the specified period of time, PD adds replicas in other nodes. @@ -795,7 +787,7 @@ Usage: - `read-priorities` and `write-leader-priorities` control which dimensions the scheduler prioritizes for scheduling hot Regions of the read and write-leader types. The dimension options are `query`, `byte`, and `key`. - `write-peer-priorities` controls which dimensions the scheduler prioritizes for scheduling hot Regions of the write-peer type. The dimension options are `byte` and `key`. - + > **Note:** > > If a cluster component is earlier than v5.2, the configuration of `query` dimension does not take effect. If some components are upgraded to v5.2 or later, the `byte` and `key` dimensions still by default have the priority for hot Region scheduling. After all components of the cluster are upgraded to v5.2 or later, such a configuration still takes effect for compatibility. You can view the real-time configuration using the `pd-ctl` command. Usually, you do not need to modify these configurations. @@ -816,7 +808,7 @@ Usage: >> scheduler config balance-hot-region-scheduler set enable-for-tiflash true ``` -### `store [delete | label | weight | remove-tombstone | limit | limit-scene] [--jq=""]` +### `store [delete | label | weight | remove-tombstone | limit ] [--jq=""]` Use this command to view the store information or remove a specified store. For a jq formatted output, see [jq-formatted-json-output-usage](#jq-formatted-json-output-usage). @@ -829,9 +821,9 @@ Usage: "stores": [...] } >> store 1 // Get the store with the store id of 1 - ...... +...... >> store delete 1 // Delete the store with the store id of 1 - ...... +...... >> store label 1 zone cn // Set the value of the label with the "zone" key to "cn" for the store with the store id of 1 >> store weight 1 5 10 // Set the leader weight to 5 and Region weight to 10 for the store with the store id of 1 >> store remove-tombstone // Remove stores that are in tombstone state @@ -844,14 +836,6 @@ Usage: >> store limit 1 5 add-peer // Set the limit of adding-peer operations to 5 per minute for store 1 >> store limit 1 5 remove-peer // Set the limit of removing-peer operations to 5 per minute for store 1 >> store limit all 5 remove-peer // Set the limit of removing-peer operations to 5 per minute for all stores ->> store limit-scene // Show all limit scenarios (experimental) -{ - "Idle": 100, - "Low": 50, - "Normal": 32, - "High": 12 -} ->> store limit-scene idle 100 // set rate to 100 in the idle scene (experimental) ``` > **Note:** @@ -880,12 +864,67 @@ system: 2017-10-09 05:50:59 +0800 CST logic: 120102 ``` +### `unsafe remove-failed-stores [store-ids | show | history]` + +> **Warning:** +> +> - This feature is a lossy recovery, so TiKV cannot guarantee data integrity and data indexes integrity after using the feature. +> - Online Unsafe Recovery is an experimental feature, and it is **NOT** recommended to use it in the production environment. The interface, strategy, and internal implementation of this feature might change when it becomes generally available (GA). Although this feature has been tested in some scenarios, it is not thoroughly validated and might cause system unavailability. +> - It is recommended to perform the feature-related operations with the support from the TiDB team. If any misoperation is performed, it might be hard to recover the cluster. + +Use this command to perform lossy recovery operations when permanently damaged replicas cause data to be unavailable. For example: + +Execute Online Unsafe Recovery to remove permanently damaged stores: + +```bash +>> unsafe remove-failed-stores 101,102,103 +``` + +```bash +Success! +``` + +Show the current or historical state of Online Unsafe Recovery: + +```bash +>> unsafe remove-failed-stores show +``` + +```bash +[ + "Collecting cluster info from all alive stores, 10/12.", + "Stores that have reports to PD: 1, 2, 3, ...", + "Stores that have not reported to PD: 11, 12", +] +``` + +```bash +>> unsafe remove-failed-stores history +``` + +```bash +[ + "Store reports collection:", + "Store 7: region 3 [start_key, end_key), {peer1, peer2, peer3} region 4 ...", + "Store 8: region ...", + "...", + "Recovery Plan:", + "Store 7, creates: region 11, region 12, ...; updates: region 21, region 22, ... deletes: ... ", + "Store 8, ..." + "...", + "Execution Progress:", + "Store 10 finished,", + "Store 7 not yet finished", + "...", +] +``` + ## Jq formatted JSON output usage ### Simplify the output of `store` ```bash -» store --jq=".stores[].store | { id, address, state_name}" +>> store --jq=".stores[].store | { id, address, state_name}" {"id":1,"address":"127.0.0.1:20161","state_name":"Up"} {"id":30,"address":"127.0.0.1:20162","state_name":"Up"} ... @@ -894,7 +933,7 @@ logic: 120102 ### Query the remaining space of the node ```bash -» store --jq=".stores[] | {id: .store.id, available: .status.available}" +>> store --jq=".stores[] | {id: .store.id, available: .status.available}" {"id":1,"available":"10 GiB"} {"id":30,"available":"10 GiB"} ... @@ -905,7 +944,7 @@ logic: 120102 {{< copyable "" >}} ```bash -» store --jq='.stores[].store | select(.state_name!="Up") | { id, address, state_name}' +>> store --jq='.stores[].store | select(.state_name!="Up") | { id, address, state_name}' ``` ``` @@ -919,7 +958,7 @@ logic: 120102 {{< copyable "" >}} ```bash -» store --jq='.stores[].store | select(.labels | length>0 and contains([{"key":"engine","value":"tiflash"}])) | { id, address, state_name}' +>> store --jq='.stores[].store | select(.labels | length>0 and contains([{"key":"engine","value":"tiflash"}])) | { id, address, state_name}' ``` ``` @@ -931,7 +970,7 @@ logic: 120102 ### Query the distribution status of the Region replicas ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id]}" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id]}" {"id":2,"peer_stores":[1,30,31]} {"id":4,"peer_stores":[1,31,34]} ... @@ -942,7 +981,7 @@ logic: 120102 For example, to filter out all Regions whose number of replicas is not 3: ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length != 3)}" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length != 3)}" {"id":12,"peer_stores":[30,32]} {"id":2,"peer_stores":[1,30,31,32]} ``` @@ -952,7 +991,7 @@ For example, to filter out all Regions whose number of replicas is not 3: For example, to filter out all Regions that have a replica on store30: ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(any(.==30))}" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(any(.==30))}" {"id":6,"peer_stores":[1,30,31]} {"id":22,"peer_stores":[1,30,32]} ... @@ -961,7 +1000,7 @@ For example, to filter out all Regions that have a replica on store30: You can also find out all Regions that have a replica on store30 or store31 in the same way: ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(any(.==(30,31)))}" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(any(.==(30,31)))}" {"id":16,"peer_stores":[1,30,34]} {"id":28,"peer_stores":[1,30,32]} {"id":12,"peer_stores":[30,32]} @@ -973,7 +1012,7 @@ You can also find out all Regions that have a replica on store30 or store31 in t For example, when [store1, store30, store31] is unavailable at its downtime, you can find all Regions whose Down replicas are more than normal replicas: ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length as $total | map(if .==(1,30,31) then . else empty end) | length>=$total-length) }" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length as $total | map(if .==(1,30,31) then . else empty end) | length>=$total-length) }" {"id":2,"peer_stores":[1,30,31,32]} {"id":12,"peer_stores":[30,32]} {"id":14,"peer_stores":[1,30,32]} @@ -983,14 +1022,14 @@ For example, when [store1, store30, store31] is unavailable at its downtime, you Or when [store1, store30, store31] fails to start, you can find Regions where the data can be manually removed safely on store1. In this way, you can filter out all Regions that have a replica on store1 but don't have other DownPeers: ```bash -» region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length>1 and any(.==1) and all(.!=(30,31)))}" +>> region --jq=".regions[] | {id: .id, peer_stores: [.peers[].store_id] | select(length>1 and any(.==1) and all(.!=(30,31)))}" {"id":24,"peer_stores":[1,32,33]} ``` When [store30, store31] is down, find out all Regions that can be safely processed by creating the `remove-peer` Operator, that is, Regions with one and only DownPeer: ```bash -» region --jq=".regions[] | {id: .id, remove_peer: [.peers[].store_id] | select(length>1) | map(if .==(30,31) then . else empty end) | select(length==1)}" +>> region --jq=".regions[] | {id: .id, remove_peer: [.peers[].store_id] | select(length>1) | map(if .==(30,31) then . else empty end) | select(length==1)}" {"id":12,"remove_peer":[30]} {"id":4,"remove_peer":[31]} {"id":22,"remove_peer":[30]} diff --git a/pd-recover.md b/pd-recover.md index 2630f4a8679ce..e49b4c557045b 100644 --- a/pd-recover.md +++ b/pd-recover.md @@ -27,7 +27,7 @@ To download the latest version of PD Recover, directly download the TiDB package > **Note:** > -> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.2.1`, the package download link is `https://download.pingcap.org/tidb-v5.2.1-linux-amd64.tar.gz`. +> `{version}` indicates the version number of TiDB. For example, if `{version}` is `v5.3.0`, the package download link is `https://download.pingcap.org/tidb-v5.3.0-linux-amd64.tar.gz`. ## Quick Start @@ -97,7 +97,7 @@ To get the allocated ID from the PD log, you need to make sure that the log you {{< copyable "shell-regular" >}} ```bash -cat {{/path/to}}/pd*.log | grep "idAllocator allocates a new id" | awk -F'=' '{print $2}' | awk -F']' '{print $1}' | sort -r | head -n 1 +cat {{/path/to}}/pd*.log | grep "idAllocator allocates a new id" | awk -F'=' '{print $2}' | awk -F']' '{print $1}' | sort -r -n | head -n 1 ``` ```bash diff --git a/pessimistic-transaction.md b/pessimistic-transaction.md index a51438c7acf37..86434fd7c4ae9 100644 --- a/pessimistic-transaction.md +++ b/pessimistic-transaction.md @@ -58,6 +58,8 @@ Pessimistic transactions in TiDB behave similarly to those in MySQL. See the min - If the `Point Get` and `Batch Point Get` operators do not read data, they still lock the given primary key or unique key, which blocks other transactions from locking or writing data to the same primary key or unique key. +- TiDB supports the `FOR UPDATE OF TABLES` syntax. For a statement that joins multiple tables, TiDB only applies pessimistic locks on the rows that are associated with the tables in `OF TABLES`. + ## Difference with MySQL InnoDB 1. When TiDB executes DML or `SELECT FOR UPDATE` statements that use range in the WHERE clause, concurrent DML statements within the range are not blocked. diff --git a/placement-rules-in-sql.md b/placement-rules-in-sql.md new file mode 100644 index 0000000000000..6f5186dfd8c6f --- /dev/null +++ b/placement-rules-in-sql.md @@ -0,0 +1,179 @@ +--- +title: Placement Rules in SQL +summary: Learn how to schedule placement of tables and partitions using SQL statements. +--- + +# Placement Rules in SQL + +> **Warning:** +> +> Placement Rules in SQL is an experimental feature introduced in v5.3.0. The syntax might change before its GA, and there might also be bugs. If you understand the risks, you can enable this experiment feature by executing `SET GLOBAL tidb_enable_alter_placement = 1;`. + +Placement Rules in SQL is a feature that enables you to specify where data is stored in a TiKV cluster using SQL interfaces. Using this feature, tables and partitions are scheduled to specific regions, data centers, racks, or hosts. This is useful for scenarios including optimizing a high availability strategy with lower cost, ensuring that local replicas of data are available for local stale reads, and adhering to data locality requirements. + +The detailed user scenarios are as follows: + +- Merge multiple databases of different applications to reduce the cost on database maintenance +- Increase replica count for important data to improve the application availability and data reliability +- Store new data into SSDs and store old data into HHDs to lower the cost on data archiving and storage +- Schedule the leaders of hotspot data to high-performance TiKV instances +- Separate cold data to lower-cost storage mediums to improve cost efficiency + +## Specify placement options + +To use Placement Rules in SQL, you need to specify one or more placement options in a SQL statement. To specify the Placement options, you can either use _direct placement_ or use a _placement policy_. + +In the following example, both tables `t1` and `t2` have the same rules. `t1` is specified rules using a direct placement while `t2` is specified rules using a placement policy. + +```sql +CREATE TABLE t1 (a INT) PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1"; +CREATE PLACEMENT POLICY eastandwest PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1"; +CREATE TABLE t2 (a INT) PLACEMENT POLICY=eastandwest; +``` + +It is recommended to use placement policies for simpler rule management. When you change a placement policy (via [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md)), the change automatically propagates to all database objects. + +If you use direct placement options, you have to alter rules for each object (for example, tables and partitions). + +`PLACEMENT POLICY` is not associated with any database schema and has the global scope. Therefore, assigning a placement policy does not require any additional privileges over the `CREATE TABLE` privilege. + +## Option reference + +> **Note:** +> +> Placement options depend on labels correctly specified in the configuration of each TiKV node. For example, the `PRIMARY_REGION` option depends on the `region` label in TiKV. To see a summary of all labels available in your TiKV cluster, use the statement [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md): +> +> ```sql +> mysql> show placement labels; +> +--------+----------------+ +> | Key | Values | +> +--------+----------------+ +> | disk | ["ssd"] | +> | region | ["us-east-1"] | +> | zone | ["us-east-1a"] | +> +--------+----------------+ +> 3 rows in set (0.00 sec) +> ``` + +| Option Name | Description | +|----------------------------|------------------------------------------------------------------------------------------------| +| `PRIMARY_REGION` | Raft leaders are placed in stores that have the `region` label that matches the value of this option. | +| `REGIONS` | Raft followers are placed in stores that have the `region` label that matches the value of this option. | +| `SCHEDULE` | The strategy used to schedule the placement of followers. The value options are `EVEN` (default) or `MAJORITY_IN_PRIMARY`. | +| `FOLLOWERS` | The number of followers. For example, `FOLLOWERS=2` means that there will be 3 replicas of the data (2 followers and 1 leader). | + +In addition to the placement options above, you can also use the advance configurations. For details, see [Advance placement](#advanced-placement). + +| Option Name | Description | +| --------------| ------------ | +| `CONSTRAINTS` | A list of constraints that apply to all roles. For example, `CONSTRAINTS="[+disk=ssd]`. | +| `FOLLOWER_CONSTRAINTS` | A list of constraints that only apply to followers. | + +## Examples + +### Increase the number of replicas + +The default configuration of [`max-replicas`](/pd-configuration-file.md#max-replicas) is `3`. To increase this for a specific set of tables, you can use a placement policy as follows: + +```sql +CREATE PLACEMENT POLICY fivereplicas FOLLOWERS=4; +CREATE TABLE t1 (a INT) PLACEMENT POLICY=fivereplicas; +``` + +Note that the PD configuration includes the leader and follower count, thus 4 followers + 1 leader equals 5 replicas in total. + +To expand on this example, you can also use `PRIMARY_REGION` and `REGIONS` placement options to describe the placement for the followers: + +```sql +CREATE PLACEMENT POLICY eastandwest PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-east-2,us-west-1" SCHEDULE="MAJORITY_IN_PRIMARY" FOLLOWERS=4; +CREATE TABLE t1 (a INT) PLACEMENT POLICY=eastandwest; +``` + +The `SCHEDULE` option instructs TiDB on how to balance the followers. The default schedule of `EVEN` ensures a balance of followers in all regions. + +To ensure that enough followers are placed in the primary region (`us-east-1`) so that quorum can be achieved, you can use the `MAJORITY_IN_PRIMARY` schedule. This schedule helps provide lower latency transactions at the expense of some availability. If the primary region fails, `MAJORITY_IN_PRIMARY` cannot provide automatic failover. + +### Assign placement to a partitioned table + +> **Note:** +> +> The following example uses list partitioning, which is currently an experimental feature of TiDB. Partitioned tables also require the `PRIMARY KEY` to be included in all columns in the table's partitioning function. + +In addition to assigning placement options to tables, you can also assign the options to table partitions. For example: + +```sql +CREATE PLACEMENT POLICY europe PRIMARY_REGION="eu-central-1" REGIONS="eu-central-1,eu-west-1"; +CREATE PLACEMENT POLICY northamerica PRIMARY_REGION="us-east-1" REGIONS="us-east-1"; + +SET tidb_enable_list_partition = 1; +CREATE TABLE t1 ( + country VARCHAR(10) NOT NULL, + userdata VARCHAR(100) NOT NULL +) PARTITION BY LIST COLUMNS (country) ( + PARTITION pEurope VALUES IN ('DE', 'FR', 'GB') PLACEMENT POLICY=europe, + PARTITION pNorthAmerica VALUES IN ('US', 'CA', 'MX') PLACEMENT POLICY=northamerica +); +``` + +### Set the default placement for a schema + +You can directly attach the default placement options to a database schema. This works similar to setting the default character set or collation for a schema. Your specified placement options apply when no other options are specified. For example: + +```sql +CREATE TABLE t1 (a INT); -- Creates a table t1 with no placement options. + +ALTER DATABASE test FOLLOWERS=4; -- Changes the default placement option, and does not apply to the existing table t1. + +CREATE TABLE t2 (a INT); -- Creates a table t2 with the default placement of FOLLOWERS=4. + +CREATE TABLE t3 (a INT) PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-east-2"; -- Creates a table t3 without the default FOLLOWERS=4 placement, because this statement has specified another placement. + +ALTER DATABASE test FOLLOWERS=2; -- Changes the default placement, and does not apply to existing tables. + +CREATE TABLE t4 (a INT); -- Creates a table t4 with the default FOLLOWERS=2 option. +``` + +Because placement options are only inherited from the database schema when a table is created, it is recommended to set the default placement option using a `PLACEMENT POLICY`. This ensures that future changes to the policy propagate to existing tables. + +### Advanced placement + +The placement options `PRIMARY_REGION`, `REGIONS`, and `SCHEDULE` meet the basic needs of data placement at the loss of some flexibility. For more complex scenarios with the need for higher flexibility, you can also use the advanced placement options of `CONSTRAINTS` and `FOLLOWER_CONSTRAINTS`. You cannot specify the `PRIMARY_REGION`, `REGIONS`, or `SCHEDULE` option with the `CONSTRAINTS` option at the same time. If you specify both at the same time, an error will be returned. + +For example, to set constraints that data must reside on a TiKV store where the label `disk` must match a value: + +```sql +CREATE PLACEMENT POLICY storeonfastssd CONSTRAINTS="[+disk=ssd]"; +CREATE PLACEMENT POLICY storeonhdd CONSTRAINTS="[+disk=hdd]"; +CREATE PLACEMENT POLICY companystandardpolicy CONSTRAINTS=""; + +CREATE TABLE t1 (id INT, name VARCHAR(50), purchased DATE) +PLACEMENT POLICY=companystandardpolicy +PARTITION BY RANGE( YEAR(purchased) ) ( + PARTITION p0 VALUES LESS THAN (2000) PLACEMENT POLICY=storeonhdd, + PARTITION p1 VALUES LESS THAN (2005), + PARTITION p2 VALUES LESS THAN (2010), + PARTITION p3 VALUES LESS THAN (2015), + PARTITION p4 VALUES LESS THAN MAXVALUE PLACEMENT POLICY=storeonfastssd +); +``` + +You can either specify constraints in list format (`[+disk=ssd]`) or in dictionary format (`{+disk=ssd: 1,+disk=hdd: 2}`). + +In list format, constraints are specified as a list of key-value pairs. The key starts with either a `+` or a `-`. `+disk=ssd` indicates that the label `disk` must be set to `ssd`, and `-disk=hdd` indicates that the label `disk` must not be `hdd`. + +In dictionary format, constraints also indicate a number of instances that apply to that rule. For example, `FOLLOWER_CONSTRAINTS="{+region=us-east-1: 1,+region=us-east-2: 1,+region=us-west-1: 1,+any: 1}";` indicates that 1 follower is in us-east-1, 1 follower is in us-east-2, 1 follower is in us-west-1, and 1 follower can be in any region. For another example, `FOLLOWER_CONSTRAINTS='{"+region=us-east-1,+disk=hdd":1,"+region=us-west-1":1}';` indicates that 1 follower is in us-east-1 with an hdd disk, and 1 follower is in us-west-1. + +> **Note:** +> +> Dictionary and list formats are based on the YAML parser, but the YAML syntax might be incorrectly parsed. For example, `"{+disk=ssd:1,+disk=hdd:2}"` is incorrectly parsed as `'{"+disk=ssd:1": null, "+disk=hdd:1": null}'`. But `"{+disk=ssd: 1,+disk=hdd: 1}"` is correctly parsed as `'{"+disk=ssd": 1, "+disk=hdd": 1}'`. + +## Known limitations + +The following known limitations exist in the experimental release of Placement Rules in SQL: + +* Dumpling does not support dumping placement policies. See [issue #29371](https://github.com/pingcap/tidb/issues/29371). +* TiDB tools, including Backup & Restore (BR), TiCDC, TiDB Lightning, and TiDB Data Migration (DM), do not yet support placement rules. +* Temporary tables do not support placement options (either via direct placement or placement policies). +* Syntactic sugar rules are permitted for setting `PRIMARY_REGION` and `REGIONS`. In the future, we plan to add varieties for `PRIMARY_RACK`, `PRIMARY_ZONE`, and `PRIMARY_HOST`. See [issue #18030](https://github.com/pingcap/tidb/issues/18030). +* TiFlash learners are not configurable through Placement Rules syntax. +* Placement rules only ensure that data at rest resides on the correct TiKV store. The rules do not guarantee that data in transit (via either user queries or internal operations) only occurs in a specific region. diff --git a/production-deployment-using-tiup.md b/production-deployment-using-tiup.md index e4ed3589af37b..9decbc9bda912 100644 --- a/production-deployment-using-tiup.md +++ b/production-deployment-using-tiup.md @@ -137,12 +137,12 @@ To prepare the TiUP offline component package, manually pack an offline componen If you want to adjust an existing offline mirror (such as adding a new version of a component), take the following steps: - 1. When pulling an offline mirror, you can get an incomplete offline mirror by specifying specific information via parameters, such as the component and version information. For example, you can pull an offline mirror that includes only the offline mirror of TiUP v1.5.2 and TiUP Cluster v1.5.2 by running the following command: + 1. When pulling an offline mirror, you can get an incomplete offline mirror by specifying specific information via parameters, such as the component and version information. For example, you can pull an offline mirror that includes only the offline mirror of TiUP v1.7.0 and TiUP Cluster v1.7.0 by running the following command: {{< copyable "shell-regular" >}} ```bash - tiup mirror clone tiup-custom-mirror-v1.5.2 --tiup v1.5.2 --cluster v1.5.2 + tiup mirror clone tiup-custom-mirror-v1.7.0 --tiup v1.7.0 --cluster v1.7.0 ``` If you only need the components for a particular platform, you can specify them using the `--os` or `--arch` parameters. @@ -174,10 +174,10 @@ To prepare the TiUP offline component package, manually pack an offline componen {{< copyable "shell-regular" >}} ```bash - tiup mirror merge tiup-custom-mirror-v1.5.2 + tiup mirror merge tiup-custom-mirror-v1.7.0 ``` - - 5. When the above steps are completed, check the result by running the `tiup list` command. In this document's example, the outputs of both `tiup list tiup` and `tiup list cluster` show that the corresponding components of `v1.5.2` are available. + + 5. When the above steps are completed, check the result by running the `tiup list` command. In this document's example, the outputs of both `tiup list tiup` and `tiup list cluster` show that the corresponding components of `v1.7.0` are available. #### Step 2: Deploy the offline TiUP component @@ -306,13 +306,13 @@ Then execute the `deploy` command to deploy the TiDB cluster: {{< copyable "shell-regular" >}} ```shell -tiup cluster deploy tidb-test v5.2.1 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] +tiup cluster deploy tidb-test v5.3.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] ``` In the above command: - The name of the deployed TiDB cluster is `tidb-test`. -- You can see the latest supported versions by running `tiup list tidb`. This document takes `v5.2.1` as an example. +- You can see the latest supported versions by running `tiup list tidb`. This document takes `v5.3.0` as an example. - The initialization configuration file is `topology.yaml`. - `--user root`: Log in to the target machine through the `root` key to complete the cluster deployment, or you can use other users with `ssh` and `sudo` privileges to complete the deployment. - `[-i]` and `[-p]`: optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the `root` user (or other users specified by `--user`) that has access to the target machine. `[-p]` is used to input the user password interactively. @@ -334,7 +334,7 @@ TiUP supports managing multiple TiDB clusters. The command above outputs informa Starting /home/tidb/.tiup/components/cluster/v1.5.0/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- -tidb-test tidb v5.2.1 /home/tidb/.tiup/storage/cluster/clusters/tidb-test /home/tidb/.tiup/storage/cluster/clusters/tidb-test/ssh/id_rsa +tidb-test tidb v5.3.0 /home/tidb/.tiup/storage/cluster/clusters/tidb-test /home/tidb/.tiup/storage/cluster/clusters/tidb-test/ssh/id_rsa ``` ## Step 6: Check the status of the deployed TiDB cluster diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index ab5d4c134b534..7654f55953846 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -58,26 +58,26 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v5.2.1 --db 2 --pd 3 --kv 3 --monitor + tiup playground v5.3.0 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v5.2.1. `--monitor` means that the monitoring component is also deployed. - - To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v5.3.0. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: ```log CLUSTER START SUCCESSFULLY, Enjoy it ^-^ - To connect TiDB: mysql --host 127.0.0.1 --port 4000 -u root - To connect TiDB: mysql --host 127.0.0.1 --port 4001 -u root + To connect TiDB: mysql --comments --host 127.0.0.1 --port 4001 -u root -p (no password) + To connect TiDB: mysql --comments --host 127.0.0.1 --port 4000 -u root -p (no password) To view the dashboard: http://127.0.0.1:2379/dashboard - To view the monitor: http://127.0.0.1:9090 + PD client endpoints: [127.0.0.1:2379 127.0.0.1:2382 127.0.0.1:2384] + To view Prometheus: http://127.0.0.1:9090 + To view Grafana: http://127.0.0.1:3000 ``` > **Note:** > - > + Since v5.2.1, TiDB supports running `tiup playground` on the machine that uses the Apple M1 chip. + > + Since v5.2.0, TiDB supports running `tiup playground` on the machine that uses the Apple M1 chip. > + For the playground operated in this way, after the test deployment is finished, TiUP will clean up the original cluster data. You will get a new cluster after re-running the command. > + If you want the data to be persisted on storage,run `tiup --tag playground ...`. For details, refer to [TiUP Reference Guide](/tiup/tiup-reference.md#-t---tag). @@ -164,12 +164,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v5.2.1 --db 2 --pd 3 --kv 3 --monitor + tiup playground v5.3.0 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v5.2.1. `--monitor` means that the monitoring component is also deployed. - - To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v5.3.0. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: diff --git a/releases/release-4.0.15.md b/releases/release-4.0.15.md index 6d2b718b75166..f0f6e168116a1 100644 --- a/releases/release-4.0.15.md +++ b/releases/release-4.0.15.md @@ -19,7 +19,6 @@ TiDB version: 4.0.15 - Fix the wrong execution results that occur when the collations around the `between` expression are different [#27146](https://github.com/pingcap/tidb/issues/27146) - Fix the result wrong that occurs when the argument of the `extract` function is a negative duration [#27236](https://github.com/pingcap/tidb/issues/27236) - Fix the wrong execution results that occur when the column in the `group_concat` function has a non-bin collation [#27429](https://github.com/pingcap/tidb/issues/27429) - - Fix the issue of wrong character set and collation for the `case when` expression [#26662](https://github.com/pingcap/tidb/issues/26662) - Fix the issue that column information is missed when converting the `Apply` operator to `Join` [#27233](https://github.com/pingcap/tidb/issues/27233) - Fix the issue of unexpected behavior when casting the invalid string to `DATE` [#26762](https://github.com/pingcap/tidb/issues/26762) - Fix a bug that the `count distinct` result on multiple columns is wrong when the new collation is enabled [#27091](https://github.com/pingcap/tidb/issues/27091) @@ -91,6 +90,7 @@ TiDB version: 4.0.15 - Fix the "index out of range" error that occurs when a query includes both `GROUP BY` and `UNION` [#26553](https://github.com/pingcap/tidb/pull/26553) - Fix the issue that TiDB might fail to send requests if TiKV has tombstone stores [#23676](https://github.com/pingcap/tidb/issues/23676) [#24648](https://github.com/pingcap/tidb/issues/24648) - Remove the undocumented `/debug/sub-optimal-plan` HTTP API [#27264](https://github.com/pingcap/tidb/pull/27264) + - Fix the issue of wrong character set and collation for the `case when` expression [#26662](https://github.com/pingcap/tidb/issues/26662) + TiKV diff --git a/releases/release-4.0.9.md b/releases/release-4.0.9.md index 944f33dac6580..c8b2125e35901 100644 --- a/releases/release-4.0.9.md +++ b/releases/release-4.0.9.md @@ -156,6 +156,7 @@ TiDB version: 4.0.9 - Fix a bug that snapshot reads hits the lock cache [#21539](https://github.com/pingcap/tidb/pull/21539) - Fix an issue of potential memory leak after reading a lot of data in a long-lived transaction [#21129](https://github.com/pingcap/tidb/pull/21129) - Fix the issue that omitting the table alias in a subquery will have a syntax error returned [#20367](https://github.com/pingcap/tidb/pull/20367) + - Fix the issue that when the argument of the `IN` function in a query is the time type, the query might return an incorrect result [#21290](https://github.com/pingcap/tidb/issues/21290) + TiKV diff --git a/releases/release-5.0.0-rc.md b/releases/release-5.0.0-rc.md index 46beaf29bb81e..7ce26da1f60c0 100644 --- a/releases/release-5.0.0-rc.md +++ b/releases/release-5.0.0-rc.md @@ -138,7 +138,7 @@ Related issue: [#18005](https://github.com/pingcap/tidb/issues/18005) ### Other performance optimizations -+ Improve the execution performance of `delete * from table where id **Note:** +> +> When upgrading from an earlier TiDB version to v5.3.0, if you want to know the compatibility change notes of all intermediate versions, you can check the [Release Notes](/releases/release-notes.md) of the corresponding version. + +### System variables + +| Variable name | Change type | Description | +| :---------- | :----------- | :----------- | +| [`tidb_enable_noop_functions`](/system-variables.md#tidb_enable_noop_functions-new-in-v40) | Modified | Temporary tables are now supported by TiDB so `CREATE TEMPORARY TABLE` and `DROP TEMPORARY TABLE` no longer require enabling `tidb_enable_noop_functions`. | +| [`tidb_enable_pseudo_for_outdated_stats`](/system-variables.md#tidb_enable_pseudo_for_outdated_stats-new-in-v530) | Newly added | Controls the behavior of the optimizer when the statistics on a table expire. The default value is `ON`. When the number of modified rows in the table is greater than 80% of the total rows (This ratio can be adjusted by the configuration [`pseudo-estimate-ratio`](/tidb-configuration-file.md#pseudo-estimate-ratio)), the optimizer considers that the statistics other than the total number of rows are no longer reliable and use pseudo statistics instead. When you set the value as `OFF`, even if the statistics expire, the optimizer still uses them. | +|[`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v53) | Newly added | Determines whether to enable or disable the TSO Follower Proxy feature. The default value is `OFF`, which means the TSO Follower Proxy feature is disabled. At this time, TiDB only gets TSO from PD leader. When this feature is enabled, TiDB evenly sends the requests to all PD nodes when acquiring TSO. The PD follower then forwards the TSO requests to reduce the CPU pressure of PD leader. | +|[`tidb_tso_client_batch_max_wait_time`](/system-variables.md#tidb_tso_client_batch_max_wait_time-new-in-v53) | Newly added | Sets the maximum waiting time for a batch saving operation when TiDB requests TSO from PD. The default value is `0`, which means no additional waiting. | +| [`tidb_tmp_table_max_size`](/system-variables.md#tidb_tmp_table_max_size-new-in-v530) | Newly added | Limits the maximum size of a single [temporary table](/temporary-tables.md). If the temporary table exceeds this size, an error will occur. | + +### Configuration file parameters + +| Configuration file | Configuration item | Change type | Description | +| :---------- | :----------- | :----------- | :----------- | +| TiDB | [`prepared-plan-cache.capacity`](/tidb-configuration-file.md#capacity) | Modified | Controls the number of cached statements. The default value is changed from `100` to `1000`.| +| TiKV | [`storage.reserve-space`](/tikv-configuration-file.md#reserve-space) | Modified | Controls space reserved for disk protection when TiKV is started. Starting from v5.3.0, 80% of the reserved space is used as the extra disk space required for operations and maintenance when the disk space is insufficient, and the other 20% is used to store temporary files. | +| TiKV | `memory-usage-limit` | Modified | This configuration item is new in TiDB v5.3.0 and its value is calculated based on storage.block-cache.capacity. | +| TiKV | [`raftstore.store-io-pool-size`](/tikv-configuration-file.md#store-io-pool-size-new-in-v530) | Newly added | The allowable number of threads that process Raft I/O tasks, which is the size of the StoreWriter thread pool. When you modify the size of this thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). | +| TiKV | [`raftstore.raft-write-size-limit`](/tikv-configuration-file.md#raft-write-size-limit-new-in-v530) | Newly added | Determines the threshold at which Raft data is written into the disk. If the data size is larger than the value of this configuration item, the data is written to the disk. When the value of `raftstore.store-io-pool-size` is `0`, this configuration item does not take effect. | +| TiKV | [`raftstore.raft-msg-flush-interval`](/tikv-configuration-file.md#raft-msg-flush-interval-new-in-v530) | Newly added | Determines the interval at which Raft messages are sent in batches. The Raft messages in batches are sent at every interval specified by this configuration item. When the value of `raftstore.store-io-pool-size` is `0`, this configuration item does not take effect. | +| TiKV | `raftstore.raft-reject-transfer-leader-duration` | Deleted | Determines the smallest duration that a Leader is transferred to a newly added node. | +| PD | [`log.file.max-days`](/pd-configuration-file.md#max-days) | Modified | Controls the maximum number of days that logs are retained for. The default value is changed from `1` to `0`. | +| PD | [`log.file.max-backups`](/pd-configuration-file.md#max-backups) | Modified | Controls the maximum number of logs that are retained for. The default value is changed from `7` to `0`. | +| PD | [`patrol-region-interval`](/pd-configuration-file.md#patrol-region-interval) | Modified | Controls the running frequency at which replicaChecker checks the health state of a Region. The smaller this value is, the faster replicaChecker runs. Normally, you do not need to adjust this parameter. The default value is changed from `100ms` to `10ms`. | +| PD | [`max-snapshot-count`](/pd-configuration-file.md#max-snapshot-count) | Modified | Controls the maximum number of snapshots that a single store receives or sends at the same time. PD schedulers depend on this configuration to prevent the resources used for normal traffic from being preempted. The default value is changed from `3` to `64`. | +| PD | [`max-pending-peer-count`](/pd-configuration-file.md#max-pending-peer-count) | Modified | Controls the maximum number of pending peers in a single store. PD schedulers depend on this configuration to prevent too many Regions with outdated logs from being generated on some nodes. The default value is changed from `16` to `64`. | + +### Others + +- Temporary tables: + + - If you have created local temporary tables in a TiDB cluster earlier than v5.3.0, these tables are actually ordinary tables, and handled as ordinary tables after the cluster is upgraded to v5.3.0 or a later version. If you have created global temporary tables in a TiDB cluster of v5.3.0 or a later version, when the cluster is downgraded to a version earlier than v5.3.0, these tables are handled as ordinary tables and cause a data error. + - Since v5.3.0, TiCDC and BR support [global temporary tables](/temporary-tables.md#global-temporary-tables). If you use TiCDC and BR of a version earlier than v5.3.0 to replicate global temporary tables to the downstream, a table definition error occurs. + - The following clusters are expected to be v5.3.0 or later; otherwise, data error is reported when you create a global temporary table: + + - the cluster to be imported using TiDB ecosystem tools + - the cluster restored using TiDB ecosystem tools + - the downstream cluster in a replication task using TiDB ecosystem tools + - For the compatibility information of temporary tables, refer to [Compatibility with MySQL temporary tables](/temporary-tables.md#compatibility-with-mysql-temporary-tables) and [Compatibility restrictions with other TiDB features](/temporary-tables.md#compatibility-restrictions-with-other-tidb-features). + +- For releases earlier than v5.3.0, TiDB reports an error when a system variable is set to an illegal value. For v5.3.0 and later releases, TiDB returns success with a warning such as "|Warning | 1292 | Truncated incorrect xxx: 'xx'" when a system variable is set to an illegal value. +- Fix the issue that the `SHOW VIEW` permission is not required to execute `SHOW CREATE VIEW`. Now you are expected to have the `SHOW VIEW` permission to execute the `SHOW CREATE VIEW` statement. +- The system variable `sql_auto_is_null` is added to the noop functions. When `tidb_enable_noop_functions = 0/OFF`, modifying this variable value causes an error. +- The `GRANT ALL ON performance_schema.*` syntax is no longer permitted. If you execute this statement in TiDB, an error occurs. +- Fix the issue that auto-analyze is unexpectedly triggered outside the specified time period when new indexes are added before v5.3.0. In v5.3.0, after you set the time period through the `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time` variables, auto-analyze is triggered only during this time period. +- The default storage directory for plugins is changed from "" to /data/deploy/plugin. +- The DM code is migrated to [the folder "dm" in TiCDC code repository](https://github.com/pingcap/ticdc/tree/master/dm). Now DM follows TiDB in version numbers. Next to v2.0.x, the new DM version is v5.3.0, and you can upgrade from v2.0.x to v5.3.0 without any risk. + +## New features + +### SQL + +- **Use SQL interface to set placement rules for data (experimental feature)** + + Support the `[CREATE | ALTER] PLACEMENT POLICY` syntax that provides a SQL interface to set placement rules for data. Using this feature, you can specify tables and partitions to be scheduled to specific regions, data centers, racks, hosts, or replica count rules. This meets your application demands for lower cost and higher flexibility. The typical user scenarios are as follows: + + - Merge multiple databases of different applications to reduce the cost on database maintenance, and achieve application resource isolation through the rule configuration + - Increase replica count for important data to improve the application availability and data reliability + - Store new data into SSDs and store old data into HHDs to lower the cost on data archiving and storage + - Schedule the leaders of hotspot data to high-performance TiKV instances + - Separate cold data to lower-cost storage mediums to improve cost efficiency + + [User document](/placement-rules-in-sql.md), [#18030](https://github.com/pingcap/tidb/issues/18030) + +- **Temporary tables** + + Support the `CREATE [GLOBAL] TEMPORARY TABLE` statement to create temporary tables. Using this feature, you can easily manage the temporary data generated in the calculation process of an application. Temporary data is stored in memory and you can use the `tidb_tmp_table_max_size` variable to limit the size of a temporary table. TiDB supports the following types of temporary tables: + + - Global temporary tables + - Visible to all sessions in the cluster, and table schemas are persistent. + - Provides transaction-level data isolation. The temporary data is effective only in the transaction. After the transaction finishes, the data is automatically dropped. + - Local temporary tables + - Visible only to the current session, and tables schemas are not persistent. + - Supports duplicated table names. You do not need to design complicated naming rules for your application. + - Provides session-level data isolation, which enables you to design a simpler application logic. After the transaction finishes, the temporary tables are dropped. + + [User document](/temporary-tables.md), [#24169](https://github.com/pingcap/tidb/issues/24169) + +- **Support the `FOR UPDATE OF TABLES` syntax** + + For a SQL statement that joins multiple tables, TiDB supports acquiring pessimistic locks on the rows correlated to the tables that are included in `OF TABLES`. + + [User document](/sql-statements/sql-statement-select.md), [#28689](https://github.com/pingcap/tidb/issues/28689) + +- **Table attributes** + + Support the `ALTER TABLE [PARTITION] ATTRIBUTES` statement that allows you to set attributes for a table or partition. Currently, TiDB only supports setting the `merge_option` attribute. By adding this attribute, you can explicitly control the Region merge behavior. + + User scenarios: When you perform the `SPLIT TABLE` operation, if no data is inserted after a certain period of time, the empty Regions are automatically merged by default. In this case, you can set the table attribute to `merge_option=deny` to avoid the automatic merging of Regions. + + [User document](/table-attributes.md), [#3839](https://github.com/tikv/pd/issues/3839) + +### Security + +- **Support creating users with the least privileges on TiDB Dashboard** + +The account system of TiDB Dashboard is consistent with that of TiDB SQL. Users accessing TiDB Dashboard are authenticated and authorized based on TiDB SQL users' privileges. Therefore, TiDB Dashboard requires limited privileges, or merely the read-only privilege. You can configure users to access TiDB Dashboard based on the principle of least privilege, thus avoiding access of high-privileged users. + +It is recommended that you create a least-privileged SQL user to access and sign in with TiDB Dashboard. This avoids access of high-privileged users and improves security. + +[User document](/dashboard/dashboard-user.md) + +### Performance + +- **Optimize the timestamp processing flow of PD** + + TiDB optimizes its timestamp processing flow and reduces the timestamp processing load of PD by enabling PD Follower Proxy and modifying the batch waiting time required when the PD client requests TSO in batches. This helps improve the overall scalability of the system. + + - Support enabling or disabling PD Follower Proxy through the system variable [`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v53). Suppose that the TSO requests load of PD is too high. In this case, enabling PD follower proxy can batch forward the TSO requests collected during the request cycle on followers to the leader nodes. This solution can effectively reduce the number of direct interactions between clients and leaders, reduce the pressure of the load on leaders, and improve the overall performance of TiDB. + + > **Note:** + > + > When the number of clients is small and the PD leader CPU load is not full, it is NOT recommended to enable PD Follower Proxy. + + - Support using the system variable [`tidb_tso_client_batch_max_wait_time`](/system-variables.md#tidb_tso_client_batch_max_wait_time-new-in-v53) to set the maximum waiting time needed for the PD client to batch request TSO. The unit of this time is milliseconds. In case that PD has a high TSO requests load, you can reduce the load and improve the throughput by increasing the waiting time to get a larger batch size. + + > **Note:** + > + > When the TSO request load is not high, it is NOT recommended to modify this variable value. + + [User document](/system-variables.md#tidb_tso_client_batch_max_wait_time-new-in-v53), [#3149](https://github.com/tikv/pd/issues/3149) + +### Stability + +- **Support Online Unsafe Recovery after some stores are permanently damaged (experimental feature)** + + Support the `unsafe remove-failed-stores` command that performs online data unsafe recovery. Suppose that the majority of data replicas encounter issues like permanent damage (such as disk damage), and these issues cause the data ranges in an application to be unreadable or unwritable. In this case, you can use the Online Unsafe Recovery feature implemented in PD to recover the data, so that the data is readable or writable again. + + It is recommended to perform the feature-related operations with the support of the TiDB team. + +[User document](/online-unsafe-recovery.md), [#10483](https://github.com/tikv/tikv/issues/10483) + +### Data migration + +- **DM replication performance enhanced** + + Supports the following features to ensure lower-latency data replication from MySQL to TiDB: + + - Compact multiple updates on a single row into one statement + - Merge batch updates of multiple rows into one statement + +- **Add DM OpenAPI to better maintain DM clusters (experimental feature)** + + DM provides the OpenAPI feature for querying and operating the DM cluster. It is similar to the feature of [dmctl tools](https://docs.pingcap.com/zh/tidb-data-migration/stable/dmctl-introduction). + + Currently, DM OpenAPI is an experimental feature and disabled by default. It is not recommended to use it in a production environment. + + [User Document](https://docs.pingcap.com/zh/tidb-data-migration/stable/open-api) + +- **TiDB Lightning Parallel Import** + + TiDB Lightning provides parallel import capability to extend the original feature. It allows you to deploy multiple Lightning instances at the same time to import single tables or multiple tables to downstream TiDB in parallel. Without changing the way customers use it, it greatly improves the data migration ability, allowing you to migrate data in a more real-time way to further process, integrate and analyze them. It improves the efficiency of enterprise data management. + + In our test, using 10 TiDB Lightning instances, a total of 20 TiB MySQL data can be imported to TiDB within 8 hours. The performance of multiple table import is also improved. A single TiDB Lightning instance can support importing at 250 GB/s, and the overall migration is 8 times faster than the original performance. + + [User Document](/tidb-lightning/tidb-lightning-distributed-import.md) + +- **TiDB Lightning Prechecks** + + TiDB Lightning provides the ability to check the configuration before running a migration task. It is enabled by default. This feature automatically performs some routine checks for disk space and execution configuration. The main purpose is to ensure that the whole subsequent import process goes smoothly. + + [User Document](/tidb-lightning/tidb-lightning-prechecks.md) + +- **TiDB Lightning supports importing files of GBK character set** + + You can specify the character set of the source data file. TiDB Lightning will convert the source file from the specified character set to UTF-8 encoding during the import process. + + [User Document](/tidb-lightning/tidb-lightning-configuration.md) + +- **Sync-diff-inspector improvement** + + - Improve the comparison speed from 375 MB/s to 700 MB/s + - Reduce the memory consumption of TiDB nodes by nearly half during comparison + - Optimize the user interface and display the progress bar during comparison + + [User Document](/sync-diff-inspector/sync-diff-inspector-overview.md) + +### Diagnostic efficiency + +- **Save and restore the on-site information of a cluster** + + When you locate and troubleshoot the issues of a TiDB cluster, you often need to provide information on the system and the query plan. To help you get the information and troubleshoot cluster issues in a more convenient and efficient way, the `PLAN REPLAY` command is introduced in TiDB v5.3.0. This command enables you to easily save and restore the on-site information of a cluster, improves the efficiency of troubleshooting, and helps you more easily archive the issues for management. + + The features of `PLAN REPLAYER` are as follows: + + - Exports the information of a TiDB cluster at an on-site troubleshooting to a ZIP-formatted file for storage. + - Imports into a cluster the ZIP-formatted file exported from another TiDB cluster. This file contains the information of the latter TiDB cluster at an on-site troubleshooting. + + [User document](/sql-plan-replayer.md), [#26325](https://github.com/pingcap/tidb/issues/26325) + +### TiDB data share subscription + +- **TiCDC Eventually Consistent Replication** + + TiCDC provides the eventually consistent replication capability in disaster scenarios. When a disaster occurs in the primary TiDB cluster and the service cannot be resumed in a short period of time, TiCDC needs to provide the ability to ensure the consistency of data in the secondary cluster. Meanwhile, TiCDC needs to allow the business to quickly switch the traffic to the secondary cluster to avoid the database being unavailable for a long time and affecting the business. + + This feature supports TiCDC to replicate incremental data from a TiDB cluster to the secondary relational database TiDB/Aurora/MySQL/MariaDB. In case the primary cluster crashes, TiCDC can recover the secondary cluster to a certain snapshot in the primary cluster within 5 minutes, given the condition that before disaster the replication status of TiCDC is normal and replication lag is small. It allows data loss of less than 30 minutes, that is, RTO <= 5min, and RPO <= 30min. + + [User Document](/ticdc/manage-ticdc.md) + +- **TiCDC supports the HTTP protocol OpenAPI for managing TiCDC tasks** + + Since TiDB v5.3.0, TiCDC OpenAPI becomes an General Availability (GA) feature. You can query and operate TiCDC clusters using OpenAPI in the production environment. + +### Deployment and maintenance + +- **Continuous Profiling (experimental feature)** + + TiDB Dashboard supports the Continuous Profiling feature, which stores instance performance analysis results automatically in real time when TiDB clusters are running. You can check the performance analysis result in a flame graph, which is more observable and shortens troubleshooting time. + + This feature is disabled by default and needs to be enabled on the **Continuous Profile** page of TiDB Dashboard. + + This feature is only available for clusters upgraded or installed using TiUP v1.7.0 or above. + + [User document](/dashboard/continuous-profiling.md) + +## Telemetry + +TiDB adds the information to the telemetry report about whether or not the TEMPORARY TABLE feature is used. This does not include table names or table data. + +To learn more about telemetry and how to disable this behavior, refer to [Telemetry](/telemetry.md). + +## Removed feature + +Starting from TiCDC v5.3.0, the cyclic replication feature between TiDB clusters (an experimental feature in v5.0.0) has been removed. If you have already used this feature to replicate data before upgrading TiCDC, the related data is not affected after the upgrade. + +## Improvements + ++ TiDB + + - Show the affected SQL statements in the debug log when the coprocessor encounters a lock, which is helpful in diagnosing problems [#27718](https://github.com/pingcap/tidb/issues/27718) + - Support showing the size of the backup and restore data when backing up and restoring data in the SQL logical layer [#27247](https://github.com/pingcap/tidb/issues/27247) + - Improve the default collection logic of ANALYZE when `tidb_analyze_version` is `2`, which accelerates collection and reduces resource overhead + - Introduce the `ANALYZE TABLE table_name COLUMNS col_1, col_2, ... , col_n` syntax. The syntax allows collecting statistics only on a portion of the columns in wide tables, which improves the speed of statistics collection + ++ TiKV + + - Enhance disk space protection to improve storage stability + + To solve the issue that TiKV might panic in case of a disk fully-written error, TiKV introduces a two-level threshold defense mechanism to protect the disk remaining space from being exhausted by excess traffic. Additionally, the mechanism provides the ability to reclaim space when the threshold is triggered. When the remaining space threshold is triggered, some write operations will fail and TiKV will return a disk full error as well as a list of disk full nodes. In this case, to recover the space and restore the service, you can execute `Drop/Truncate Table` or scale out the nodes. + + - Simplify the algorithm of L0 flow control [#10879](https://github.com/tikv/tikv/issues/10879) + - Improve the error log report in the raft client module [#10944](https://github.com/tikv/tikv/pull/10944) + - Improve logging threads to avoid them becoming a performance bottleneck [#10841](https://github.com/tikv/tikv/issues/10841) + - Add more statistics types of write queries [#10507](https://github.com/tikv/tikv/issues/10507) + ++ PD + + - Add more types of write queries to QPS dimensions in the hotspot scheduler [#3869](https://github.com/tikv/pd/issues/3869) + - Support dynamically adjusting the retry limit of the balance region scheduler to improve the performance of the scheduler [#3744](https://github.com/tikv/pd/issues/3744) + - Update TiDB Dashboard to v2021.10.08.1 [#4070](https://github.com/tikv/pd/pull/4070) + - Support that the evict leader scheduler can schedule regions with unhealthy peers [#4093](https://github.com/tikv/pd/issues/4093) + - Speed up the exit process of schedulers [#4146](https://github.com/tikv/pd/issues/4146) + ++ TiFlash + + - Improve the execution efficiency of the TableScan operator greatly + - Improve the execution efficiency of the Exchange operator + - Reduce write amplification and memory usage during GC of the storage engine (experimental feature) + - Improve the stability and availability of TiFlash when TiFlash restarts, which reduces possible query failures following the restart + - Support pushing down multiple new String and Time functions to the MPP engine + + - String functions: LIKE pattern, FORMAT(), LOWER(), LTRIM(), RTRIM(), SUBSTRING_INDEX(), TRIM(), UCASE(), UPPER() + - Mathematical functions: ROUND (decimal, int) + - Date and time functions: HOUR(), MICROSECOND(), MINUTE(), SECOND(), SYSDATE() + - Type conversion function: CAST(time, real) + - Aggregation functions: GROUP_CONCAT(), SUM(enum) + + - Support 512-bit SIMD + - Enhance the cleanup algorithm for outdated data to reduce disk usage and read files more efficiently + - Fix the issue that dashboard does not display memory or CPU information in some non-Linux systems + - Unify the naming style of TiFlash log files (keep the naming style consistent with that of TiKV) and support dynamic modification of logger.count and logger.size + - Improve the data validation capability of column-based files (checksums, experimental feature) + ++ Tools + + + TiCDC + + - Reduce the default value of the Kafka sink configuration item `MaxMessageBytes` from 64 MB to 1 MB to fix the issue that large messages are rejected by the Kafka Broker [#3104](https://github.com/pingcap/ticdc/pull/3104) + - Reduce memory usage in the replication pipeline [#2553](https://github.com/pingcap/ticdc/issues/2553)[#3037](https://github.com/pingcap/ticdc/pull/3037) [#2726](https://github.com/pingcap/ticdc/pull/2726) + - Optimize monitoring items and alert rules to improve observability of synchronous links, memory GC, and stock data scanning processes [#2735](https://github.com/pingcap/ticdc/pull/2735) [#1606](https://github.com/pingcap/ticdc/issues/1606) [#3000](https://github.com/pingcap/ticdc/pull/3000) [#2985](https://github.com/pingcap/ticdc/issues/2985) [#2156](https://github.com/pingcap/ticdc/issues/2156) + - When the sync task status is normal, no more historical error messages are displayed to avoid misleading users [#2242](https://github.com/pingcap/ticdc/issues/2242) + +## Bug Fixes + ++ TiDB + + - Fix an error that occurs during execution caused by the wrong execution plan. The wrong execution plan is caused by the shallow copy of schema columns when pushing down the aggregation operators on partitioned tables. [#27797](https://github.com/pingcap/tidb/issues/27797) [#26554](https://github.com/pingcap/tidb/issues/26554) + - Fix the issue that plan-cache cannot detect changes of unsigned flags [#28254](https://github.com/pingcap/tidb/issues/28254) + - Fix the wrong partition pruning when the partition function is out of range [#28233](https://github.com/pingcap/tidb/issues/28233) + - Fix the issue that planner might cache invalid plans for `join` in some cases [#28087](https://github.com/pingcap/tidb/issues/28087) + - Fix wrong index hash join when hash column type is enum [#27893](https://github.com/pingcap/tidb/issues/27893) + - Fix a batch client bug that recycling idle connection might block sending requests in some rare cases [#27688](https://github.com/pingcap/tidb/pull/27688) + - Fix the TiDB Lightning panic issue when it fails to perform checksum on a target cluster [#27686](https://github.com/pingcap/tidb/pull/27686) + - Fix wrong results of the `date_add` and `date_sub` functions in some cases [#27232](https://github.com/pingcap/tidb/issues/27232) + - Fix wrong results of the `hour` function in vectorized expression [#28643](https://github.com/pingcap/tidb/issues/28643) + - Fix the authenticating issue when connecting to MySQL 5.1 or an older client version [#27855](https://github.com/pingcap/tidb/issues/27855) + - Fix the issue that auto analyze might be triggered out of the specified time when a new index is added [#28698](https://github.com/pingcap/tidb/issues/28698) + - Fix a bug that setting any session variable invalidates `tidb_snapshot` [#28683](https://github.com/pingcap/tidb/pull/28683) + - Fix a bug that BR is not working for clusters with many missing-peer regions [#27534](https://github.com/pingcap/tidb/issues/27534) + - Fix the unexpected error like `tidb_cast to Int32 is not supported` when the unsupported `cast` is pushed down to TiFlash [#23907](https://github.com/pingcap/tidb/issues/23907) + - Fix the issue that `DECIMAL overflow` is missing in the `%s value is out of range in '%s'`error message [#27964](https://github.com/pingcap/tidb/issues/27964) + - Fix a bug that the availability detection of MPP node does not work in some corner cases [#3118](https://github.com/pingcap/tics/issues/3118) + - Fix the `DATA RACE` issue when assigning `MPP task ID` [#27952](https://github.com/pingcap/tidb/issues/27952) + - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table`. [#28250](https://github.com/pingcap/tidb/issues/28250) + - Fix the issue of false positive error log `invalid cop task execution summaries length` for MPP queries [#1791](https://github.com/pingcap/tics/issues/1791) + - Fix the issue of error log `cannot found column in Schema column` for MPP queries [#28149](https://github.com/pingcap/tidb/pull/28149) + - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Remove the support for insecure 3DES (Triple Data Encryption Algorithm) based TLS cipher suites [#27859](https://github.com/pingcap/tidb/pull/27859) + - Fix the issue that Lightning connects to offline TiKV nodes during pre-check and causes import failures [#27826](https://github.com/pingcap/tidb/pull/27826) + - Fix the issue that pre-check cost too much time when importing many files to tables [#27605](https://github.com/pingcap/tidb/issues/27605) + - Fix the issue that rewriting expressions makes `between` infer wrong collation [#27146](https://github.com/pingcap/tidb/issues/27146) + - Fix the issue that `group_concat` function did not consider the collation [#27429](https://github.com/pingcap/tidb/issues/27429) + - Fix the result wrong that occurs when the argument of the `extract` function is a negative duration [#27236](https://github.com/pingcap/tidb/issues/27236) + - Fix the issue that creating partition fails if `NO_UNSIGNED_SUBTRACTION` is set [#26765](https://github.com/pingcap/tidb/issues/26765) + - Avoid expressions with side effects in column pruning and aggregation pushdown [#27106](https://github.com/pingcap/tidb/issues/27106) + - Remove useless gRPC logs [#24190](https://github.com/pingcap/tidb/issues/24190) + - Limit the valid decimal length to fix precision-related issues [#3091](https://github.com/pingcap/tics/issues/3091) + - Fix the issue of a wrong way to check for overflow in `plus` expression [#26977](https://github.com/pingcap/tidb/issues/26977) + - Fix the issue of `data too long` error when dumping statistics from the table with `new collation` data [#27024](https://github.com/pingcap/tidb/issues/27024) + - Fix the issue that the retried transactions' statements are not included in `TIDB_TRX` [#28670](https://github.com/pingcap/tidb/pull/28670) + - Fix the wrong default value of the `plugin_dir` configuration [28084](https://github.com/pingcap/tidb/issues/28084) + ++ TiKV + + - Fix the issue of unavailable TiKV caused by Raftstore deadlock when migrating Regions. The workaround is to disable the scheduling and restart the unavailable TiKV. [#10909](https://github.com/tikv/tikv/issues/10909) + - Fix the issue that CDC adds scan retries frequently due to the Congest error [#11082](https://github.com/tikv/tikv/issues/11082) + - Fix the issue that the Raft connection is broken when the channel is full [#11047](https://github.com/tikv/tikv/issues/11047) + - Fix the issue that batch messages are too large in Raft client implementation [#9714](https://github.com/tikv/tikv/issues/9714) + - Fix the issue that some coroutines leak in `resolved_ts` [#10965](https://github.com/tikv/tikv/issues/10965) + - Fix a panic issue that occurs to the coprocessor when the size of response exceeds 4 GiB [#9012](https://github.com/tikv/tikv/issues/9012) + - Fix the issue that snapshot Garbage Collection (GC) misses GC snapshot files when snapshot files cannot be garbage collected [#10813](https://github.com/tikv/tikv/issues/10813) + - Fix a panic issue caused by timeout when processing Coprocessor requests [#10852](https://github.com/tikv/tikv/issues/10852) + - Fix a memory leak caused by monitoring data of statistics threads [#11195](https://github.com/tikv/tikv/issues/11195) + - Fix a panic issue caused by getting the cgroup information from some platforms [#10980](https://github.com/tikv/tikv/pull/10980) + - Fix the issue of poor scan performance because MVCC Deletion versions are not dropped by compaction filter GC [#11248](https://github.com/tikv/tikv/pull/11248) + ++ PD + + - Fix the issue that PD incorrectly delete the peers with data and in pending status because the number of peers exceeds the number of configured peers [#4045](https://github.com/tikv/pd/issues/4045) + - Fix the issue that PD does not fix down peers in time [#4077](https://github.com/tikv/pd/issues/4077) + - Fix the issue that the scatter range scheduler cannot schedule empty regions [#4118](https://github.com/tikv/pd/pull/4118) + - Fix the issue that the key manager cost too much CPU [#4071](https://github.com/tikv/pd/issues/4071) + - Fix the data race issue that might occur when setting configurations of hot region scheduler [#4159](https://github.com/tikv/pd/issues/4159) + - Fix slow leader election caused by stucked region syncer[#3936](https://github.com/tikv/pd/issues/3936) + ++ TiFlash + + - Fix the issue of inaccurate TiFlash Store Size statistics + - Fix the issue that TiFlash fails to start up on some platforms due to the absence of library `nsl` + - Block the infinite wait of `wait index` when writing pressure is heavy (a default timeout of 5 minutes is added), which prevents TiFlash from waiting too long for data replication to provide services + - Fix the slow and no result issues of the log search when the log volume is large + - Fix the issue that only the most recent logs can be searched when searching old historical logs + - Fix the possible wrong result when a new collation is enabled + - Fix the possible parsing errors when an SQL statement contains extremely long nested expressions + - Fix the possible `Block schema mismatch` error of the Exchange operator + - Fix the possible `Can't compare` error when comparing Decimal types + - Fix the `3rd arguments of function substringUTF8 must be constants` error of the `left/substring` function + ++ Tools + + + TiCDC + + - Fix the issue that TiCDC replication task might terminate when the upstream TiDB instance unexpectedly exits [#3061](https://github.com/pingcap/ticdc/issues/3061) + - Fix the issue that TiCDC process might panic when TiKV sends duplicate requests to the same Region [#2386](https://github.com/pingcap/ticdc/issues/2386) + - Fix unnecessary CPU consumption when verifying downstream TiDB/MySQL availability [#3073](https://github.com/pingcap/ticdc/issues/3073) + - Fix the issue that the volume of Kafka messages generated by TiCDC is not constrained by `max-message-size` [#2962](https://github.com/pingcap/ticdc/issues/2962) + - Fix the issue that TiCDC sync task might pause when an error occurs during writing a Kafka message [#2978](https://github.com/pingcap/ticdc/issues/2978) + - Fix the issue that some partitioned tables without valid indexes might be ignored when `force-replicate` is enabled [#2834](https://github.com/pingcap/ticdc/issues/2834) + - Fix the issue that scanning stock data might fail due to TiKV performing GC when scanning stock data takes too long [#2470](https://github.com/pingcap/ticdc/issues/2470) + - Fix a possible panic issue when encoding some types of columns into Open Protocol format [#2758](https://github.com/pingcap/ticdc/issues/2758) + - Fix a possible panic issue when encoding some types of columns into Avro format [#2648](https://github.com/pingcap/ticdc/issues/2648) + + + TiDB Binlog + + - Fix the issue that when most tables are filtered out, checkpoint can not be updated under some special load [#1075](https://github.com/pingcap/tidb-binlog/pull/1075) diff --git a/releases/release-notes.md b/releases/release-notes.md index b374e82c5989b..85eaefb20bd41 100644 --- a/releases/release-notes.md +++ b/releases/release-notes.md @@ -5,8 +5,14 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] # TiDB Release Notes +## 5.3 + +- [5.3.0](/releases/release-5.3.0.md) + ## 5.2 +- [5.2.3](/releases/release-5.2.3.md) +- [5.2.2](/releases/release-5.2.2.md) - [5.2.1](/releases/release-5.2.1.md) - [5.2.0](/releases/release-5.2.0.md) @@ -131,7 +137,7 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] - [2.0.3](/releases/release-2.0.3.md) - [2.0.2](/releases/release-2.0.2.md) - [2.0.1](/releases/release-2.0.1.md) -- [2.0](/releases/release-2.0-ga.md) +- [2.0 GA](/releases/release-2.0-ga.md) - [2.0 RC5](/releases/release-2.0-rc.5.md) - [2.0 RC4](/releases/release-2.0-rc.4.md) - [2.0 RC3](/releases/release-2.0-rc.3.md) @@ -149,7 +155,7 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] - [1.0.3](/releases/release-1.0.3.md) - [1.0.2](/releases/release-1.0.2.md) - [1.0.1](/releases/release-1.0.1.md) -- [1.0](/releases/release-1.0-ga.md) +- [1.0 GA](/releases/release-1.0-ga.md) - [Pre-GA](/releases/release-pre-ga.md) - [RC4](/releases/release-rc.4.md) - [RC3](/releases/release-rc.3.md) diff --git a/releases/release-rc.2.md b/releases/release-rc.2.md index b7bdbfac95a4b..6a62d4d86c678 100644 --- a/releases/release-rc.2.md +++ b/releases/release-rc.2.md @@ -5,7 +5,7 @@ aliases: ['/docs/dev/releases/release-rc.2/','/docs/dev/releases/rc2/'] # TiDB RC2 Release Notes -On August 4, 2017, TiDB RC4 is released! This release is focused on the compatibility with MySQL, SQL query optimizer, system stability and performance in this version. What’s more, a new permission management mechanism is added and users can control data access in the same way as the MySQL privilege management system. +On March 1, 2017, TiDB RC2 is released! This release is focused on the compatibility with MySQL, SQL query optimizer, system stability and performance in this version. What’s more, a new permission management mechanism is added and users can control data access in the same way as the MySQL privilege management system. ## TiDB diff --git a/releases/release-rc.3.md b/releases/release-rc.3.md index d2542066fdbfa..c63980ff53e5d 100644 --- a/releases/release-rc.3.md +++ b/releases/release-rc.3.md @@ -5,14 +5,14 @@ aliases: ['/docs/dev/releases/release-rc.3/','/docs/dev/releases/rc3/'] # TiDB RC3 Release Notes -On June 20, 2017, TiDB RC4 is released!This release is focused on MySQL compatibility, SQL optimization, stability, and performance. +On June 16, 2017, TiDB RC3 is released! This release is focused on MySQL compatibility, SQL optimization, stability, and performance. ## Highlight - The privilege management is refined to enable users to manage the data access privileges using the same way as in MySQL. - DDL is accelerated. - The load balancing policy and process are optimized for performance. -- TiDB Ansible is open sourced. By using TiDB-Ansilbe, you can deploy, upgrade, start and shutdown a TiDB cluster with one click. +- TiDB Ansible is open sourced. By using TiDB-Ansible, you can deploy, upgrade, start and shutdown a TiDB cluster with one click. ## Detailed updates diff --git a/releases/release-timeline.md b/releases/release-timeline.md new file mode 100644 index 0000000000000..1d2d7b4db0016 --- /dev/null +++ b/releases/release-timeline.md @@ -0,0 +1,138 @@ +--- +title: TiDB Release Timeline +summary: Learn about the TiDB release timeline. +--- + +# TiDB Release Timeline + +This document shows all the released TiDB versions in reverse chronological order. + +| Version | Release Date | +| :--- | :--- | +| [5.2.3](/releases/release-5.2.3.md) | 2021-12-03 | +| [5.3.0](/releases/release-5.3.0.md) | 2021-11-30 | +| [5.2.2](/releases/release-5.2.2.md) | 2021-10-29 | +| [5.1.2](/releases/release-5.1.2.md) | 2021-09-27 | +| [5.0.4](/releases/release-5.0.4.md) | 2021-09-27 | +| [4.0.15](/releases/release-4.0.15.md) | 2021-09-27 | +| [5.2.1](/releases/release-5.2.1.md) | 2021-09-09 | +| [5.2.0](/releases/release-5.2.0.md) | 2021-08-27 | +| [5.1.1](/releases/release-5.1.1.md) | 2021-07-30 | +| [4.0.14](/releases/release-4.0.14.md) | 2021-07-27 | +| [5.0.3](/releases/release-5.0.3.md) | 2021-07-02 | +| [5.1.0](/releases/release-5.1.0.md) | 2021-06-24 | +| [5.0.2](/releases/release-5.0.2.md) | 2021-06-10 | +| [4.0.13](/releases/release-4.0.13.md) | 2021-05-28 | +| [5.0.1](/releases/release-5.0.1.md) | 2021-04-24 | +| [5.0.0](/releases/release-5.0.0.md) | 2021-04-07 | +| [4.0.12](/releases/release-4.0.12.md) | 2021-04-02 | +| [4.0.11](/releases/release-4.0.11.md) | 2021-02-26 | +| [4.0.10](/releases/release-4.0.10.md) | 2021-01-15 | +| [5.0.0-rc](/releases/release-5.0.0-rc.md) | 2021-01-12 | +| [3.0.20](/releases/release-3.0.20.md) | 2020-12-25 | +| [4.0.9](/releases/release-4.0.9.md) | 2020-12-21 | +| [4.0.8](/releases/release-4.0.8.md) | 2020-10-30 | +| [4.0.7](/releases/release-4.0.7.md) | 2020-09-29 | +| [3.0.19](/releases/release-3.0.19.md) | 2020-09-25 | +| [4.0.6](/releases/release-4.0.6.md) | 2020-09-15 | +| [4.0.5](/releases/release-4.0.5.md) | 2020-08-31 | +| [3.0.18](/releases/release-3.0.18.md) | 2020-08-21 | +| [3.0.17](/releases/release-3.0.17.md) | 2020-08-03 | +| [4.0.4](/releases/release-4.0.4.md) | 2020-07-31 | +| [4.0.3](/releases/release-4.0.3.md) | 2020-07-24 | +| [3.0.16](/releases/release-3.0.16.md) | 2020-07-03 | +| [4.0.2](/releases/release-4.0.2.md) | 2020-07-01 | +| [4.0.1](/releases/release-4.0.1.md) | 2020-06-12 | +| [3.0.15](/releases/release-3.0.15.md) | 2020-06-05 | +| [3.1.2](/releases/release-3.1.2.md) | 2020-06-04 | +| [4.0.0](/releases/release-4.0-ga.md) | 2020-05-28 | +| [4.0.0-rc.2](/releases/release-4.0.0-rc.2.md) | 2020-05-15 | +| [3.0.14](/releases/release-3.0.14.md) | 2020-05-09 | +| [3.1.1](/releases/release-3.1.1.md) | 2020-04-30 | +| [4.0.0-rc.1](/releases/release-4.0.0-rc.1.md) | 2020-04-28 | +| [3.0.13](/releases/release-3.0.13.md) | 2020-04-22 | +| [3.1.0](/releases/release-3.1.0-ga.md) | 2020-04-16 | +| [4.0.0-rc](/releases/release-4.0.0-rc.md) | 2020-04-08 | +| [3.1.0-rc](/releases/release-3.1.0-rc.md) | 2020-04-02 | +| [4.0.0-beta.2](/releases/release-4.0.0-beta.2.md) | 2020-03-18 | +| [3.0.12](/releases/release-3.0.12.md) | 2020-03-16 | +| [3.1.0-beta.2](/releases/release-3.1.0-beta.2.md) | 2020-03-09 | +| [3.0.11](/releases/release-3.0.11.md) | 2020-03-04 | +| [4.0.0-beta.1](/releases/release-4.0.0-beta.1.md) | 2020-02-28 | +| [3.0.10](/releases/release-3.0.10.md) | 2020-02-20 | +| [4.0.0-beta](/releases/release-4.0.0-beta.md) | 2020-01-17 | +| [3.0.9](/releases/release-3.0.9.md) | 2020-01-14 | +| [3.1.0-beta.1](/releases/release-3.1.0-beta.1.md) | 2020-01-10 | +| [3.0.8](/releases/release-3.0.8.md) | 2019-12-31 | +| [2.1.19](/releases/release-2.1.19.md) | 2019-12-27 | +| [3.1.0-beta](/releases/release-3.1.0-beta.md) | 2019-12-20 | +| [3.0.7](/releases/release-3.0.7.md) | 2019-12-04 | +| [3.0.6](/releases/release-3.0.6.md) | 2019-11-28 | +| [2.1.18](/releases/release-2.1.18.md) | 2019-11-04 | +| [3.0.5](/releases/release-3.0.5.md) | 2019-10-25 | +| [3.0.4](/releases/release-3.0.4.md) | 2019-10-08 | +| [2.1.17](/releases/release-2.1.17.md) | 2019-09-11 | +| [3.0.3](/releases/release-3.0.3.md) | 2019-08-29 | +| [2.1.16](/releases/release-2.1.16.md) | 2019-08-15 | +| [3.0.2](/releases/release-3.0.2.md) | 2019-08-07 | +| [2.1.15](/releases/release-2.1.15.md) | 2019-07-18 | +| [3.0.1](/releases/release-3.0.1.md) | 2019-07-16 | +| [2.1.14](/releases/release-2.1.14.md) | 2019-07-04 | +| [3.0.0](/releases/release-3.0-ga.md) | 2019-06-28 | +| [3.0.0-rc.3](/releases/release-3.0.0-rc.3.md) | 2019-06-21 | +| [2.1.13](/releases/release-2.1.13.md) | 2019-06-21 | +| [2.1.12](/releases/release-2.1.12.md) | 2019-06-13 | +| [2.1.11](/releases/release-2.1.11.md) | 2019-06-03 | +| [3.0.0-rc.2](/releases/release-3.0.0-rc.2.md) | 2019-05-28 | +| [2.1.10](/releases/release-2.1.10.md) | 2019-05-22 | +| [3.0.0-rc.1](/releases/release-3.0.0-rc.1.md) | 2019-05-10 | +| [2.1.9](/releases/release-2.1.9.md) | 2019-05-06 | +| [2.1.8](/releases/release-2.1.8.md) | 2019-04-12 | +| [2.1.7](/releases/release-2.1.7.md) | 2019-03-28 | +| [3.0.0-beta.1](/releases/release-3.0.0-beta.1.md) | 2019-03-26 | +| [2.1.6](/releases/release-2.1.6.md) | 2019-03-15 | +| [2.1.5](/releases/release-2.1.5.md) | 2019-02-28 | +| [2.1.4](/releases/release-2.1.4.md) | 2019-02-15 | +| [2.1.3](/releases/release-2.1.3.md) | 2019-01-28 | +| [3.0.0-beta](/releases/release-3.0-beta.md) | 2019-01-19 | +| [2.0.11](/releases/release-2.0.11.md) | 2019-01-03 | +| [2.1.2](/releases/release-2.1.2.md) | 2018-12-22 | +| [2.0.10](/releases/release-2.0.10.md) | 2018-12-18 | +| [2.1.1](/releases/release-2.1.1.md) | 2018-12-12 | +| [2.1.0](/releases/release-2.1-ga.md) | 2018-11-30 | +| [2.0.9](/releases/release-2.0.9.md) | 2018-11-19 | +| [2.1.0-rc.5](/releases/release-2.1-rc.5.md) | 2018-11-12 | +| [2.1.0-rc.4](/releases/release-2.1-rc.4.md) | 2018-10-23 | +| [2.0.8](/releases/release-2.0.8.md) | 2018-10-16 | +| [2.1.0-rc.3](/releases/release-2.1-rc.3.md) | 2018-09-29 | +| [2.1.0-rc.2](/releases/release-2.1-rc.2.md) | 2018-09-14 | +| [2.0.7](/releases/release-2.0.7.md) | 2018-09-07 | +| [2.1.0-rc.1](/releases/release-2.1-rc.1.md) | 2018-08-24 | +| [2.0.6](/releases/release-2.0.6.md) | 2018-08-06 | +| [2.0.5](/releases/release-2.0.5.md) | 2018-07-06 | +| [2.1.0-beta](/releases/release-2.1-beta.md) | 2018-06-29 | +| [2.0.4](/releases/release-2.0.4.md) | 2018-06-15 | +| [2.0.3](/releases/release-2.0.3.md) | 2018-06-01 | +| [2.0.2](/releases/release-2.0.2.md) | 2018-05-21 | +| [2.0.1](/releases/release-2.0.1.md) | 2018-05-16 | +| [2.0.0](/releases/release-2.0-ga.md) | 2018-04-27 | +| [2.0.0-rc.5](/releases/release-2.0-rc.5.md) | 2018-04-17 | +| [2.0.0-rc.4](/releases/release-2.0-rc.4.md) | 2018-03-30 | +| [2.0.0-rc.3](/releases/release-2.0-rc.3.md) | 2018-03-23 | +| [2.0.0-rc.1](/releases/release-2.0-rc.1.md) | 2018-03-09 | +| [1.1.0-beta](/releases/release-1.1-beta.md) | 2018-02-24 | +| [1.0.8](/releases/release-1.0.8.md) | 2018-02-11 | +| [1.0.7](/releases/release-1.0.7.md) | 2018-01-22 | +| [1.1.0-alpha](/releases/release-1.1-alpha.md) | 2018-01-19 | +| [1.0.6](/releases/release-1.0.6.md) | 2018-01-08 | +| [1.0.5](/releases/release-1.0.5.md) | 2017-12-26 | +| [1.0.4](/releases/release-1.0.4.md) | 2017-12-11 | +| [1.0.3](/releases/release-1.0.3.md) | 2017-11-28 | +| [1.0.2](/releases/release-1.0.2.md) | 2017-11-13 | +| [1.0.1](/releases/release-1.0.1.md) | 2017-11-01 | +| [1.0.0](/releases/release-1.0-ga.md) | 2017-10-16 | +| [Pre-GA](/releases/release-pre-ga.md) | 2017-08-30 | +| [rc4](/releases/release-rc.4.md) | 2017-08-04 | +| [rc3](/releases/release-rc.3.md) | 2017-06-16 | +| [rc2](/releases/release-rc.2.md) | 2017-03-01 | +| [rc1](/releases/release-rc.1.md) | 2016-12-23 | diff --git a/scale-tidb-using-tiup.md b/scale-tidb-using-tiup.md index d0de352bdf7a9..ba845af25d4ee 100644 --- a/scale-tidb-using-tiup.md +++ b/scale-tidb-using-tiup.md @@ -204,7 +204,11 @@ If you want to add two TiCDC nodes to the `10.0.1.3` and `10.0.1.4` hosts, take ```ini cdc_servers: - host: 10.0.1.3 + gc-ttl: 86400 + data_dir: /data/deploy/install/data/cdc-8300 - host: 10.0.1.4 + gc-ttl: 86400 + data_dir: /data/deploy/install/data/cdc-8300 ``` 2. Run the scale-out command: @@ -260,13 +264,13 @@ If you want to remove a TiKV node from the `10.0.1.5` host, take the following s ``` ``` - Starting /root/.tiup/components/cluster/v1.5.0/cluster display + Starting /root/.tiup/components/cluster/v1.7.0/cluster display TiDB Cluster: - TiDB Version: v5.2.1 + TiDB Version: v5.3.0 ID Role Host Ports Status Data Dir Deploy Dir -- ---- ---- ----- ------ -------- ---------- - 10.0.1.3:8300 cdc 10.0.1.3 8300 Up - deploy/cdc-8300 - 10.0.1.4:8300 cdc 10.0.1.4 8300 Up - deploy/cdc-8300 + 10.0.1.3:8300 cdc 10.0.1.3 8300 Up data/cdc-8300 deploy/cdc-8300 + 10.0.1.4:8300 cdc 10.0.1.4 8300 Up data/cdc-8300 deploy/cdc-8300 10.0.1.4:2379 pd 10.0.1.4 2379/2380 Healthy data/pd-2379 deploy/pd-2379 10.0.1.1:20160 tikv 10.0.1.1 20160/20180 Up data/tikv-20160 deploy/tikv-20160 10.0.1.2:20160 tikv 10.0.1.2 20160/20180 Up data/tikv-20160 deploy/tikv-20160 diff --git a/schedule-replicas-by-topology-labels.md b/schedule-replicas-by-topology-labels.md index 3f613b7a4644e..fd5f1197c2326 100644 --- a/schedule-replicas-by-topology-labels.md +++ b/schedule-replicas-by-topology-labels.md @@ -6,6 +6,10 @@ aliases: ['/docs/dev/location-awareness/','/docs/dev/how-to/deploy/geographic-re # Schedule Replicas by Topology Labels +> **Note:** +> +> TiDB v5.3.0 introduces an experimental support for [Placement Rules in SQL](/placement-rules-in-sql.md). This offers a more convenient way to configure the placement of tables and partitions. Placement Rules in SQL might replace placement configuration with PD in future releases. + To improve the high availability and disaster recovery capability of TiDB clusters, it is recommended that TiKV nodes are physically scattered as much as possible. For example, TiKV nodes can be distributed on different racks or even in different data centers. According to the topology information of TiKV, the PD scheduler automatically performs scheduling at the background to isolate each replica of a Region as much as possible, which maximizes the capability of disaster recovery. To make this mechanism effective, you need to properly configure TiKV and PD so that the topology information of the cluster, especially the TiKV location information, is reported to PD during deployment. Before you begin, see [Deploy TiDB Using TiUP](/production-deployment-using-tiup.md) first. diff --git a/scripts/merge_by_toc.py b/scripts/merge_by_toc.py index bdafe95794057..d4d97b54b9405 100755 --- a/scripts/merge_by_toc.py +++ b/scripts/merge_by_toc.py @@ -4,7 +4,6 @@ # Generate all-in-one Markdown file for ``doc-cn`` # Tip: 不支持中文文件名 # readme.md 中的目录引用的md多次(或者md的sub heading),以第一次出现为主 -# 每个版本都会生成一个自己的 PDF from __future__ import print_function, unicode_literals @@ -32,17 +31,10 @@ level = 0 current_level = "" for line in fp: - if not in_toc and line.startswith("## "): + if not in_toc and not line.startswith(" -> **Warning:** -> -> TiCDC OpenAPI is still an experimental feature. It is not recommended to use it in a production environment. - TiCDC provides the OpenAPI feature for querying and operating the TiCDC cluster, which is similar to the feature of [`cdc cli` tool](/ticdc/manage-ticdc.md#use-cdc-cli-to-manage-cluster-status-and-data-replication-task). You can use the APIs to perform the following maintenance operations on the TiCDC cluster: @@ -148,9 +144,9 @@ The configuration parameters of sink are as follows: { "dispatchers":[ {"matcher":["test1.*", "test2.*"], "dispatcher":"ts"}, - {"matcher":["test3.*", "test4.*"], "dispatcher":"rowid"}, + {"matcher":["test3.*", "test4.*"], "dispatcher":"rowid"} ], - "protocal":"default", + "protocal":"default" } ``` @@ -352,7 +348,7 @@ curl -X GET http://127.0.0.1:8300/api/v1/changefeeds/test1 "capture_id": "d8924259-f52f-4dfb-97a9-c48d26395945", "table_ids": [ 63, - 65, + 65 ], "table_operations": {} } @@ -474,7 +470,7 @@ curl -X GET http://127.0.0.1:8300/api/v1/processors/test1/561c3784-77f0-4863-ad5 "resolved_ts": 426919123369066496, "table_ids": [ 63, - 65, + 65 ], "error": null } diff --git a/ticdc/ticdc-open-protocol.md b/ticdc/ticdc-open-protocol.md index 3e514ce565db1..572614a3f3673 100644 --- a/ticdc/ticdc-open-protocol.md +++ b/ticdc/ticdc-open-protocol.md @@ -272,10 +272,10 @@ COMMIT; ## Protocol parsing for consumers -Currently, TiCDC does not provide the standard parsing library for TiCDC Open Protocol, but the Golang version and Java version of parsing demonstrations are provided. You can refer to the data format provided in this document and the following demonstrations to implement the protocol parsing for consumers. +Currently, TiCDC does not provide the standard parsing library for TiCDC Open Protocol, but the Golang version and Java version of parsing examples are provided. You can refer to the data format provided in this document and the following examples to implement the protocol parsing for consumers. -- [Golang demo](https://github.com/pingcap/ticdc/tree/master/cmd/kafka-consumer) -- [Java demo](https://github.com/pingcap/ticdc/tree/master/demo/java) +- [Golang examples](https://github.com/pingcap/ticdc/tree/master/cmd/kafka-consumer) +- [Java examples](https://github.com/pingcap/ticdc/tree/master/examples/java) ## Column type code diff --git a/ticdc/ticdc-overview.md b/ticdc/ticdc-overview.md index 3f61b8083adb4..4b869c955c406 100644 --- a/ticdc/ticdc-overview.md +++ b/ticdc/ticdc-overview.md @@ -6,10 +6,6 @@ aliases: ['/docs/dev/ticdc/ticdc-overview/','/docs/dev/reference/tools/ticdc/ove # TiCDC Overview -> **Note:** -> -> TiCDC is a feature for general availability (GA) since v4.0.6. You can use it in the production environment. - [TiCDC](https://github.com/pingcap/ticdc) is a tool for replicating the incremental data of TiDB. This tool is implemented by pulling TiKV change logs. It can restore data to a consistent state with any upstream TSO, and provides [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md) to support other systems to subscribe to data changes. ## TiCDC Architecture @@ -27,7 +23,7 @@ The architecture of TiCDC is shown in the following figure: - Assembles KV change logs in the internal logic. - Provides the interface to output KV change logs. The data sent includes real-time change logs and incremental scan change logs. -- `capture`: The operating process of TiCDC. Multiple `capture`s form a TiCDC cluster that replicates KV change logs. +- `capture`: The operating process of TiCDC. Multiple `captures` form a TiCDC cluster that replicates KV change logs. - Each `capture` pulls a part of KV change logs. - Sorts the pulled KV change log(s). @@ -73,6 +69,10 @@ Currently, the TiCDC sink component supports replicating data to the following d ## Restrictions +There are some restrictions when using TiCDC. + +### Requirements for valid index + TiCDC only replicates the table that has at least one **valid index**. A **valid index** is defined as follows: - The primary key (`PRIMARY KEY`) is a valid index. @@ -84,14 +84,33 @@ Since v4.0.8, TiCDC supports replicating tables **without a valid index** by mod ### Unsupported scenarios -Currently, The following scenarios are not supported: +Currently, the following scenarios are not supported: - The TiKV cluster that uses RawKV alone. - The [DDL operation `CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) and the [SEQUENCE function](/sql-statements/sql-statement-create-sequence.md#sequence-function) in TiDB. When the upstream TiDB uses `SEQUENCE`, TiCDC ignores `SEQUENCE` DDL operations/functions performed upstream. However, DML operations using `SEQUENCE` functions can be correctly replicated. TiCDC only provides partial support for scenarios of large transactions in the upstream. For details, refer to [FAQ: Does TiCDC support replicating large transactions? Is there any risk?](/ticdc/troubleshoot-ticdc.md#does-ticdc-support-replicating-large-transactions-is-there-any-risk). -## Notice for compatibility issues +> **Note:** +> +> Since v5.3.0, TiCDC no longer supports the cyclic replication feature. + +## Install and deploy TiCDC + +You can either deploy TiCDC along with a new TiDB cluster or add the TiCDC component to an existing TiDB cluster. For details, see [Deploy TiCDC](/ticdc/deploy-ticdc.md). + +## Manage TiCDC Cluster and Replication Tasks + +Currently, you can use the `cdc cli` tool to manage the status of a TiCDC cluster and data replication tasks. For details, see: + +- [Use `cdc cli` to manage cluster status and data replication task](/ticdc/manage-ticdc.md#use-cdc-cli-to-manage-cluster-status-and-data-replication-task) +- [Use OpenAPI to manage cluster status and data replication task](/ticdc/ticdc-open-api.md) + +## TiCDC Open Protocol + +TiCDC Open Protocol is a row-level data change notification protocol that provides data sources for monitoring, caching, full-text indexing, analysis engines, and primary-secondary replication between different databases. TiCDC complies with TiCDC Open Protocol and replicates data changes of TiDB to third-party data medium such as MQ (Message Queue). For more information, see [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md). + +## Compatibility notes ### Incompatibility issue caused by using the TiCDC v5.0.0-rc `cdc cli` tool to operate a v4.0.x cluster @@ -111,31 +130,22 @@ Solutions: Use the `cdc` executable file corresponding to the TiCDC cluster vers > > The above issue exists only when `cdc cli` is v5.0.0-rc. Other v5.0.x `cdc cli` tool can be compatible with v4.0.x clusters. -## Install and deploy TiCDC +### Compatibility notes for `sort-dir` and `data-dir` -You can either deploy TiCDC along with a new TiDB cluster or add the TiCDC component to an existing TiDB cluster. For details, see [Deploy TiCDC](/ticdc/deploy-ticdc.md). +The `sort-dir` configuration is used to specify the temporary file directory for the TiCDC sorter. Its functionalities might vary in different versions. The following table lists `sort-dir`'s compatibility changes across versions. -## Manage TiCDC Cluster and Replication Tasks +| Version | `sort-engine` functionality | Note | Recommendation | +| :--- | :--- | :-- | :-- | +| v4.0.11 or an earlier v4.0 version, v5.0.0-rc | It is a changefeed configuration item and specifies temporary file directory for the `file` sorter and `unified` sorter. | In these versions, `file` sorter and `unified` sorter are **experimental features** and **NOT** recommended for the production environment.

If multiple changefeeds use the `unified` sorter as its `sort-engine`, the actual temporary file directory might be the `sort-dir` configuration of any changefeed, and the directory used for each TiCDC node might be different. | It is not recommended to use `unified` sorter in the production environment. | +| v4.0.12, v4.0.13, v5.0.0, and v5.0.1 | It is a configuration item of changefeed or of `cdc server`. | By default, the `sort-dir` configuration of a changefeed does not take effect, and the `sort-dir` configuration of `cdc server` defaults to `/tmp/cdc_sort`. It is recommended to only configure `cdc server` in the production environment.

If you use TiUP to deploy TiCDC, it is recommended to use the latest TiUP version and set `sorter.sort-dir` in the TiCDC server configuration.

The `unified` sorter is enabled by default in v4.0.13, v5.0.0, and v5.0.1. If you want to upgrade your cluster to these versions, make sure that you have correctly configured `sorter.sort-dir` in the TiCDC server configuration. | You need to configure `sort-dir` using the `cdc server` command-line parameter (or TiUP). | +| v4.0.14 and later v4.0 versions, v5.0.3 and later v5.0 versions, later TiDB versions | `sort-dir` is deprecated. It is recommended to configure `data-dir`. | You can configure `data-dir` using the latest version of TiUP. In these TiDB versions, `unified` sorter is enabled by default. Make sure that `data-dir` has been configured correctly when you upgrade your cluster. Otherwise, `/tmp/cdc_data` will be used by default as the temporary file directory.

If the storage capacity of the device where the directory is located is insufficient, the problem of insufficient hard disk space might occur. In this situation, the previous `sort-dir` configuration of changefeed will become invalid.| You need to configure `data-dir` using the `cdc server` command-line parameter (or TiUP). | -Currently, you can use the `cdc cli` tool to manage the status of a TiCDC cluster and data replication tasks. For details, see: +### Compatibility with temporary tables -- [Use `cdc cli` to manage cluster status and data replication task](/ticdc/manage-ticdc.md#use-cdc-cli-to-manage-cluster-status-and-data-replication-task) -- [Use OpenAPI to manage cluster status and data replication task](/ticdc/ticdc-open-api.md) +Since v5.3.0, TiCDC supports [global temporary tables](/temporary-tables.md#global-temporary-tables). Replicating global temporary tables to the downstream using TiCDC of a version earlier than v5.3.0 causes table definition error. + +If the upstream cluster contains a global temporary table, the downstream TiDB cluster is expected to be v5.3.0 or a later version. Otherwise, an error occurs during the replication process. ## Troubleshoot TiCDC For details, refer to [Troubleshoot TiCDC](/ticdc/troubleshoot-ticdc.md). - -## TiCDC Open Protocol - -TiCDC Open Protocol is a row-level data change notification protocol that provides data sources for monitoring, caching, full-text indexing, analysis engines, and primary-secondary replication between different databases. TiCDC complies with TiCDC Open Protocol and replicates data changes of TiDB to third-party data medium such as MQ (Message Queue). For more information, see [TiCDC Open Protocol](/ticdc/ticdc-open-protocol.md). - -## Compatibility notes for `sort-dir` and `data-dir` - -The `sort-dir` configuration is used to specify the temporary file directory for the TiCDC sorter. Its functionalities might vary in different versions. The following table lists `sort-dir`'s compatibility changes across versions. - -| Version | `sort-engine` functionality | Note | Recommendation | -| :--- | :--- | :-- | :-- | -| v4.0.11 or an earlier v4.0 version, v5.0.0-rc | It is a changefeed configuration item and specifies temporary file directory for the `file` sorter and `unified` sorter. | In these versions, `file` sorter and `unified` sorter are **experimental features** and **NOT** recommended for the production environment.

If multiple changefeeds use the `unified` sorter as its `sort-engine`, the actual temporary file directory might be the `sort-dir` configuration of any changefeed, and the directory used for each TiCDC node might be different. | It is not recommended to use `unified` sorter in the production environment. | -| v4.0.12, v4.0.13, v5.0.0, and v5.0.1 | It is a configuration item of changefeed or of `cdc server`. | By default, the `sort-dir` configuration of a changefeed does not take effect, and the `sort-dir` configuration of `cdc server` defaults to `/tmp/cdc_sort`. It is recommended to only configure `cdc server` in the production environment.

If you use TiUP to deploy TiCDC, it is recommended to use the latest TiUP version and set `sorter.sort-dir` in the TiCDC server configuration.

The `unified` sorter is enabled by default in v4.0.13, v5.0.0, and v5.0.1. If you want to upgrade your cluster to these versions, make sure that you have correctly configured `sorter.sort-dir` in the TiCDC server configuration. | You need to configure `sort-dir` using the `cdc server` command-line parameter (or TiUP). | -| v4.0.14 and later v4.0 versions, v5.0.2 and later v5.0 versions, later TiDB versions | `sort-dir` is deprecated. It is recommended to configure `data-dir`. | You can configure `data-dir` using the latest version of TiUP. In these TiDB versions, `unified` sorter is enabled by default. Make sure that `data-dir` has been configured correctly when you upgrade your cluster. Otherwise, `/tmp/cdc_data` will be used by default as the temporary file directory.

If the storage capacity of the device where the directory is located is insufficient, the problem of insufficient hard disk space might occur. In this situation, the previous `sort-dir` configuration of changefeed will become invalid.| You need to configure `data-dir` using the `cdc server` command-line parameter (or TiUP). | diff --git a/ticdc/troubleshoot-ticdc.md b/ticdc/troubleshoot-ticdc.md index be47a53d9caf9..af1c2869bcb8c 100644 --- a/ticdc/troubleshoot-ticdc.md +++ b/ticdc/troubleshoot-ticdc.md @@ -25,13 +25,74 @@ If you do not specify `start-ts`, or specify `start-ts` as `0`, when a replicati When you execute `cdc cli changefeed create` to create a replication task, TiCDC checks whether the upstream tables meet the [replication restrictions](/ticdc/ticdc-overview.md#restrictions). If some tables do not meet the restrictions, `some tables are not eligible to replicate` is returned with a list of ineligible tables. You can choose `Y` or `y` to continue creating the task, and all updates on these tables are automatically ignored during the replication. If you choose an input other than `Y` or `y`, the replication task is not created. -## How do I handle replication interruption? +## How do I view the state of TiCDC replication tasks? + +To view the status of TiCDC replication tasks, use `cdc cli`. For example: + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed list --pd=http://10.0.10.25:2379 +``` + +The expected output is as follows: + +```json +[{ + "id": "4e24dde6-53c1-40b6-badf-63620e4940dc", + "summary": { + "state": "normal", + "tso": 417886179132964865, + "checkpoint": "2020-07-07 16:07:44.881", + "error": null + } +}] +``` + +* `checkpoint`: TiCDC has replicated all data before this timestamp to downstream. +* `state`: The state of this replication task: + * `normal`: The task runs normally. + * `stopped`: The task is stopped manually or encounters an error. + * `removed`: The task is removed. + +> **Note:** +> +> This feature is introduced in TiCDC 4.0.3. + +## TiCDC replication interruptions + +### How do I know whether a TiCDC replication task is interrupted? + +- Check the `changefeed checkpoint` monitoring metric of the replication task (choose the right `changefeed id`) in the Grafana dashboard. If the metric value stays unchanged, or the `checkpoint lag` metric keeps increasing, the replication task might be interrupted. +- Check the `exit error count` monitoring metric. If the metric value is greater than `0`, an error has occurred in the replication task. +- Execute `cdc cli changefeed list` and `cdc cli changefeed query` to check the status of the replication task. `stopped` means the task has stopped, and the `error` item provides the detailed error message. After the error occurs, you can search `error on running processor` in the TiCDC server log to see the error stack for troubleshooting. +- In some extreme cases, the TiCDC service is restarted. You can search the `FATAL` level log in the TiCDC server log for troubleshooting. + +### How do I know whether the replication task is stopped manually? + +You can know whether the replication task is stopped manually by executing `cdc cli`. For example: + +{{< copyable "shell-regular" >}} + +```shell +cdc cli changefeed query --pd=http://10.0.10.25:2379 --changefeed-id 28c43ffc-2316-4f4f-a70b-d1a7c59ba79f +``` + +In the output of the above command, `admin-job-type` shows the state of this replication task: + +* `0`: In progress, which means that the task is not stopped manually. +* `1`: Paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpiont-ts`. +* `2`: Resumed. The replication task resumes from `checkpoint-ts`. +* `3`: Removed. When the task is removed, all replicated `processor`s are ended, and the configuration information of the replication task is cleared up. The replication status is retained only for later queries. + +### How do I handle replication interruptions? A replication task might be interrupted in the following known scenarios: - The downstream continues to be abnormal, and TiCDC still fails after many retries. - In this scenario, TiCDC saves the task information. Because TiCDC has set the service GC safepoint in PD, the data after the task checkpoint is not cleaned by TiKV GC within the valid period of `gc-ttl`. + - Handling method: You can resume the replication task via the HTTP interface after the downstream is back to normal. - Replication cannot continue because of incompatible SQL statement(s) in the downstream. @@ -42,38 +103,64 @@ A replication task might be interrupted in the following known scenarios: 2. Use the new task configuration file and add the `ignore-txn-start-ts` parameter to skip the transaction corresponding to the specified `start-ts`. 3. Stop the old replication task via HTTP API. Execute `cdc cli changefeed create` to create a new task and specify the new task configuration file. Specify `checkpoint-ts` recorded in step 1 as the `start-ts` and start a new task to resume the replication. -## How do I know whether a TiCDC replication task is interrupted? +- In TiCDC v4.0.13 and earlier versions, when TiCDC replicates the partitioned table, it might encounter an error that leads to replication interruption. -- Check the `changefeed checkpoint` monitoring metric of the replication task (choose the right `changefeed id`) in the Grafana dashboard. If the metric value stays unchanged, or the `checkpoint lag` metric keeps increasing, the replication task might be interrupted. -- Check the `exit error count` monitoring metric. If the metric value is greater than `0`, an error has occurred in the replication task. -- Execute `cdc cli changefeed list` and `cdc cli changefeed query` to check the status of the replication task. `stopped` means the task has stopped and the `error` item provides the detailed error information. After the error occurs, you can search `error on running processor` in the TiCDC server log to see the error stack for troubleshooting. -- In some extreme cases, the TiCDC service is restarted. You can search the `FATAL` level log in the TiCDC server log for troubleshooting. + - In this scenario, TiCDC saves the task information. Because TiCDC has set the service GC safepoint in PD, the data after the task checkpoint is not cleaned by TiKV GC within the valid period of `gc-ttl`. + - Handling procedures: + 1. Pause the replication task by executing `cdc cli changefeed pause -c `. + 2. Wait for about one munite, and then resume the replication task by executing `cdc cli changefeed resume -c `. -## What is `gc-ttl` in TiCDC? +### What should I do to handle the OOM that occurs after TiCDC is restarted after a task interruption? + +- Update your TiDB cluster and TiCDC cluster to the latest versions. The OOM problem has already been resolved in **v4.0.14 and later v4.0 versions, v5.0.2 and later v5.0 versions, and the latest versions**. -Since v4.0.0-rc.1, PD supports external services in setting the service-level GC safepoint. Any service can register and update its GC safepoint. PD ensures that the key-value data smaller than this GC safepoint is not cleaned by GC. Enabling this feature in TiCDC ensures that the data to be consumed by TiCDC is retained in TiKV without being cleaned by GC when the replication task is unavailable or interrupted. +- In the above updated versions, you can enable the Unified Sorter to help you sort data in the disk when the system memory is insufficient. To enable this function, you can pass `--sort-engine=unified` to the `cdc cli` command when creating a replication task. For example: -When starting the TiCDC server, you can specify the Time To Live (TTL) duration of GC safepoint through `gc-ttl`, which means the longest time that data is retained within the GC safepoint. This value is set by TiCDC in PD, which is 86,400 seconds by default. +{{< copyable "shell-regular" >}} -## How do I handle the OOM that occurs after TiCDC is restarted after a task interruption? +```shell +cdc cli changefeed update -c --sort-engine="unified" --pd=http://10.0.10.25:2379 +``` -If the replication task is interrupted for a long time and a large volume of new data has been written to TiDB, Out of Memory (OOM) might occur when TiCDC is restarted. In this situation, you can enable unified sorter, TiCDC's experimental sorting engine. This engine sorts data in the disk when the memory is insufficient. To enable this feature, pass `--sort-engine=unified` and `--sort-dir=/path/to/sort_dir` to the `cdc cli` command when creating a replication task. For example: +If you fail to update your cluster to the above new versions, you can still enable Unified Sorter in **previous versions**. You can pass `--sort-engine=unified` and `--sort-dir=/path/to/sort_dir` to the `cdc cli` command when creating a replication task. For example: {{< copyable "shell-regular" >}} ```shell -cdc cli changefeed update -c [changefeed-id] --sort-engine="unified" --sort-dir="/data/cdc/sort" --pd=http://10.0.10.25:2379 +cdc cli changefeed update -c --sort-engine="unified" --sort-dir="/data/cdc/sort" --pd=http://10.0.10.25:2379 ``` > **Note:** > > + Since v4.0.9, TiCDC supports the unified sorter engine. > + TiCDC (the 4.0 version) does not support dynamically modifying the sorting engine yet. Make sure that the changefeed has stopped before modifying the sorter settings. +> + `sort-dir` has different behaviors in different versions. Refer to [compatibility notes for`sort-dir` and `data-dir`](/ticdc/ticdc-overview.md#compatibility-notes-for-sort-dir-and-data-dir), and configure it with caution. > + Currently, the unified sorter is an experimental feature. When the number of tables is too large (>=100), the unified sorter might cause performance issues and affect replication throughput. Therefore, it is not recommended to use it in a production environment. Before you enable the unified sorter, make sure that the machine of each TiCDC node has enough disk capacity. If the total size of unprocessed data changes might exceed 1 TB, it is not recommend to use TiCDC for replication. +## What is `gc-ttl` in TiCDC? + +Since v4.0.0-rc.1, PD supports external services in setting the service-level GC safepoint. Any service can register and update its GC safepoint. PD ensures that the key-value data later than this GC safepoint is not cleaned by GC. + +When the replication task is unavailable or interrupted, this feature ensures that the data to be consumed by TiCDC is retained in TiKV without being cleaned by GC. + +When starting the TiCDC server, you can specify the Time To Live (TTL) duration of GC safepoint by configuring `gc-ttl`. You can also [use TiUP to modify](/ticdc/manage-ticdc.md#modify-ticdc-configuration-using-tiup) `gc-ttl`. The default value is 24 hours. In TiCDC, this value means: + +- The maximum time the GC safepoint is retained at the PD after the TiCDC service is stopped. +- The maximum time a replication task can be suspended after the task is interrupted or manually stopped. If the time for a suspended replication task is longer than the value set by `gc-ttl`, the replication task enters the `failed` status, cannot be resumed, and cannot continue to affect the progress of the GC safepoint. + +The second behavior above is introduced in TiCDC v4.0.13 and later versions. The purpose is to prevent a replication task in TiCDC from suspending for too long, causing the GC safepoint of the upstream TiKV cluster not to continue for a long time and retaining too many outdated data versions, thus affecting the performance of the upstream cluster. + +> **Note:** +> +> In some scenarios, for example, when you use TiCDC for incremental replication after full replication with Dumpling/BR, the default 24 hours of `gc-ttl` may not be sufficient. You need to specify an appropriate value for `gc-ttl` when you start the TiCDC server. + ## What is the complete behavior of TiCDC garbage collection (GC) safepoint? -If a replication task starts after the TiCDC service starts, the TiCDC owner updates the PD service GC safepoint with the smallest value of `checkpoint-ts` among all replication tasks. The service GC safepoint ensures that TiCDC does not delete data generated at that time and after that time. If the replication task is interrupted, the `checkpoint-ts` of this task does not change and PD's corresponding service GC safepoint is not updated either. The Time-To-Live (TTL) that TiCDC sets for a service GC safepoint is 24 hours, which means that the GC mechanism does not delete any data if the TiCDC service can be recovered within 24 hours after it is interrupted. +If a replication task starts after the TiCDC service starts, the TiCDC owner updates the PD service GC safepoint with the smallest value of `checkpoint-ts` among all replication tasks. The service GC safepoint ensures that TiCDC does not delete data generated at that time and after that time. If the replication task is interrupted, or manually stopped, the `checkpoint-ts` of this task does not change. Meanwhile, PD's corresponding service GC safepoint is not updated either. + +If the replication task is suspended longer than the time specified by `gc-ttl`, the replication task enters the `failed` status and cannot be resumed. The PD corresponding service GC safepoint will continue. + +The Time-To-Live (TTL) that TiCDC sets for a service GC safepoint is 24 hours, which means that the GC mechanism does not delete any data if the TiCDC service can be recovered within 24 hours after it is interrupted. ## How do I handle the `Error 1298: Unknown or incorrect time zone: 'UTC'` error when creating the replication task or replicating data to MySQL? @@ -177,61 +264,9 @@ cdc cli changefeed create --pd=http://10.0.10.25:2379 --sink-uri="kafka://127.0. For more information, refer to [Create a replication task](/ticdc/manage-ticdc.md#create-a-replication-task). -## How do I view the status of TiCDC replication tasks? - -To view the status of TiCDC replication tasks, use `cdc cli`. For example: - -{{< copyable "shell-regular" >}} - -```shell -cdc cli changefeed list --pd=http://10.0.10.25:2379 -``` - -The expected output is as follows: - -```json -[{ - "id": "4e24dde6-53c1-40b6-badf-63620e4940dc", - "summary": { - "state": "normal", - "tso": 417886179132964865, - "checkpoint": "2020-07-07 16:07:44.881", - "error": null - } -}] -``` - -* `checkpoint`: TiCDC has replicated all data before this timestamp to downstream. -* `state`: The state of the replication task: - - * `normal`: The task runs normally. - * `stopped`: The task is stopped manually or encounters an error. - * `removed`: The task is removed. - -> **Note:** -> -> This feature is introduced in TiCDC 4.0.3. - -## How do I know whether the replication task is stopped manually? - -You can know whether the replication task is stopped manually by using `cdc cli`. For example: - -{{< copyable "shell-regular" >}} - -```shell -cdc cli changefeed query --pd=http://10.0.10.25:2379 --changefeed-id 28c43ffc-2316-4f4f-a70b-d1a7c59ba79f -``` - -In the output of this command, `admin-job-type` shows the state of the replication task: - -* `0`: In progress, which means that the task is not stopped manually. -* `1`: Paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpiont-ts`. -* `2`: Resumed. The replication task resumes from `checkpoint-ts`. -* `3`: Removed. When the task is removed, all replicated `processor`s are ended, and the configuration information of the replication task is cleared up. Only the replication status is retained for later queries. - ## Why does the latency from TiCDC to Kafka become higher and higher? -* Check [how do I view the status of TiCDC replication tasks](#how-do-i-view-the-status-of-ticdc-replication-tasks). +* Check [how do I view the state of TiCDC replication tasks](#how-do-i-view-the-state-of-ticdc-replication-tasks). * Adjust the following parameters of Kafka: * Increase the `message.max.bytes` value in `server.properties` to `1073741824` (1 GB). diff --git a/tidb-binlog/binlog-consumer-client.md b/tidb-binlog/binlog-consumer-client.md index b2bf52526b3be..9372142179b7a 100644 --- a/tidb-binlog/binlog-consumer-client.md +++ b/tidb-binlog/binlog-consumer-client.md @@ -118,7 +118,7 @@ message Binlog { } ``` -For the definition of the data format, see [`binlog.proto`](https://github.com/pingcap/tidb-tools/blob/master/tidb-binlog/proto/proto/binlog.proto). +For the definition of the data format, see [`secondary_binlog.proto`](https://github.com/pingcap/tidb-tools/blob/master/tidb-binlog/proto/proto/secondary_binlog.proto) ### Driver diff --git a/tidb-binlog/tidb-binlog-overview.md b/tidb-binlog/tidb-binlog-overview.md index 1818a2bee269a..a93c24c87987c 100644 --- a/tidb-binlog/tidb-binlog-overview.md +++ b/tidb-binlog/tidb-binlog-overview.md @@ -71,7 +71,7 @@ The TiDB Binlog cluster is composed of Pump and Drainer. * Drainer supports replicating binlogs to MySQL, TiDB, Kafka or local files. If you need to replicate binlogs to other Drainer unsuppored destinations, you can set Drainer to replicate the binlog to Kafka and read the data in Kafka for customized processing according to binlog consumer protocol. See [Binlog Consumer Client User Guide](/tidb-binlog/binlog-consumer-client.md). -* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/master/proto/binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. +* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/master/proto/pb_binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. Pay attention to the value of `db-type`: diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index b658a018f9a94..56eac73bf104d 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -255,7 +255,8 @@ Configuration items related to log files. #### `max-size` - The size limit of the log file. -- Default value: 300MB +- Default value: 300 +- Unit: MB - The maximum size is 4GB. #### `max-days` @@ -482,7 +483,7 @@ The Plan Cache configuration of the `PREPARE` statement. ### `capacity` - The number of cached statements. -- Default value: `100` +- Default value: `1000` - The type is `UINT`. Values less than `0` are converted to large integers. ### `memory-guard-ratio` @@ -553,7 +554,7 @@ This section introduces configuration items related to the Coprocessor Cache fea Configuration related to the transaction latch. It is recommended to enable it when many local transaction conflicts occur. -### `enable` +### `enabled` - Determines whether to enable the memory lock of transactions. - Default value: `false` diff --git a/tidb-lightning/migrate-from-csv-using-tidb-lightning.md b/tidb-lightning/migrate-from-csv-using-tidb-lightning.md index a44feeff3c048..78e999205190c 100644 --- a/tidb-lightning/migrate-from-csv-using-tidb-lightning.md +++ b/tidb-lightning/migrate-from-csv-using-tidb-lightning.md @@ -1,10 +1,12 @@ --- -title: TiDB Lightning CSV Support +title: TiDB Lightning CSV Support and Restrictions summary: Learn how to import CSV files via TiDB Lightning. aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/csv/'] --- -# TiDB Lightning CSV Support +# TiDB Lightning CSV Support and Restrictions + +This document describes how to migrate data from CSV files to TiDB using TiDB Lightning. For information about how to generate CSV files from MySQL, see [Export to CSV files using Dumpling](/dumpling-overview.md#export-to-csv-files). TiDB Lightning supports reading CSV (comma-separated values) data source, as well as other delimited format such as TSV (tab-separated values). @@ -29,10 +31,12 @@ The CSV format can be configured in `tidb-lightning.toml` under the `[mydumper.c ```toml [mydumper.csv] -# Separator between fields, should be an ASCII character. +# Separator between fields. Must be ASCII characters. It is not recommended to use the default ','. It is recommended to use '\|+\|' or other uncommon character combinations. separator = ',' -# Quoting delimiter, can either be an ASCII character or empty string. +# Quoting delimiter. Empty value means no quoting. delimiter = '"' +# Line terminator. Empty value means both "\n" (LF) and "\r\n" (CRLF) are line terminators. +terminator = '' # Whether the CSV files contain a header. # If `header` is true, the first line will be skipped. header = true @@ -48,16 +52,25 @@ backslash-escape = true trim-last-separator = false ``` +In all string fields such as `separator`, `delimiter` and `terminator`, if the input involves special characters, you can use backslash escape sequence to represent them in a *double-quoted* string (`"…"`). For example, `separator = "\u001f"` means using the ASCII character 0x1F as separator. + +Additionally, you can use *single-quoted* strings (`'…'`) to suppress backslash escaping. For example, `terminator = '\n'` means using the two-character string: a backslash followed by the letter "n", as the terminator. + +See the [TOML v1.0.0 specification] for details. + [`LOAD DATA`]: https://dev.mysql.com/doc/refman/8.0/en/load-data.html +[TOML v1.0.0 specification]: https://toml.io/en/v1.0.0#string + ### `separator` - Defines the field separator. -- Must be a single ASCII character. +- Can be multiple characters, but must not be empty. - Common values: - * `','` for CSV - * `"\t"` for TSV + * `','` for CSV (comma-separated values) + * `"\t"` for TSV (tab-separated values) + * `"\u0001"` to use the ASCII character 0x01 as separator - Corresponds to the `FIELDS TERMINATED BY` option in the LOAD DATA statement. @@ -74,6 +87,12 @@ trim-last-separator = false [RFC 4180]: https://tools.ietf.org/html/rfc4180 +### `terminator` + +- Defines the line terminator. +- If `terminator` is empty, both `"\r"` (U+000D Carriage Return) and `"\n"` (U+000A Line Feed) are used as terminator. +- Corresponds to the `LINES TERMINATED BY` option in the `LOAD DATA` statement. + ### `header` - Whether *all* CSV files contain a header row. @@ -127,14 +146,26 @@ trim-last-separator = false - When `trim-last-separator = false`, this is interpreted as a row of 5 fields `('A', '', 'B', '', '')`. - When `trim-last-separator = true`, this is interpreted as a row of 3 fields `('A', '', 'B')`. +- This option is deprecated, because the behavior with multiple trailing separators is not intuitive. Use the `terminator` option instead. If your old configuration was + + ```toml + separator = ',' + trim-last-separator = true + ``` + + we recommend changing this to + + ```toml + separator = ',' + terminator = ",\n" + ``` + ### Non-configurable options TiDB Lightning does not support every option supported by the `LOAD DATA` statement. Some examples: -* The line terminator must only be CR (`\r`), LF (`\n`) or CRLF (`\r\n`), which means `LINES TERMINATED BY` is not customizable. * There cannot be line prefixes (`LINES STARTING BY`). -* The header cannot be simply skipped (`IGNORE n LINES`), it must be valid column names if present. -* Delimiters and separators can only be a single ASCII character. +* The header cannot be simply skipped (`IGNORE n LINES`). It must be valid column names if present. ## Strict format @@ -147,10 +178,10 @@ This can be fixed by splitting the CSV into multiple files first. For the generi strict-format = true ``` -Currently, a strict CSV file means every field occupies only a single line. In the other words, one of the following must be true: +Currently, a strict CSV file means every field occupies only a single line. In other words, one of the following must be true: * Delimiter is empty, or -* Every field does not contain CR (`\r`) or LF (`\n`). +* Every field does not contain the terminator itself. In the default configuration, this means every field does not contain CR (`\r`) or LF (`\n`). If a CSV file is not strict, but `strict-format` was wrongly set to `true`, a field spanning multiple lines may be cut in half into two chunks, causing parse failure, or even worse, quietly importing corrupted data. @@ -162,13 +193,12 @@ The default setting is already tuned for CSV following RFC 4180. ```toml [mydumper.csv] -separator = ',' +separator = ',' # It is not recommended to use the default ‘,’. It is recommended to use ‘\|+\|‘ or other uncommon character combinations. delimiter = '"' header = true not-null = false null = '\N' backslash-escape = true -trim-last-separator = false ``` Example content: @@ -191,7 +221,6 @@ header = true not-null = false null = 'NULL' backslash-escape = false -trim-last-separator = false ``` Example content: @@ -210,10 +239,10 @@ ID Region Count [mydumper.csv] separator = '|' delimiter = '' +terminator = "|\n" header = false not-null = true backslash-escape = false -trim-last-separator = true ``` Example content: diff --git a/tidb-lightning/tidb-lightning-backends.md b/tidb-lightning/tidb-lightning-backends.md index eb0aab23a2f5d..6c9e57131d0bd 100644 --- a/tidb-lightning/tidb-lightning-backends.md +++ b/tidb-lightning/tidb-lightning-backends.md @@ -79,6 +79,165 @@ backend = "tidb" or supplying the `--backend tidb` arguments when executing `tidb-lightning`. +#### Configuration description and samples + +This section provides the samples for task configuration in TiDB Lightning. + +```toml +# tidb-lightning task configuration + +[lightning] +# Checks whether the cluster satisfies the minimum requirement before starting. +check-requirements = true + +# Each table is split into one "index engine" to store indices, and multiple +# "data engines" to store row data. These settings control the maximum +# concurrent number for each type of engines. +# Controls the maximum number of tables that can be imported in parallel. For TiDB-backend, the default value is the number of CPU cores. +index-concurrency = 40 + +# Controls the maximum number of "data engines" allowed to be imported in parallel. The default value is the number of CPU cores. The value should be no less than the value of index-concurrency. +table-concurrency = 40 + +# The number of concurrent SQL statements executed. It is set to the number of logical CPU cores by default. The bottleneck of TiDB-backend is usually not the CPU. You can increase this value based on the actual load of the downstream cluster to optimize the write speed. At the same time, when adjusting this configuration, it is recommended to adjust the index-concurrency and table-concurrency to the same value. +region-concurrency = 40 + +# Logging +level = "info" +# The directory to which the log is output. If it is empty (default), the file is saved to /tmp/lightning.log.{timestamp}. If you want the logs to be written to the system standard output, set it to "-". +file = "tidb-lightning.log" + +[checkpoint] +# Whether to enable checkpoints. +# While importing data, TiDB Lightning records which tables have been imported, so +# even if TiDB Lightning or some other component crashes, you can start from a known +# good state instead of restarting from scratch. +enable = true + +# Where to store the checkpoints. +# - file (default): store as a local file (requires v2.1.1 or later) +# - mysql: store into a remote MySQL-compatible database +driver = "file" + +# The schema name (database name) to store the checkpoints +# Enabled only when `driver = "mysql"`. +# schema = "tidb_lightning_checkpoint" + +# The data source name (DSN) indicating the location of the checkpoint storage. +# +# For the "file" driver, the DSN is a path. If the path is not specified, Lightning would +# default to "/tmp/CHECKPOINT_SCHEMA.pb". +# +# For the "mysql" driver, the DSN is a URL in the form of "USER:PASS@tcp(HOST:PORT)/". +# If the URL is not specified, the TiDB server from the [tidb] section is used to +# store the checkpoints. You should specify a different MySQL-compatible +# database server to reduce the load of the target TiDB cluster. +#dsn = "/tmp/tidb_lightning_checkpoint.pb" + +# Whether to keep the checkpoints after all data are imported. If false, the +# checkpoints are deleted. Keeping the checkpoints can aid debugging but +# might leak metadata about the data source. +# keep-after-success = false + +[tikv-importer] +# use the TiDB-backend. +backend = "tidb" + +# Action to do when trying to insert a duplicated entry in the "tidb" backend. +# - replace: use new entry to replace the existing entry +# - ignore: keep the existing entry, and ignore the new entry +# - error: report error and quit the program +# on-duplicate = "replace" + +[mydumper] +# Block size for file reading. Keep it longer than the longest string of +# the data source. +# read-block-size = "64KiB" + +# Minimum size (in terms of source data file) of each batch of import. +# TiDB Lightning splits a large table into multiple data engine files according to this size. +# batch-size = 107_374_182_400 # Byte (default = 100 GB) + +# Local source data directory or the URL of the external storage. +data-source-dir = "/data/my_database" + +# the input data in a "strict" format speeds up processing. +# "strict-format = true" requires that: +# in CSV, every value cannot contain literal new lines (U+000A and U+000D, or \r and \n) even +# when quoted, which means new lines are strictly used to separate rows. +# "Strict" format allows TiDB Lightning to quickly locate split positions of a large file for parallel processing. +# However, if the input data is not "strict", it may split a valid data in half and +# corrupt the result. +# The default value is false for safety instead of speed. +strict-format = false + +# If strict-format is true, TiDB Lightning splits large CSV files into multiple chunks to process in +# parallel. max-region-size is the maximum size of each chunk after splitting. +# max-region-size = 268_435_456 # Byte (default = 256 MB) + +# Only import tables if these wildcard rules are matched. See the corresponding section for details. +filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*'] + +# Configures how CSV files are parsed. +[mydumper.csv] +# Separator between fields, should be an ASCII character. +separator = ',' +# Quoting delimiter, can either be an ASCII character or empty string. +delimiter = '"' +# Whether the CSV files contain a header. +# If `header` is true, the first line will be skipped. +header = true +# Whether the CSV contains any NULL value. +# If `not-null` is true, all columns from CSV cannot be NULL. +not-null = false +# When `not-null` is false (that is, CSV can contain NULL), +# fields equal to this value will be treated as NULL. +null = '\N' +# Whether to interpret backslash escapes inside fields. +backslash-escape = true +# If a line ends with a separator, remove it. +trim-last-separator = false + +[tidb] +# Configuration of any TiDB server from the cluster. +host = "172.16.31.1" +port = 4000 +user = "root" +password = "" + +# The default SQL mode used to parse and execute the SQL statements. +sql-mode = "ONLY_FULL_GROUP_BY,NO_ENGINE_SUBSTITUTION" + +# Whether to use TLS for SQL connections. Valid values are: +# * "" - force TLS (same as "cluster") if [tidb.security] section is populated, otherwise same as "false" +# * "false" - disable TLS +# * "cluster" - force TLS and verify the server's certificate with the CA specified in the [tidb.security] section +# * "skip-verify" - force TLS but do not verify the server's certificate (insecure!) +# * "preferred" - same as "skip-verify", but if the server does not support TLS, fallback to unencrypted connection +# tls = "" + +# Specifies certificates and keys for TLS-enabled MySQL connections. +# [tidb.security] + +# Public certificate of the CA. Set to empty string to disable TLS for SQL. +# ca-path = "/path/to/ca.pem" + +# Public certificate of this service. Default to copy of `security.cert-path` +# cert-path = "/path/to/lightning.pem" + +# Private key of this service. Default to copy of `security.key-path` +# key-path = "/path/to/lightning.key" + +# Configures the background periodic actions. +# Supported units: h (hour), m (minute), s (second). +[cron] + +# Duration between which an import progress is printed to the log. +log-progress = "5m" +``` + +For detailed descriptions of the configuration items, see [TiDB Lightning Configuration](/tidb-lightning/tidb-lightning-configuration.md). + ### Conflict resolution The TiDB-backend supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. diff --git a/tidb-lightning/tidb-lightning-checkpoints.md b/tidb-lightning/tidb-lightning-checkpoints.md index ef6e8e7583a7d..c68bc8201496d 100644 --- a/tidb-lightning/tidb-lightning-checkpoints.md +++ b/tidb-lightning/tidb-lightning-checkpoints.md @@ -17,7 +17,7 @@ This document describes how to enable, configure, store, and control *checkpoint # Whether to enable checkpoints. # While importing data, TiDB Lightning records which tables have been imported, so # even if TiDB Lightning or some other component crashes, you can start from a known -# good state instead of redoing everything. +# good state instead of restarting from scratch. enable = true # Where to store the checkpoints. diff --git a/tidb-lightning/tidb-lightning-configuration.md b/tidb-lightning/tidb-lightning-configuration.md index d2432d520ae2f..c89fd0e3ab341 100644 --- a/tidb-lightning/tidb-lightning-configuration.md +++ b/tidb-lightning/tidb-lightning-configuration.md @@ -76,7 +76,7 @@ io-concurrency = 5 # Whether to enable checkpoints. # While importing data, TiDB Lightning records which tables have been imported, so # even if TiDB Lightning or another component crashes, you can start from a known -# good state instead of redoing everything. +# good state instead of restarting from scratch. enable = true # The schema name (database name) to store the checkpoints. schema = "tidb_lightning_checkpoint" @@ -91,11 +91,11 @@ driver = "file" # If the URL is not specified, the TiDB server from the [tidb] section is used to # store the checkpoints. You should specify a different MySQL-compatible # database server to reduce the load of the target TiDB cluster. -#dsn = "/tmp/tidb_lightning_checkpoint.pb" +# dsn = "/tmp/tidb_lightning_checkpoint.pb" # Whether to keep the checkpoints after all data are imported. If false, the # checkpoints will be deleted. Keeping the checkpoints can aid debugging but # will leak metadata about the data source. -#keep-after-success = false +# keep-after-success = false [tikv-importer] # Delivery backend, can be "local", "importer" or "tidb". @@ -103,13 +103,12 @@ driver = "file" # The listening address of tikv-importer when backend is "importer". Change it to the actual address. addr = "172.16.31.10:8287" # Action to do when trying to insert a duplicated entry in the "tidb" backend. -# - replace: new entry replaces existing entry -# - ignore: keep existing entry, ignore new entry -# - error: report error and quit the program +# - replace: use new entry to replace the existing entry +# - ignore: keep the existing entry, and ignore the new entry +# - error: report error and quit the program # on-duplicate = "replace" -# The size limit of generated SST files in the "local" backend. It is better -# to be the same as the Region size of TiKV (96 MB by default). -# region-split-size = 100_663_296 +# state in the target TiDB. +# duplicate-resolution = 'none' # The number of KV pairs sent in one request in the "local" backend. # send-kv-pairs = 32768 # The directory of local KV sorting in the "local" backend. If the disk @@ -126,10 +125,6 @@ addr = "172.16.31.10:8287" # the data source. read-block-size = 65536 # Byte (default = 64 KB) -# Minimum size (in terms of source data file) of each batch of import. -# TiDB Lightning splits a large table into multiple data engine files according to this size. -# batch-size = 107_374_182_400 # Byte (default = 100 GB) - # The engine file needs to be imported sequentially. Due to parallel processing, # multiple data engines will be imported at nearly the same time, and this # creates a queue and wastes resources. Therefore, TiDB Lightning slightly @@ -157,33 +152,49 @@ no-schema = false # - auto: (default) automatically detects whether the schema is UTF-8 or # GB-18030. An error is reported if the encoding is neither. # - binary: do not try to decode the schema files -# Note that the *data* files are always parsed as binary regardless of -# schema encoding. character-set = "auto" -# Assumes the input data are "strict" to speed up processing. -# Implications of strict-format = true are: -# * in CSV, every value cannot contain literal new lines (U+000A and U+000D, or \r and \n) even -# when quoted, which means new lines are strictly used to separate rows. -# Strict format allows TiDB Lightning to quickly locate split positions of a large file for parallel -# processing. However, if the input data is not strict, it may split a valid data in half and +# Specifies the character set of the source data file. Lightning converts the source file from the specified character set to UTF-8 encoding when importing. +# Currently, this configuration only specifies the character set of the CSV files with the following options supported: +# - utf8mb4: Indicates that the source data file uses UTF-8 encoding. +# - GB18030: Indicates that the source data file uses the GB-18030 encoding. +# - GBK: The source data file uses GBK encoding (GBK encoding is an extension of the GB-2312 character set, also known as Code Page 936). +# - binary: Indicates that Lightning does not convert the encoding (by default). +# If left blank, the default value "binary" is used, that is to say, Lightning does not convert the encoding. +# Note that Lightning does not predict about the character set of the source data file and only converts the source file and import the data based on this configuration. +# If the value of this configuration is not the same as the actual encoding of the source data file, a failed import, data loss or data disorder might appear. +data-character-set = "binary" +# Specifies the replacement character in case of incompatible characters during the character set conversion of the source data file. +# This configuration must not be duplicated with field separators, quote definers, and line breaks. +# The default value is "\uFFFD", which is the "error" Rune or Unicode replacement character in UTF-8 encoding. +# Changing the default value might result in potential degradation of parsing performance for the source data file. +data-invalid-char-replace = "\uFFFD" + +# the input data in a "strict" format speeds up processing. +# "strict-format = true" requires that: +# in CSV, every value cannot contain literal new lines (U+000A and U+000D, or \r and \n) even +# when quoted, which means new lines are strictly used to separate rows. +# "Strict" format allows TiDB Lightning to quickly locate split positions of a large file for parallel processing. +# However, if the input data is not "strict", it may split a valid data in half and # corrupt the result. -# The default value is false for safety over speed. +# The default value is false for safety instead of speed. strict-format = false -# If strict-format is true, TiDB Lightning will split large CSV files into multiple chunks to process in +# If strict-format is true, TiDB Lightning splits large CSV files into multiple chunks to process in # parallel. max-region-size is the maximum size of each chunk after splitting. # max-region-size = 268_435_456 # Byte (default = 256 MB) # Only import tables if these wildcard rules are matched. See the corresponding section for details. -filter = ['*.*'] +filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*'] # Configures how CSV files are parsed. [mydumper.csv] -# Separator between fields, should be an ASCII character. +# Separator between fields. Must not be empty. separator = ',' -# Quoting delimiter, can either be an ASCII character or empty string. +# Quoting delimiter. Empty value means no quoting. delimiter = '"' +# Line terminator. Empty value means both "\n" (LF) and "\r\n" (CRLF) are line terminators. +terminator = '' # Whether the CSV files contain a header. # If `header` is true, the first line will be skipped. header = true @@ -249,18 +260,35 @@ max-allowed-packet = 67_108_864 # these as true in the production environment. # The execution order: Checksum -> Analyze [post-restore] -# Performs `ADMIN CHECKSUM TABLE ` for each table to verify data integrity. -checksum = true +# Specifies the behavior of `ADMIN CHECKSUM TABLE
` for each table to verify data integrity. +# The following options are available: +# - "off": Do not perform checksum. +# - "optional": Perform admin checksum, but will ignore any error if checksum fails. +# - "required": Perform admin checksum. If checksum fails, TiDB Lightning will exit with failure. +# The default value is "required". Note that since v4.0.8, the default value has changed from "true" to "required". +# For backward compatibility, bool values "true" and "false" are also allowed for this field. +# "true" is equivalent to "required" and "false" is equivalent to "off". +checksum = required + # If the value is set to `true`, a level-1 compaction is performed # every time a table is imported. # The default value is `false`. level-1-compact = false + # If the value is set to `true`, a full compaction on the whole # TiKV cluster is performed at the end of the import. # The default value is `false`. compact = false -# Performs `ANALYZE TABLE
` for each table. -analyze = true + +# Specifies the behavior of `ANALYZE TABLE
` for each table. +# The following options are available: +# - "off": Do not perform `ANALYZE TABLE
`. +# - "optional": Perform `ANALYZE TABLE
`, but will ignore any error if checksum fails. +# - "required": Perform `ANALYZE TABLE
`. If it fails, TiDB Lightning will exit with failure. +# The default value is "optional". Note that since v4.0.8, the default value has changed from "true" to "optional". +# For backward compatibility, bool values "true" and "false" are also allowed for this field. +# "true" is equivalent to "required" and "false" is equivalent to "off". +analyze = optional # Configures the background periodic actions. # Supported units: h (hour), m (minute), s (second). @@ -363,7 +391,7 @@ min-available-ratio = 0.05 | -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | | -f *rule* | [Table filter rules](/table-filter.md) (can be specified multiple times) | `mydumper.filter` | | --backend *backend* | [Delivery backend](/tidb-lightning/tidb-lightning-backends.md) (`local`, `importer`, or `tidb`) | `tikv-importer.backend` | -| --log-file *file* | Log file path (default = a temporary file in `/tmp`) | `lightning.log-file` | +| --log-file *file* | Log file path. By default, it is `/tmp/lightning.log.{timestamp}`. If set to '-', it means that the log files will be output to stdout. | `lightning.log-file` | | --status-addr *ip:port* | Listening address of the TiDB Lightning server | `lightning.status-port` | | --importer *host:port* | Address of TiKV Importer | `tikv-importer.addr` | | --pd-urls *host:port* | PD endpoint address | `tidb.pd-addr` | @@ -374,8 +402,8 @@ min-available-ratio = 0.05 | --tidb-password *password* | Password to connect to TiDB | `tidb.password` | | --no-schema | Ignore schema files, get schema directly from TiDB | `mydumper.no-schema` | | --enable-checkpoint *bool* | Whether to enable checkpoints (default = true) | `checkpoint.enable` | -| --analyze *bool* | Analyze tables after importing (default = true) | `post-restore.analyze` | -| --checksum *bool* | Compare checksum after importing (default = true) | `post-restore.checksum` | +| --analyze *bool* | Analyze tables after importing (default = optional) | `post-restore.analyze` | +| --checksum *bool* | Compare checksum after importing (default = required) | `post-restore.checksum` | | --check-requirements *bool* | Check cluster version compatibility before starting (default = true) | `lightning.check-requirements` | | --ca *file* | CA certificate path for TLS connection | `security.ca-path` | | --cert *file* | Certificate path for TLS connection | `security.cert-path` | diff --git a/tidb-lightning/tidb-lightning-distributed-import.md b/tidb-lightning/tidb-lightning-distributed-import.md new file mode 100644 index 0000000000000..8ddc732540528 --- /dev/null +++ b/tidb-lightning/tidb-lightning-distributed-import.md @@ -0,0 +1,166 @@ +--- +title: Use TiDB Lightning to Import Data in Parallel +summary: Learn the concept, user scenarios, usages, and limitations of importing Data in parallel when using TiDB Lightning. +--- + +# Use TiDB Lightning to Import Data in Parallel + +Since v5.3.0, the [Local-backend mode](/tidb-lightning/tidb-lightning-backends.md#tidb-lightning-local-backend) of TiDB Lightning supports the parallel import of a single table or multiple tables. By simultaneously running multiple TiDB Lightning instances, you can import data in parallel from different single tables or multiple tables. In this way, TiDB Lightning provides the ability to scale horizontally, which can greatly reduce the time required to import large amounts of data. + +In technical implementation, TiDB Lightning records the meta data of each instance and the data of each imported table in the target TiDB, and coordinates the Row ID allocation range of different instances, the record of global Checksum, and the configuration changes and recovery of TiKV and PD. + +You can use TiDB Lightning to import data in parallel in the following scenarios: + +- Import sharded schemas and sharded tables. In this scenario, multiple tables from multiple upstream database instances are imported into the downstream TiDB database by different TiDB Lightning instances in parallel. +- Import single tables in parallel. In this scenario, single tables stored in a certain directory or cloud storage (such as Amazon S3) are imported into the downstream TiDB cluster by different TiDB Lightning instances in parallel. This is a new feature introduced in TiDB 5.3.0. + +> **Note:** +> +> Parallel import only supports the initialized empty tables in TiDB. It does not support migrating data to tables with data written by existing services. Otherwise, data inconsistencies may occur. + +The following diagram shows how importing sharded schemas and sharded tables works. In this scenario, you can use multiple TiDB Lightning instances to import MySQL sharded tables to a downstream TiDB cluster. + +![Import sharded schemas and sharded tables](/media/parallel-import-shard-tables-en.png) + +The following diagram shows how importing single tables works. In this scenario, you can use multiple TiDB Lightning instances to split data from a single table and import it in parallel to a downstream TiDB cluster. + +![Import single tables](/media/parallel-import-single-tables-en.png) + +## Considerations + +No additional configuration is required for parallel import using TiDB Lightning. When TiDB Lightning is started, it registers meta data in the downstream TiDB cluster and automatically detects whether there are other instances migrating data to the target cluster at the same time. If there is, it automatically enters the parallel import mode. + +But when migrating data in parallel, you need to take the following into consideration: + +- Handle conflicts between primary keys or unique indexes across multiple sharded tables +- Optimize import performance + +### Handle conflicts between primary keys or unique indexes + +When using [Local-backend mode](/tidb-lightning/tidb-lightning-backends.md#tidb-lightning-local-backend) to import in parallel, you need to ensure that there are no primary key or unique index conflicts between data sources, and between the tables in the target TiDB cluster. Ensure that there are no data writes in the target table during import. Otherwise, TiDB Lightning will not be able to guarantee the correctness of the imported data, and the target table will contain inconsistent indexes after the import is completed. + +### Optimize import performance + +Because TiDB Lightning needs to upload the generated Key-Value data to the TiKV node where each copy of the corresponding Region is located, the import speed is limited by the size of the target cluster. It is recommended to ensure that the number of TiKV instances in the target TiDB cluster and the number of TiDB Lightning instances are greater than n:1 (n is the number of copies of the Region). At the same time, you need to meet the following requirements to achieve the optimal import performance: + +- The total size of source files for each TiDB Lightning instances performing parallel import should be smaller than 5 TiB +- The total number of TiDB Ligntning instances should be smaller than 10 + +When using TiDB Lightning to import shared databases and tables in parallel, choose an appropriate number of TiDB Lightning instances according to the amount of data. + +- If the MySQL data volume is less than 2 TiB, you can use one TiDB Lightning instance for parallel import. +- If the MySQL data volume exceeds 2 TiB and the total number of MySQL instance is smaller than 10, it is recommended that you use one TiDB Lightning instance for each MySQL instance, and the number of parallel TiDB Lightning instances should not exceed 10. +- If the MySQL data volume exceeds 2 TiB and the total number of MySQL instance exceeds 10, it is recommended that you allocate 5 to 10 TiDB Lightning instances for importing the data exported by these MySQL instances. + +Next, this document uses two examples to detail the operation steps of parallel import in different scenarios: + +- Example 1: Use Dumpling + TiDB Lightning to import sharded databases and tables into TiDB in parallel +- Example 2: Import single tables in parallel + +## Example 1: Use Dumpling + TiDB Lightning to Import Sharded Databases and Tables into TiDB in Parallel + +In this example, assume that the upstream is a MySQL cluster with 10 sharded tables, with a total size of 10 TiB. You can Use 5 TiDB Lightning instances to perform parallel import, and each instance imports 2 TiB. It is estimated that the total import time (excluding the time required for Dumpling export) can be reduced from about 40 hours to about 10 hours. + +Assume that the upstream library is named `my_db`, and the name of each sharded table is `my_table_01` ~ `my_table_10`. You want to merge and import them into the downstream `my_db.my_table` table. The specific steps are described in the following sections. + +### Step 1: Use Dumpling to export data + +Export two sharded tables on the 5 nodes where TiDB Lightning is deployed: + +- If the two sharded tables are in the same MySQL instance, you can use the `-f` parameter of Dumpling to directly export them. When using TiDB Lightning to import, you can specify `data-source-dir` as the directory where Dumpling exports data to; +- If the data of the two sharded tables are distributed on different MySQL nodes, you need to use Dumpling to separately export them. The exported data needs to be placed in the same parent directory but in different sub-directories. When using TiDB Lightning to perform parallel import, you need to specify `data-source-dir` as the parent directory. + +For more information on how to use Dumpling to export data, see [Dumpling](/dumpling-overview.md). + +### Step 2: Configure TiDB Lightning data sources + +Create a configuration file `tidb-lightning.toml`, and then add the following content: + +``` +[lightning] +status-addr = ":8289" + +[mydumper] +# Specify the path for Dumpling to export data. If Dumpling performs several times and the data belongs to different directories, you can place all the exported data in the same parent directory and specify this parent directory here. +data-source-dir = "/path/to/source-dir" + +[tikv-importer] +# Use the Local backend mode. +backend = "local" + +# Specify the path for local sorting data. +sorted-kv-dir = "/path/to/sorted-dir" + +# Specify the routes for shard schemas and tables. +[[routes]] +schema-pattern = "my_db" +table-pattern = "my_table_*" +target-schema = "my_db" +target-table = "my_table" +``` + +If the data source is stored in a distributed storage cache such as Amazon S3 or GCS, see [External Storages](/br/backup-and-restore-storages.md). + +### Step 3: Start TiDB Lightning to import data + +During parallel import, the server configuration requirements for each TiDB Lightning node are the same as the non-parallel import mode. Each TiDB Lightning node needs to consume the same resources. It is recommended to deploy them on different servers. For detailed deployment steps, see [Deploy TiDB Lightning](/tidb-lightning/deploy-tidb-lightning.md). + +Start TiDB Lightning on each server in turn. If you use `nohup` to directly start it from the command line, it might exit due to the SIGHUP signal. So it is recommended to put `nohup` in the script, for example: + +```shell +# !/bin/bash +nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & +``` + +During parallel import, TiDB Lightning automatically performs the following checks after starting the task. + +- Check whether there is enough space on the local disk and on the TiKV cluster for importing data. TiDB Lightning samples the data sources and estimates the percentage of the index size from the sample result. Because indexes are included in the estimation, there may be cases where the size of the source data is less than the available space on the local disk, but still the check fails. +- Check whether the regions in the TiKV cluster are distributed evenly and whether there are too many empty regions. If the number of empty regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables ", then the import cannot be executed. +- Check whether the data is imported in order from the data sources. The size of `mydumper.batch-size` is automatically adjusted based on the result of the check. Therefore, the `mydumper.batch-size` configuration is no longer available. + +You can also turn off the check and perform a forced import with the `lightning.check-requirements` configuration. For more detailed checks, see [TiDB Lightning prechecks](/tidb-lightning/tidb-lightning-prechecks.md) + +### Step 4: Check the import progress + +After starting the import, you can check the progress in either of the following ways: + +- Check the progress through the `grep` log keyword `progress`. It is updated every 5 minutes by default. +- Check the progress through the monitoring console. For details, see [TiDB Lightning Monitoring](/tidb-lightning/monitor-tidb-lightning.md). + +Wait for all TiDB Lightning instances to finish, then the entire import is completed. + +## Example 2: Import single tables in parallel + +TiDB Lightning also supports parallel import of single tables. For example, import multiple single tables stored in Amazon S3 by different TiDB Lightning instances into the downstream TiDB cluster in parallel. This method can speed up the overall import speed. For more information on external storages, see [External Storages](/br/backup-and-restore-storages.md)). + +> **Note:** +> +>In the local environment, you can use the --where parameter of Dumpling to divide the data of a single table into different parts and export it to the local disks of multiple servers in advance. This way, you can still perform parallel import. The configuration is the same as Example 1. + +Assuming that the source files are stored in Amazon S3, the table files are `my_db.my_table.00001.sql` ~ `my_db.my_table.10000.sql`, a total of 10,000 SQL files. If you want to use 2 TiDB Lightning instances to speed up the import, you need to add the following settings in the configuration file: + +``` +[[mydumper.files]] +# the db schema file +pattern = '(?i)^(?:[^/]*/)*my_db-schema-create\.sql' +schema = "my_db" +type = "schema-schema" + +[[mydumper.files]] +# the table schema file +pattern = '(?i)^(?:[^/]*/)*my_db\.my_table-schema\.sql' +schema = "my_db" +table = "my_table" +type = "table-schema" + +[[mydumper.files]] +# Only import 00001~05000 and ignore other files +pattern = '(?i)^(?:[^/]*/)*my_db\.my_table\.(0[0-4][0-9][0-9][0-9]|05000)\.sql' +schema = "my_db" +table = "my_table" +type = "sql" +``` + +You can modify the configuration of the other instance to only import the `05001 ~ 10000` data files. + +For other steps, see the relevant steps in Example 1. diff --git a/tidb-lightning/tidb-lightning-faq.md b/tidb-lightning/tidb-lightning-faq.md index d24584f24ae0b..972d4d5d96ef9 100644 --- a/tidb-lightning/tidb-lightning-faq.md +++ b/tidb-lightning/tidb-lightning-faq.md @@ -223,22 +223,21 @@ Try the latest version! Maybe there is new speed improvement. ## `checksum failed: checksum mismatched remote vs local` -**Cause**: The checksum of a table in the local data source and the remote imported database differ. This error has several deeper reasons: +**Cause**: The checksum of a table in the local data source and the remote imported database differ. This error has several deeper reasons. You can further locate the reason by checking the log that contains `checksum mismatched`. -1. The table might already have data before. These old data can affect the final checksum. +The lines that contain `checksum mismatched` provide the information `total_kvs: x vs y`, where `x` indicates the number of key-value pairs (KV pairs) calculated by the target cluster after the import is completed, and `y` indicates the number of key-value pairs generated by the local data source. -2. If the remote checksum is 0, which means nothing is imported, it is possible that the cluster is too hot and fails to take in any data. - -3. If the data is mechanically generated, ensure it respects the constrains of the table: - - * `AUTO_INCREMENT` columns need to be positive, and do not contain the value "0". - * The UNIQUE and PRIMARY KEYs must have no duplicated entries. - -4. If TiDB Lightning has failed before and was not properly restarted, a checksum mismatch may happen due to data being out-of-sync. +- If `x` is greater, it means that there are more KV pairs in the target cluster. + - It is possible that this table is not empty before the import and therefore affects the data checksum. It is also possible that TiDB Lightning has previously failed and shut down, but did not restart correctly. +- If `y` is greater, it means that there are more KV pairs in the local data source. + - If the checksum of the target database is all 0, it means that no import has occurred. It is possible that the cluster is too busy to receive any data. + - It is possible that the exported data contains duplicate data, such as the UNIQUE and PRIMARY KEYs with duplicate values, or that the downstream table structure is case-insensitive while the data is case-sensitive. +- Other possible reasons + - If the data source is machine-generated and not backed up by Dumpling, make sure the data conforms to the table limits. For example, the AUTO_INCREMENT column needs to be positive and not 0. **Solutions**: -1. Delete the corrupted data using `tidb-lightning-ctl`, and restart TiDB Lightning to import the affected tables again. +1. Delete the corrupted data using `tidb-lightning-ctl`, check the table structure and the data, and restart TiDB Lightning to import the affected tables again. {{< copyable "shell-regular" >}} diff --git a/tidb-lightning/tidb-lightning-prechecks.md b/tidb-lightning/tidb-lightning-prechecks.md new file mode 100644 index 0000000000000..88486b1f961ef --- /dev/null +++ b/tidb-lightning/tidb-lightning-prechecks.md @@ -0,0 +1,19 @@ +--- +title: TiDB Lightning Prechecks +summary: This document describes the checks that TiDB Lightning performs before performing a data migration task. These precheckes ensure that TiDB Lightning can perform the task smoothly. +--- + +# TiDB Lightning Prechecks + +Starting from TiDB 5.3.0, TiDB Lightning provides the ability to check the configuration before running a migration task. It is enabled by default. This feature automatically performs some routine checks for disk space and execution configuration. The main purpose is to ensure that the whole subsequent import process goes smoothly. + +The following table describes each check item and detailed explanation. + +| Check Items | Description | +| ---- | ---- | +| Cluster version and status| Check whether the cluster can be connected in the configuration, and whether the TiKV/PD/TiFlash version supports the Local import mode when the backend mode is Local. | +| Disk space | Check whether there is enough space on the local disk and on the TiKV cluster for importing data. TiDB Lightning samples the data sources and estimates the percentage of the index size from the sample result. Because indexes are included in the estimation, there may be cases where the size of the source data is less than the available space on the local disk, but still the check fails. When the backend is Local, it also checks whether the local storage is sufficient because external sorting needs to be done locally. | +| Region distribution status | Check whether the regions in the TiKV cluster are distributed evenly and whether there are too many empty regions. If the number of empty regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables ", then the import cannot be executed. | +| Exceedingly Large CSV files in the data file | When there are CSV files larger than 10 GiB in the backup file and auto-slicing is not enabled (StrictFormat=false), it will impact the import performance. The purpose of this check is to remind you to ensure the data is in the right format and to enable auto-slicing. | +| Recovery from breakpoints | This check ensures that no changes are made to the source file or schema in the database during the breakpoint recovery process that would result in importing the wrong data. | +| Import into an existing table | When importing into an already created table, it checks, as much as possible, whether the source file matches the existing table. Check if the number of columns matches. If the source file has column names, check if the column names match. When there are default columns in the source file, it checks if the default columns have Default Value, and if they have, the check passes. | \ No newline at end of file diff --git a/tidb-monitoring-api.md b/tidb-monitoring-api.md index 13dc89b2f47d7..cc0843a76ac31 100644 --- a/tidb-monitoring-api.md +++ b/tidb-monitoring-api.md @@ -6,15 +6,14 @@ aliases: ['/docs/dev/tidb-monitoring-api/'] # TiDB Monitoring API -You can use the following two types of interfaces to monitor the TiDB cluster state: +You can use the following types of interfaces to monitor the TiDB cluster status: -- [The state interface](#running-status): this interface uses the HTTP interface to get the component information. -- [Storage information](#storage-information): this interface uses the HTTP interface to get the storage information of data tables. +- [The status interface](#use-the-status-interface): this interface uses the HTTP interface to get the component information. Using this interface, you can get the [running status](#running-status) of the current TiDB server and the [storage information](#storage-information) of a table. - [The metrics interface](#use-the-metrics-interface): this interface uses Prometheus to record the detailed information of the various operations in components and views these metrics using Grafana. -## Use the state interface +## Use the status interface -The state interface monitors the basic information of a specific component in the TiDB cluster. It can also act as the monitor interface for Keepalive messages. In addition, the state interface for the Placement Driver (PD) can get the details of the entire TiKV cluster. +The status interface monitors the basic information of a specific component in the TiDB cluster. It can also act as the monitor interface for Keepalive messages. In addition, the status interface for the Placement Driver (PD) can get the details of the entire TiKV cluster. ### TiDB server @@ -23,7 +22,7 @@ The state interface monitors the basic information of a specific component in th ### Running status -The following example uses `http://${host}:${port}/status` to get the current state of the TiDB server and to determine whether the server is alive. The result is returned in **JSON** format. +The following example uses `http://${host}:${port}/status` to get the current status of the TiDB server and to determine whether the server is alive. The result is returned in **JSON** format. ```bash curl http://127.0.0.1:10080/status @@ -82,7 +81,7 @@ curl http://127.0.0.1:10080/schema_storage/test - Default port: `2379` - Details about API names: see [PD API doc](https://download.pingcap.com/pd-api-v1.html) -The PD interface provides the state of all the TiKV servers and the information about load balancing. See the following example for the information about a single-node TiKV cluster: +The PD interface provides the status of all the TiKV servers and the information about load balancing. See the following example for the information about a single-node TiKV cluster: ```bash curl http://127.0.0.1:2379/pd/api/v1/stores @@ -118,7 +117,7 @@ curl http://127.0.0.1:2379/pd/api/v1/stores ## Use the metrics interface -The metrics interface monitors the state and performance of the entire TiDB cluster. +The metrics interface monitors the status and performance of the entire TiDB cluster. - If you use other deployment ways, [deploy Prometheus and Grafana](/deploy-monitoring-services.md) before using this interface. diff --git a/tiflash/use-tiflash.md b/tiflash/use-tiflash.md index 32b05252e262e..4e77c68cab673 100644 --- a/tiflash/use-tiflash.md +++ b/tiflash/use-tiflash.md @@ -242,17 +242,17 @@ In TiDB, operators are organized in a tree structure. For an operator to be push Currently, TiFlash supports the following push-down expressions: -* Mathematical functions: `+, -, /, *, %, >=, <=, =, !=, <, >, round(int), round(double), round(decimal), abs, floor(int), ceil(int), ceiling(int), sqrt, log, log2, log10, ln, exp, pow, sign, radians, degrees, conv, crc32` +* Mathematical functions: `+, -, /, *, %, >=, <=, =, !=, <, >, round, abs, floor(int), ceil(int), ceiling(int), sqrt, log, log2, log10, ln, exp, pow, sign, radians, degrees, conv, crc32` * Logical functions: `and, or, not, case when, if, ifnull, isnull, in, like, coalesce` * Bitwise operations: `bitand, bitor, bigneg, bitxor` -* String functions: `substr, char_length, replace, concat, concat_ws, left, right, ascii, length, trim, position` -* Date functions: `date_format, timestampdiff, from_unixtime, unix_timestamp(int), unix_timestamp(decimal), str_to_date(date), str_to_date(datetime), datediff, year, month, day, extract(datetime), date` +* String functions: `substr, char_length, replace, concat, concat_ws, left, right, ascii, length, trim, ltrim, rtrim, position, format, lower, ucase, upper, substring_index` +* Date functions: `date_format, timestampdiff, from_unixtime, unix_timestamp(int), unix_timestamp(decimal), str_to_date(date), str_to_date(datetime), datediff, year, month, day, extract(datetime), date, hour, microsecond, minute, second, sysdate` * JSON function: `json_length` -* Conversion functions: `cast(int as double), cast(int as decimal), cast(int as string), cast(int as time), cast(double as int), cast(double as decimal), cast(double as string), cast(double as time), cast(string as int), cast(string as double), cast(string as decimal), cast(string as time), cast(decimal as int), cast(decimal as string), cast(decimal as time), cast(time as int), cast(time as decimal), cast(time as string)` -* Aggregate functions: `min, max, sum, count, avg, approx_count_distinct` +* Conversion functions: `cast(int as double), cast(int as decimal), cast(int as string), cast(int as time), cast(double as int), cast(double as decimal), cast(double as string), cast(double as time), cast(string as int), cast(string as double), cast(string as decimal), cast(string as time), cast(decimal as int), cast(decimal as string), cast(decimal as time), cast(time as int), cast(time as decimal), cast(time as string), cast(time as real)` +* Aggregate functions: `min, max, sum, count, avg, approx_count_distinct, group_concat` * Miscellaneous functions: `inetntoa, inetaton, inet6ntoa, inet6aton` -In addition, expressions that contain the Time/Bit/Set/Enum/Geometry type cannot be pushed down to TiFlash. +In addition, expressions that contain the Bit/Set/Geometry type cannot be pushed down to TiFlash. If a query encounters unsupported push-down calculations, TiDB needs to complete the remaining calculations, which might greatly affect the TiFlash acceleration effect. The currently unsupported operators and expressions might be supported in future versions. diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index d071974942829..ae1b01c6ca3a0 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -76,11 +76,11 @@ This document only describes the parameters that are not included in command-lin + The compression algorithm for gRPC messages + Optional values: `"none"`, `"deflate"`, `"gzip"` + Default value: `"none"` -+ Note: When the value is `gzip`, TiDB Dashboard will have a display error because it might not complete the corresponding compression algorithm in some cases. If you adjust the value back to the default `none`, TiDB Dashboard will display normally. ++ Note: When the value is `gzip`, TiDB Dashboard will have a display error because it might not complete the corresponding compression algorithm in some cases. If you adjust the value back to the default `none`, TiDB Dashboard will display normally. ### `grpc-concurrency` -+ The number of gRPC worker threads ++ The number of gRPC worker threads. When you modify the size of the gRPC thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `5` + Minimum value: `1` @@ -102,6 +102,13 @@ This document only describes the parameters that are not included in command-lin + Default: `1` + Minimum value: `1` +### `max-grpc-send-msg-len` + ++ Sets the maximum length of a gRPC message that can be sent ++ Default: `10485760` ++ Unit: Bytes ++ Maximum value: `2147483647` + ### `grpc-stream-initial-window-size` + The window size of the gRPC stream @@ -160,7 +167,7 @@ This document only describes the parameters that are not included in command-lin ### `raft-client-queue-size` -+ Specifies the queue size of the Raft messages in TiKV. If too many messages not sent in time result in a full buffer, or messages discarded, you can specify a greater value to improve system stability. ++ Specifies the queue size of the Raft messages in TiKV. If too many messages not sent in time result in a full buffer, or messages discarded, you can specify a greater value to improve system stability. + Default value: `8192` ## readpool.unified @@ -174,7 +181,7 @@ Configuration items related to the single thread pool serving read requests. Thi ### `max-thread-count` -+ The maximum working thread count of the unified read pool ++ The maximum working thread count of the unified read pool or the UnifyReadPool thread pool. When you modify the size of this thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `MAX(4, CPU * 0.8)` ### `stack-size` @@ -312,7 +319,7 @@ Configuration items related to storage ### `scheduler-worker-pool-size` -+ The number of `scheduler` threads, mainly used for checking transaction consistency before data writing. If the number of CPU cores is greater than or equal to `16`, the default value is `8`; otherwise, the default value is `4`. ++ The number of `scheduler` threads, mainly used for checking transaction consistency before data writing. If the number of CPU cores is greater than or equal to `16`, the default value is `8`; otherwise, the default value is `4`. When you modify the size of the Scheduler thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `4` + Minimum value: `1` @@ -324,7 +331,9 @@ Configuration items related to storage ### `reserve-space` -+ The size of the temporary file that preoccupies the extra space when TiKV is started. The name of temporary file is `space_placeholder_file`, located in the `storage.data-dir` directory. When TiKV runs out of disk space and cannot be started normally, you can delete this file as an emergency intervention and set `reserve-space` to `"0MB"`. ++ When TiKV is started, some space is reserved on the disk as disk protection. When the remaining disk space is less than the reserved space, TiKV restricts some write operations. The reserved space is divided into two parts: 80% of the reserved space is used as the extra disk space required for operations when the disk space is insufficient, and the other 20% is used to store the temporary file. In the process of reclaiming space, if the storage is exhausted by using too much extra disk space, this temporary file serves as the last protection for restoring services. ++ The name of the temporary file is `space_placeholder_file`, located in the `storage.data-dir` directory. When TiKV goes offline because its disk space ran out, if you restart TiKV, the temporary file is automatically deleted and TiKV tries to reclaim the space. ++ When the remaining space is insufficient, TiKV does not create the temporary file. The effectiveness of the protection is related to the size of the reserved space. The size of the reserved space is the larger value between 5% of the disk capacity and this configuration value. When the value of this configuration item is `"0MB"`, TiKV disables this disk protection feature. + Default value: `"5GB"` + Unite: MB|GB @@ -493,12 +502,6 @@ Configuration items related to Raftstore + Default value: `"30s"` + Minimum value: `0` -### `raft-reject-transfer-leader-duration` - -+ The protection time for new nodes, which is used to control the shortest interval to migrate a leader to the newly added node. Setting this value too small might cause the failure of leader transfer. -+ Default value: `"3s"` -+ Minimum value: `0` - ### `hibernate-regions` + Enables or disables Hibernate Region. When this option is enabled, a Region idle for a long time is automatically set as hibernated. This reduces the extra overhead caused by heartbeat messages between the Raft leader and the followers for idle Regions. You can use `peer-stale-state-check-interval` to modify the heartbeat interval between the leader and the followers of hibernated Regions. @@ -682,7 +685,7 @@ Configuration items related to Raftstore ### `apply-pool-size` -+ The allowable number of threads in the pool that flushes data to storage ++ The allowable number of threads in the pool that flushes data to storage. When you modify the size of this thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `2` + Minimum value: greater than `0` @@ -694,10 +697,16 @@ Configuration items related to Raftstore ### `store-pool-size` -+ The allowable number of threads that process Raft ++ The allowable number of threads that process Raft, which is the size of the Raftstore thread pool. When you modify the size of this thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `2` + Minimum value: greater than `0` +### `store-io-pool-size` New in v5.3.0 + ++ The allowable number of threads that process Raft I/O tasks, which is the size of the StoreWriter thread pool. When you modify the size of this thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). ++ Default value: `0` ++ Minimum value: `0` + ### `future-poll-size` + The allowable number of threads that drive `future` @@ -706,16 +715,28 @@ Configuration items related to Raftstore ### `cmd-batch` -+ Controls whether to enable batch processing of the requests. When it is enabled, the write performance is significantly improved. ++ Controls whether to enable batch processing of the requests. When it is enabled, the write performance is significantly improved. + Default value: `true` ### `inspect-interval` -+ At a certain interval, TiKV inspects the latency of the Raftstore component. This parameter specifies the interval of the inspection. If the latency exceeds this value, this inspection is marked as timeout. -+ Judges whether the TiKV node is slow based on the ratio of timeout inspection. ++ At a certain interval, TiKV inspects the latency of the Raftstore component. This parameter specifies the interval of the inspection. If the latency exceeds this value, this inspection is marked as timeout. ++ Judges whether the TiKV node is slow based on the ratio of timeout inspection. + Default value: `"500ms"` + Minimum value: `"1ms"` +### `raft-write-size-limit` New in v5.3.0 + ++ Determines the threshold at which Raft data is written into the disk. If the data size is larger than the value of this configuration item, the data is written to the disk. When the value of `store-io-pool-size` is `0`, this configuration item does not take effect. ++ Default value: `1MB` ++ Minimum value: `0` + +### `raft-msg-flush-interval` New in v5.3.0 + ++ Determines the interval at which Raft messages are sent in batches. The Raft messages in batches are sent at every interval specified by this configuration item. When the value of `store-io-pool-size` is `0`, this configuration item does not take effect. ++ Default value: `250us` ++ Minimum value: `0` + ## Coprocessor Configuration items related to Coprocessor @@ -759,7 +780,7 @@ Configuration items related to RocksDB ### `max-background-jobs` -+ The number of background threads in RocksDB ++ The number of background threads in RocksDB. When you modify the size of the RocksDB thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `8` + Minimum value: `2` @@ -1233,7 +1254,7 @@ Configuration items related to `raftdb` ### `max-background-jobs` -+ The number of background threads in RocksDB ++ The number of background threads in RocksDB. When you modify the size of the RocksDB thread pool, refer to [Performance tuning for TiKV thread pools](/tune-tikv-thread-performance.md#performance-tuning-for-tikv-thread-pools). + Default value: `4` + Minimum value: `2` @@ -1259,14 +1280,19 @@ Configuration items related to security ### `cert-path` -+ The path of the Privacy Enhanced Mail (PEM) file that contains the X509 certificate ++ The path of the Privacy Enhanced Mail (PEM) file that contains the X.509 certificate + Default value: "" ### `key-path` -+ The path of the PEM file that contains the X509 key ++ The path of the PEM file that contains the X.509 key + Default value: "" +### `cert-allowed-cn` + ++ A list of acceptable X.509 Common Names in certificates presented by clients. Requests are permitted only when the presented Common Name is an exact match with one of the entries in the list. ++ Default value: `[]`. This means that the client certificate CN check is disabled by default. + ### `redact-info-log` New in v4.0.8 + This configuration item enables or disables log redaction. If the configuration value is set to `true`, all user data in the log will be replaced by `?`. diff --git a/tikv-control.md b/tikv-control.md index d5cb2f52325b7..87bda9a97b007 100644 --- a/tikv-control.md +++ b/tikv-control.md @@ -405,22 +405,40 @@ tikv-ctl --host ip:port modify-tikv-config -n rocksdb.rate-bytes-per-sec -v "1GB success ``` -### Force Region to recover the service from failure of multiple replicas +### Force Regions to recover services from failure of multiple replicas (use with caution) -Use the `unsafe-recover remove-fail-stores` command to remove the failed machines from the peer list of Regions. Then after you restart TiKV, these Regions can continue to provide services using the other healthy replicas. This command is usually used in circumstances where multiple TiKV stores are damaged or deleted. +You can use the `unsafe-recover remove-fail-stores` command to remove the failed machines from the peer list of Regions. Before running this command, you need to stop the service of the target TiKV store to release file locks. -The `-s` option accepts multiple `store_id` separated by comma and uses the `-r` flag to specify involved Regions. Otherwise, all Regions' peers located on these stores will be removed by default. +The `-s` option accepts multiple `store_id` separated by comma and uses the `-r` flag to specify involved Regions. If you need to perform this operation on all Regions in a specific store, you can simply specify `--all-regions`. -```bash -$ tikv-ctl --db /path/to/tikv/db unsafe-recover remove-fail-stores -s 3 -r 1001,1002 +> **Warning:** +> +> - If any misoperation is performed, it might be hard to recover the cluster. Be aware of the potential risks and avoid using this feature in a production environment. +> - If the `--all-regions` option is used, you are expected to run this command on all the remaining stores connected to the cluster. You need to ensure that these healthy stores stop providing services before recovering the damaged stores. Otherwise, the inconsistent peer lists in Region replicas will cause errors when you run `split-region` or `remove-peer`. This further causes inconsistency between other metadata, and finally, the Regions will become unavailable. +> - Once you have run `remove-fail-stores`, you cannot restart the removed nodes or add these nodes to the cluster. Otherwise, the metadata will be inconsistent, and finally, the Regions will be unavailable. + +{{< copyable "shell-regular" >}} + +```shell +tikv-ctl --db /path/to/tikv/db unsafe-recover remove-fail-stores -s 3 -r 1001,1002 +``` + +``` success! ``` +{{< copyable "shell-regular" >}} + +```shell +tikv-ctl --db /path/to/tikv/db unsafe-recover remove-fail-stores -s 4,5 --all-regions +``` + +Then, after you restart TiKV, the Regions can continue providing services with the remaining healthy replicas. This command is commonly used when multiple TiKV stores are damaged or deleted. + > **Note:** > +> - You are expected to run this command for all stores where the specified Regions' peers are located. > - This command only supports the local mode. It prints `success!` when successfully run. -> - You must run this command for all stores where specified Regions' peers are located. -> - If the `--all-regions` option is used, usually you need to run this command on all the remaining healthy stores in the cluster. You need to ensure that the healthy stores stop providing services before recovering the damaged stores. Otherwise, the inconsistent peer lists in Region replicas will cause errors when you execute `split-region` or `remove-peer`. This further causes inconsistency between other metadata, and finally, the Regions will become unavailable. ### Recover from MVCC data corruption diff --git a/tiup/tiup-cluster-topology-reference.md b/tiup/tiup-cluster-topology-reference.md index bdfe33bff30d4..4c774db07b347 100644 --- a/tiup/tiup-cluster-topology-reference.md +++ b/tiup/tiup-cluster-topology-reference.md @@ -24,7 +24,7 @@ A topology configuration file for TiDB deployment using TiUP might contain the f - [cdc_servers](#cdc_servers): The configuration of the TiCDC instance. This configuration specifies the machines to which the TiCDC component is deployed. - [tispark_masters](#tispark_masters): The configuration of the TiSpark master instance. This configuration specifies the machines to which the TiSpark master component is deployed. Only one node of TiSpark master can be deployed. - [tispark_workers](#tispark_workers): The configuration of the TiSpark worker instance. This configuration specifies the machines to which the TiSpark worker component is deployed. -- [monitoring_servers](#monitoring_servers): Specifies the machines to which Prometheus is deployed. TiUP supports deploying multiple Prometheus instances but only the first instance is used. +- [monitoring_servers](#monitoring_servers): Specifies the machines to which Prometheus and NGMonitoring are deployed. TiUP supports deploying multiple Prometheus instances but only the first instance is used. - [grafana_servers](#grafana_servers): The configuration of the Grafana instance. This configuration specifies the machines to which Grafana is deployed. - [alertmanager_servers](#alertmanager_servers): The configuration of the Alertmanager instance. This configuration specifies the machines to which Alertmanager is deployed. @@ -518,8 +518,6 @@ For the above fields, you cannot modify these configured fields after the deploy - `deploy_dir` - `data_dir` - `log_dir` -- `gc-ttl` -- `tz` - `arch` - `os` @@ -529,7 +527,10 @@ A `cdc_servers` configuration example is as follows: cdc_servers: - host: 10.0.1.20 gc-ttl: 86400 + data_dir: "/cdc-data" - host: 10.0.1.21 + gc-ttl: 86400 + data_dir: "/cdc-data" ``` ### `tispark_masters` @@ -635,6 +636,8 @@ tispark_workers: - `host`: Specifies the machine to which the monitoring services are deployed. The field value is an IP address and is mandatory. +- `ng_port`: Specifies the SSH port connecting to NGMonitoring. Introduced in TiUP v1.7.0, this field supports [Continuous Profiling](/dashboard/dashboard-profiling.md) and Top SQL in TiDB 5.3.0 and above. + - `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. - `port`: The listening port of the Prometheus services. The default value is `9090`. diff --git a/tiup/tiup-component-cluster-deploy.md b/tiup/tiup-component-cluster-deploy.md index 92acb198c48c3..be176146b3ff0 100644 --- a/tiup/tiup-component-cluster-deploy.md +++ b/tiup/tiup-component-cluster-deploy.md @@ -13,7 +13,7 @@ tiup cluster deploy [flags] ``` - ``: the name of the new cluster, which cannot be the same as the existing cluster names. -- ``: the version number of the TiDB cluster to deploy, such as `v5.2.1`. +- ``: the version number of the TiDB cluster to deploy, such as `v5.3.0`. - ``: the prepared [topology file](/tiup/tiup-cluster-topology-reference.md). ## Options diff --git a/tiup/tiup-component-dm-import.md b/tiup/tiup-component-dm-import.md index b84c1de36a198..f5a99bf3000ab 100644 --- a/tiup/tiup-component-dm-import.md +++ b/tiup/tiup-component-dm-import.md @@ -12,7 +12,7 @@ In DM v1.0, the cluster is basically deployed using TiDB Ansible. TiUP DM provid > - Before importing the cluster, stop running the original cluster first. > - For data migration tasks that need to be upgraded to v2.0, do not execute `stop-task` on these tasks. > - The command only supports importing to DM v2.0.0-rc.2 and later versions. -> - The `import` command is used to import a DM v1.0 cluster to a new DM v2.0 cluster. If you need to import data migration tasks to an existing v2.0 cluster, refer to [Manually Upgrade TiDB Data Migration from v1.0.x to v2.0.x](https://docs.pingcap.com/tidb-data-migration/stable/manually-upgrade-dm-1.0-to-2.0) +> - The `import` command is used to import a DM v1.0 cluster to a new DM v2.0 cluster. If you need to import data migration tasks to an existing v2.0 cluster, refer to [Manually Upgrade TiDB Data Migration from v1.0.x to v2.0+](https://docs.pingcap.com/tidb-data-migration/stable/manually-upgrade-dm-1.0-to-2.0) > - The deployment directories of some components might be different from those in the original cluster. You can check it with the `display` command. > - Before importing the cluster, run `tiup update --self && tiup update dm` to upgrade TiUP DM components to the latest version. > - After the cluster is imported, there is only one DM-master node in the cluster. You can refer to [the `scale out` command](/tiup/tiup-component-dm-scale-out.md) to scale out the DM-master node. diff --git a/tiup/tiup-component-management.md b/tiup/tiup-component-management.md index 0aa46aca25faa..5542b5ac6d755 100644 --- a/tiup/tiup-component-management.md +++ b/tiup/tiup-component-management.md @@ -70,12 +70,12 @@ Example 2: Use TiUP to install the nightly version of TiDB. tiup install tidb:nightly ``` -Example 3: Use TiUP to install TiKV v5.2.1. +Example 3: Use TiUP to install TiKV v5.3.0. {{< copyable "shell-regular" >}} ```shell -tiup install tikv:v5.2.1 +tiup install tikv:v5.3.0 ``` ## Upgrade components @@ -128,12 +128,12 @@ Before the component is started, TiUP creates a directory for it, and then puts If you want to start the same component multiple times and reuse the previous working directory, you can use `--tag` to specify the same name when the component is started. After the tag is specified, the working directory will *not be automatically deleted* when the instance is terminated, which makes it convenient to reuse the working directory. -Example 1: Operate TiDB v5.2.1. +Example 1: Operate TiDB v5.3.0. {{< copyable "shell-regular" >}} ```shell -tiup tidb:v5.2.1 +tiup tidb:v5.3.0 ``` Example 2: Specify the tag with which TiKV operates. @@ -219,12 +219,12 @@ The following flags are supported in this command: - If the version is ignored, adding `--all` means to uninstall all versions of this component. - If the version and the component are both ignored, adding `--all` means to uninstall all components of all versions. -Example 1: Uninstall TiDB v5.2.1. +Example 1: Uninstall TiDB v5.3.0. {{< copyable "shell-regular" >}} ```shell -tiup uninstall tidb:v5.2.1 +tiup uninstall tidb:v5.3.0 ``` Example 2: Uninstall TiKV of all versions. diff --git a/tiup/tiup-mirror.md b/tiup/tiup-mirror.md index a3c626211b98f..1f35eb23d725b 100644 --- a/tiup/tiup-mirror.md +++ b/tiup/tiup-mirror.md @@ -77,9 +77,9 @@ The `tiup mirror clone` command provides many optional flags (might provide more If you want to clone only one version (not all versions) of a component, use `--=` to specify this version. For example: - - Execute the `tiup mirror clone --tidb v5.2.1` command to clone the v5.2.1 version of the TiDB component. - - Execute the `tiup mirror clone --tidb v5.2.1 --tikv all` command to clone the v5.2.1 version of the TiDB component and all versions of the TiKV component. - - Execute the `tiup mirror clone v5.2.1` command to clone the v5.2.1 version of all components in a cluster. + - Execute the `tiup mirror clone --tidb v5.3.0` command to clone the v5.3.0 version of the TiDB component. + - Execute the `tiup mirror clone --tidb v5.3.0 --tikv all` command to clone the v5.3.0 version of the TiDB component and all versions of the TiKV component. + - Execute the `tiup mirror clone v5.3.0` command to clone the v5.3.0 version of all components in a cluster. ## Usage examples diff --git a/tiup/tiup-playground.md b/tiup/tiup-playground.md index 582bbd080827c..f4919a4e7bcd7 100644 --- a/tiup/tiup-playground.md +++ b/tiup/tiup-playground.md @@ -16,42 +16,52 @@ The basic usage of the playground component is shown as follows: tiup playground [version] [flags] ``` -If you directly execute the `tiup playground` command, TiUP uses the locally installed TiDB, TiKV, and PD components or installs the stable version of these components to start a TiDB cluster that consists of one TiKV instance, one TiDB instance, and one PD instance. +If you directly execute the `tiup playground` command, TiUP uses the locally installed TiDB, TiKV, and PD components or installs the stable version of these components to start a TiDB cluster that consists of one TiKV instance, one TiDB instance, one PD instance, and one TiFlash instance. This command actually performs the following operations: -- Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.5.0, then this command works the same as `tiup playground:v1.5.0`. +- Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.7.0, then this command works the same as `tiup playground:v1.7.0`. - If you have not used TiUP playground to install the TiDB, TiKV, and PD components, the playground component installs the latest stable version of these components, and then start these instances. -- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v5.0.0, then this command works the same as `tiup playground:v1.5.0 v5.2.1`. -- Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, and one PD instance. +- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v5.3.0, then this command works the same as `tiup playground:v1.7.0 v5.3.0`. +- Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, one PD instance, and one TiFlash instance. - After starting each TiDB component, TiUP playground reminds you that the cluster is successfully started and provides you some useful information, such as how to connect to the TiDB cluster through the MySQL client and how to access the [TiDB Dashboard](/dashboard/dashboard-intro.md). The command-line flags of the playground component are described as follows: ```bash Flags: - --db int TiDB instance number (default 1) - --db.binpath string TiDB instance binary path - --db.config string TiDB instance configuration file - --db.host host Set the listening address of TiDB - --drainer int Set the Drainer data of the cluster + --db int Specify the number of TiDB instances (default: 1) + ----db.host host Specify the listening address of TiDB + ----db.port int Specify the port of TiDB + --db.binpath string Specify the TiDB instance binary path (optional, for debugging) + --db.config string Specify the TiDB instance configuration file (optional, for debugging) + --db.timeout int Specify TiDB maximum wait time in seconds for starting. 0 means no limit + --drainer int Specify Drainer data of the cluster --drainer.binpath string Specify the location of the Drainer binary files (optional, for debugging) --drainer.config string Specify the Drainer configuration file -h, --help help for tiup - --host string Playground cluster host (default "127.0.0.1") - --kv int TiKV instance number (default 1) - --kv.binpath string TiKV instance binary path - --kv.config string TiKV instance configuration file - --monitor Start prometheus component - --pd int PD instance number (default 1) - --pd.binpath string PD instance binary path - --pd.config string PD instance configuration file - --pump int Specify the number of Pump nodes in the cluster. If the value is not "0", TiDB Binlog is enabled. + --host string Specify the listening address of each component (default: `127.0.0.1`). Set it to `0.0.0.0` if provided for access of other machines + --kv int Specify the number of TiKV instances (default: 1) + --kv.binpath string Specify the TiKV instance binary path (optional, for debugging) + --kv.config string Specify the TiKV instance configuration file (optional, for debugging) + --mode string Specify the playground mode: 'tidb' (default) and 'tikv-slim' + --pd int Specify the number of PD instances (default: 1) + --pd.host host Specify the listening address of PD + --pd.binpath string Specify the PD instance binary path (optional, for debugging) + --pd.config string Specify the PD instance configuration file (optional, for debugging) + --pump int Specify the number of Pump instances. If the value is not `0`, TiDB Binlog is enabled. --pump.binpath string Specify the location of the Pump binary files (optional, for debugging) --pump.config string Specify the Pump configuration file (optional, for debugging) - --tiflash int TiFlash instance number - --tiflash.binpath string TiFlash instance binary path - --tiflash.config string TiFlash instance configuration file + -T, --tag string Specify a tag for playground + --ticdc int Specify the number of TiCDC instances (default: 0) + --ticdc.binpath string Specify the TiCDC instance binary path (optional, for debugging) + --ticdc.config string Specify the TiCDC instance configuration file (optional, for debugging) + --tiflash int Specify the number of TiFlash instances (default: 1) + --tiflash.binpath string Specify the TiFlash instance binary path (optional, for debugging) + --tiflash.config string Specify the TiFlash instance configuration file (optional, for debugging) + --tiflash.timeout int Specify TiFlash maximum wait time in seconds for starting. 0 means no limit + -v, --version Specify the version of playground + --without-monitor Disable the monitoring function of Prometheus and Grafana. If you do not add this flag, the monitoring function is enabled by default. ``` ## Examples @@ -64,14 +74,14 @@ Flags: tiup playground nightly ``` -In the command above, `nightly` is the version number of the cluster. Similarly, you can replace `nightly` with `v5.2.1`, and the command is `tiup playground v5.2.1`. +In the command above, `nightly` indicates the latest development version of TiDB. ### Start a cluster with monitor {{< copyable "shell-regular" >}} ```shell -tiup playground nightly --monitor +tiup playground nightly ``` This command starts Prometheus on port 9090 to display the time series data in the cluster. @@ -103,7 +113,7 @@ By default, only one instance is started for each TiDB, TiKV, and PD component. {{< copyable "shell-regular" >}} ```shell -tiup playground v3.0.10 --db 3 --pd 3 --kv 3 +tiup playground v5.3.0 --db 3 --pd 3 --kv 3 ``` ## Quickly connect to the TiDB cluster started by playground @@ -149,7 +159,7 @@ The command-line parameter for scaling out a cluster is similar to that for star tiup playground scale-out --db 2 ``` -## Scale in clusters +## Scale in a cluster You can specify a `pid` in the `tiup playground scale-in` command to scale in the corresponding instance. To view the `pid`, execute `tiup playground display`. diff --git a/troubleshoot-lock-conflicts.md b/troubleshoot-lock-conflicts.md index eb3c31ff23f8c..14bf3ef550597 100644 --- a/troubleshoot-lock-conflicts.md +++ b/troubleshoot-lock-conflicts.md @@ -133,8 +133,8 @@ Solutions: Checking the time interval using the PD control tool: ```shell - ./pd-ctl tso [start_ts] - ./pd-ctl tso [commit_ts] + tiup ctl pd tso [start_ts] + tiup ctl pd tso [commit_ts] ``` * It is recommended to check whether the write performance is slow, which might cause that the efficiency of transaction commit is poor, and thus the lock is cleared. diff --git a/troubleshoot-write-conflicts.md b/troubleshoot-write-conflicts.md index 944c42dbba4cc..adea5ee563210 100644 --- a/troubleshoot-write-conflicts.md +++ b/troubleshoot-write-conflicts.md @@ -70,7 +70,7 @@ You can use the `pd-ctl` tool to convert the timestamp to readable time: {{< copyable "" >}} ```shell -./pd-ctl -u https://127.0.0.1:2379 tso {TIMESTAMP} +tiup ctl pd -u https://127.0.0.1:2379 tso {TIMESTAMP} ``` You can use `tableID` to find the name of the related table: @@ -86,7 +86,7 @@ You can use `indexID` and the table name to find the name of the related index: {{< copyable "sql" >}} ```sql -SELECT * FROM INFORMATION_SCHEMA.TIDB_INDEXES WHERE TABLE_SCHEMA='{table_name}' AND TABLE_NAME='{table_name}' AND INDEX_ID={indexID}; +SELECT * FROM INFORMATION_SCHEMA.TIDB_INDEXES WHERE TABLE_SCHEMA='{db_name}' AND TABLE_NAME='{table_name}' AND INDEX_ID={indexID}; ``` In addition, in TiDB v3.0.8 and later versions, the pessimistic transaction becomes the default model. The pessimistic transaction model can avoid write conflicts during the transaction prewrite stage, so you do not need to modify the application any more. In the pessimistic transaction mode, each DML statement writes a pessimistic lock to the related keys during execution. This pessimistic lock can prevent other transactions from modifying the same keys, thus ensuring no write conflicts exist in the `prewrite` stage of the transaction 2PC. diff --git a/tune-tikv-thread-performance.md b/tune-tikv-thread-performance.md index 0ed2c23d20b4c..6b9205d1c8b36 100644 --- a/tune-tikv-thread-performance.md +++ b/tune-tikv-thread-performance.md @@ -10,13 +10,19 @@ This document introduces TiKV internal thread pools and how to tune their perfor ## Thread pool introduction -The TiKV thread pool is mainly composed of gRPC, Scheduler, UnifyReadPool, Raftstore, Apply, RocksDB, and some scheduled tasks and detection components that do not consume much CPU. This document mainly introduces a few CPU-intensive thread pools that affect the performance of read and write requests. +The TiKV thread pool is mainly composed of gRPC, Scheduler, UnifyReadPool, Raftstore, StoreWriter, Apply, RocksDB, and some scheduled tasks and detection components that do not consume much CPU. This document mainly introduces a few CPU-intensive thread pools that affect the performance of read and write requests. * The gRPC thread pool: it handles all network requests and forwards requests of different task types to different thread pools. * The Scheduler thread pool: it detects write transaction conflicts, converts requests like the two-phase commit, pessimistic locking, and transaction rollbacks into key-value pair arrays, and then sends them to the Raftstore thread for Raft log replication. -* The Raftstore thread pool: it processes all Raft messages and the proposal to add a new log, and writing the log to a disk. When the logs in the majority of replicas are consistent, this thread pool sends the log to the Apply thread. +* The Raftstore thread pool: + + - It processes all Raft messages and the proposal to add a new log. + - It writes Raft logs to the disk. If the value of [`store-io-pool-size`](/tikv-configuration-file.md#store-io-pool-size-new-in-v530) is `0`, the Raftstore thread writes the logs to the disk; if the value is not `0`, the Raftstore thread sends the logs to the StoreWriter thread. + - When Raft logs in the majority of replicas are consistent, the Raftstore thread sends the logs to the Apply thread. + +* The StoreWriter thread pool: it writes all Raft logs to the disk and returns the result to the Raftstore thread. * The Apply thread pool: it receives the submitted log sent from the Raftstore thread pool, parses it as a key-value request, then writes it to RocksDB, calls the callback function to notify the gRPC thread pool that the write request is complete, and returns the result to the client. @@ -55,9 +61,22 @@ Starting from TiKV v5.0, all read requests use the unified thread pool for queri * The Raftstore thread pool. - The Raftstore thread pool is the most complex thread pool in TiKV. The default size (configured by `raftstore.store-pool-size`) is `2`. All write requests are written into RocksDB in the way of `fsync` from the Raftstore thread. + The Raftstore thread pool is the most complex thread pool in TiKV. The default size (configured by `raftstore.store-pool-size`) of this thread pool is `2`. For the StoreWriter thread pool, the default size (configured by `raftstore.store-io-pool-size`) is `0`. + + - When the size of the StoreWriter thread pool is 0, all write requests are written into RocksDB in the way of `fsync` by the Raftstore thread. In this case, it is recommended to tune the performance as follows: + + - Keep the overall CPU usage of the Raftstore thread below 60%. When the number of Raftstore threads is 2, keep the **TiKV-Details**, **Thread CPU**, **Raft store CPU** on Grafana below 120%. Due to I/O requests, the CPU usage of Raftstore threads in theory is always lower than 100%. + - Do not increase the size of the Raftstore thread pool to improve write performance without careful consideration, because this might increase the disk burden and degrade performance. + + - When the size of the StoreWriter thread pool is not 0, all write requests are written into RocksDB in the way of `fsync` by the StoreWriter thread. In this case, it is recommended to tune the performance as follows: + + - Enable the StoreWriter thread pool ONLY when the overall CPU resources are sufficient. When the StoreWriter thread pool is enabled, keep the CPU usage of the StoreWriter thread and the Raftstore thread below 80%. + + Compared with the case that the write requests are processed by the Raftstore thread, in theory, when the write requests are processed by the StoreWriter thread, write latency and the tail latency of data read are significantly reduced. However, as the write speed grows faster, the number of Raft logs increases accordingly. This can cause the CPU overhead of the Raftstore threads, the Apply threads, and the gRPC threads to increase. In this case, insufficient CPU resources might offset the tuning effect, and as a result, the write speed might become slower than before. Therefore, if the CPU resources are not sufficient, it is not recommended to enable the StoreWriter thread. Because the Raftstore thread sends most of the I/O requests to the StoreWriter thread, you need to keep the CPU usage of the Raftstore thread below 80%. + + - In most cases, set the size of the StoreWriter thread pool to 1 or 2. This is because the size of the StoreWriter thread pool affects the number of Raft logs, so the value of the thread pool size should not be too large. If the CPU usage is higher than 80%, consider increasing the thread pool size. - Due to I/O, Raftstore threads cannot reach 100% CPU usage theoretically. To reduce disk writes as much as possible, you can put together multiple write requests and write them to RocksDB. It is recommended to keep the overall CPU usage below 60% (If the default number of threads is `2`, it is recommended to keep `TiKV-Details.Thread CPU.Raft store CPU` on Grafana within 120%). Do not increase the size of the Raftstore thread pool to improve write performance without thinking, because this might increase the disk burden and degrade performance. + - Pay attention to the impact of increasing Raft logs on the CPU overhead of other thread pools. If necessary, you need to increase the number of Raftstore threads, Apply threads, and gRPC threads accordingly. * The UnifyReadPool thread pool. diff --git a/upgrade-tidb-using-tiup.md b/upgrade-tidb-using-tiup.md index a491cf3b8d5f4..d55e00889b1d2 100644 --- a/upgrade-tidb-using-tiup.md +++ b/upgrade-tidb-using-tiup.md @@ -8,23 +8,24 @@ aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-t This document is targeted for the following upgrade paths: -- Upgrade from TiDB 4.0 versions to TiDB 5.2 versions. -- Upgrade from TiDB 5.0 versions to TiDB 5.2 versions. -- Upgrade from TiDB 5.2 versions to TiDB 5.2 versions. +- Upgrade from TiDB 4.0 versions to TiDB 5.3 versions. +- Upgrade from TiDB 5.0 versions to TiDB 5.3 versions. +- Upgrade from TiDB 5.1 versions to TiDB 5.3 versions. +- Upgrade from TiDB 5.2 versions to TiDB 5.3 versions. > **Note:** > -> If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v5.2 or its patch versions is not supported. You need to upgrade your cluster first to v4.0 and then to v5.2. +> If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v5.3 or its patch versions is not supported. You need to upgrade your cluster first to v4.0 and then to v5.3. ## Upgrade caveat - TiDB currently does not support version downgrade or rolling back to an earlier version after the upgrade. -- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v5.2 or its patch versions according to this document. -- To update versions earlier than 3.0 to 5.2: +- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v5.3 or its patch versions according to this document. +- To update versions earlier than 3.0 to 5.3: 1. Update this version to 3.0 using [TiDB Ansible](https://docs.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). 2. Use TiUP (`tiup cluster`) to import the TiDB Ansible configuration. 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). - 4. Upgrade the cluster to v5.2 according to this document. + 4. Upgrade the cluster to v5.3 according to this document. - Support upgrading the versions of TiDB Binlog, TiCDC, TiFlash, and other components. - For detailed compatibility changes of different versions, see the [Release Notes](/releases/release-notes.md) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. @@ -46,7 +47,7 @@ Before upgrading your TiDB cluster, you first need to upgrade TiUP or TiUP mirro > > If the control machine of the cluster to upgrade cannot access `https://tiup-mirrors.pingcap.com`, skip this section and see [Upgrade TiUP offline mirror](#upgrade-tiup-offline-mirror). -1. Upgrade the TiUP version. It is recommended that the TiUP version is `1.5.0` or later. +1. Upgrade the TiUP version. It is recommended that the TiUP version is `1.7.0` or later. {{< copyable "shell-regular" >}} @@ -55,7 +56,7 @@ Before upgrading your TiDB cluster, you first need to upgrade TiUP or TiUP mirro tiup --version ``` -2. Upgrade the TiUP Cluster version. It is recommended that the TiUP Cluster version is `1.5.0` or later. +2. Upgrade the TiUP Cluster version. It is recommended that the TiUP Cluster version is `1.7.0` or later. {{< copyable "shell-regular" >}} @@ -97,7 +98,7 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > Skip this step if one of the following situations applies: > > + You have not modified the configuration parameters of the original cluster. Or you have modified the configuration parameters using `tiup cluster` but no more modification is needed. -> + After the upgrade, you want to use v5.2's default parameter values for the unmodified configuration items. +> + After the upgrade, you want to use v5.3's default parameter values for the unmodified configuration items. 1. Enter the `vi` editing mode to edit the topology file: @@ -113,9 +114,9 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > **Note:** > -> Before you upgrade the cluster to v5.2, make sure that the parameters you have modified in v4.0 are compatible in v5.2. For details, see [TiKV Configuration File](/tikv-configuration-file.md). +> Before you upgrade the cluster to v5.3, make sure that the parameters you have modified in v4.0 are compatible in v5.3. For details, see [TiKV Configuration File](/tikv-configuration-file.md). > -> The following three TiKV parameters are obsolete in TiDB v5.2. If the following parameters have been configured in your original cluster, you need to delete these parameters through `edit-config`: +> The following three TiKV parameters are obsolete in TiDB v5.3. If the following parameters have been configured in your original cluster, you need to delete these parameters through `edit-config`: > > - pessimistic-txn.enabled > - server.request-batch-enable-cross-command @@ -156,12 +157,12 @@ If your application has a maintenance window for the database to be stopped for tiup cluster upgrade ``` -For example, if you want to upgrade the cluster to v5.2.1: +For example, if you want to upgrade the cluster to v5.3.0: {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v5.2.1 +tiup cluster upgrade v5.3.0 ``` > **Note:** @@ -211,7 +212,7 @@ tiup cluster display ``` Cluster type: tidb Cluster name: -Cluster version: v5.2.1 +Cluster version: v5.3.0 ``` > **Note:** @@ -261,10 +262,10 @@ You can upgrade the tool version by using TiUP to install the `ctl` component of {{< copyable "shell-regular" >}} ```shell -tiup install ctl:v5.2.1 +tiup install ctl:v5.3.0 ``` -## TiDB 5.2 compatibility changes +## TiDB 5.3 compatibility changes -- See TiDB 5.2 Release Notes for the compatibility changes. +- See TiDB 5.3 Release Notes for the compatibility changes. - Try to avoid creating a new clustered index table when you apply rolling updates to the clusters using TiDB Binlog.