From 36e7d2fd08b1fbe356cf18de48159ca2fae3ff90 Mon Sep 17 00:00:00 2001 From: Suhaha Date: Wed, 16 Feb 2022 10:57:28 +0800 Subject: [PATCH] fix: missing fields in slow query (#1150) * fix: missing fields in slowquery * fix: lower case field names in tests * test: slow query available fields * test: move available fields tests to compatibility test * chore: remove test log * tweak: json names len --- pkg/apiserver/slowquery/model.go | 25 +++-- pkg/apiserver/slowquery/queries.go | 12 ++- pkg/apiserver/slowquery/service.go | 15 +-- pkg/apiserver/statement/models.go | 26 ++++-- pkg/apiserver/statement/service.go | 20 ++-- .../slowquery/compatibility_test.go | 91 ++++++++++++++++--- .../slow_query/list.compat_spec.js | 84 +++++++++++++++++ .../apps/SlowQuery/utils/useSchemaColumns.ts | 2 +- .../apps/Statement/utils/useSchemaColumns.ts | 2 +- ui/lib/components/ColumnsSelector/index.tsx | 3 +- ui/package.json | 1 + ui/yarn.lock | 5 + 12 files changed, 236 insertions(+), 50 deletions(-) create mode 100644 ui/cypress/integration/slow_query/list.compat_spec.js diff --git a/pkg/apiserver/slowquery/model.go b/pkg/apiserver/slowquery/model.go index e98980c222..dff9cea9b3 100644 --- a/pkg/apiserver/slowquery/model.go +++ b/pkg/apiserver/slowquery/model.go @@ -3,7 +3,7 @@ package slowquery import ( - "github.com/thoas/go-funk" + "strings" "github.com/pingcap/tidb-dashboard/pkg/apiserver/utils" "github.com/pingcap/tidb-dashboard/util/reflectutil" @@ -112,12 +112,19 @@ func getFieldsAndTags() (slowQueryFields []Field) { return } -func getVirtualFields() []string { - fields := getFieldsAndTags() - vFields := funk.Filter(fields, func(f Field) bool { - return f.Projection != "" - }).([]Field) - return funk.Map(vFields, func(f Field) string { - return f.ColumnName - }).([]string) +func filterFieldsByColumns(fields []Field, columns []string) []Field { + colMap := map[string]struct{}{} + for _, c := range columns { + colMap[strings.ToLower(c)] = struct{}{} + } + + filteredFields := []Field{} + for _, f := range fields { + _, ok := colMap[strings.ToLower(f.ColumnName)] + if ok || (f.Projection != "") { + filteredFields = append(filteredFields, f) + } + } + + return filteredFields } diff --git a/pkg/apiserver/slowquery/queries.go b/pkg/apiserver/slowquery/queries.go index 8958233d93..1ffda65d39 100644 --- a/pkg/apiserver/slowquery/queries.go +++ b/pkg/apiserver/slowquery/queries.go @@ -5,7 +5,6 @@ package slowquery import ( "strings" - "github.com/thoas/go-funk" "gorm.io/gorm" "github.com/pingcap/tidb-dashboard/pkg/utils" @@ -121,10 +120,17 @@ func QuerySlowLogDetail(req *GetDetailRequest, db *gorm.DB) (*Model, error) { return &result, nil } -func QueryTableColumns(sysSchema *utils.SysSchema, db *gorm.DB) ([]string, error) { +func GetAvailableFields(sysSchema *utils.SysSchema, db *gorm.DB) ([]string, error) { cs, err := sysSchema.GetTableColumnNames(db, SlowQueryTable) if err != nil { return nil, err } - return funk.UniqString(append(cs, getVirtualFields()...)), nil + + fields := filterFieldsByColumns(getFieldsAndTags(), cs) + jsonNames := make([]string, 0, len(fields)) + for _, f := range fields { + jsonNames = append(jsonNames, f.JSONName) + } + + return jsonNames, nil } diff --git a/pkg/apiserver/slowquery/service.go b/pkg/apiserver/slowquery/service.go index 577a5a6d56..c5a35b1b93 100644 --- a/pkg/apiserver/slowquery/service.go +++ b/pkg/apiserver/slowquery/service.go @@ -51,7 +51,7 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint.POST("/download/token", s.downloadTokenHandler) - endpoint.GET("/table_columns", s.queryTableColumns) + endpoint.GET("/available_fields", s.getAvailableFields) } } } @@ -165,19 +165,20 @@ func (s *Service) downloadHandler(c *gin.Context) { utils.DownloadByToken(token, "slowquery/download", c) } -// @Summary Query table columns -// @Description Query slowquery table columns +// @Summary Get available field names +// @Description Get available field names by slowquery table columns // @Success 200 {array} string // @Failure 400 {object} rest.ErrorResponse // @Failure 401 {object} rest.ErrorResponse // @Security JwtAuth -// @Router /slow_query/table_columns [get] -func (s *Service) queryTableColumns(c *gin.Context) { +// @Router /slow_query/available_fields [get] +func (s *Service) getAvailableFields(c *gin.Context) { db := utils.GetTiDBConnection(c) - cs, err := QueryTableColumns(s.params.SysSchema, db) + jsonNames, err := GetAvailableFields(s.params.SysSchema, db) if err != nil { rest.Error(c, err) return } - c.JSON(http.StatusOK, cs) + + c.JSON(http.StatusOK, jsonNames) } diff --git a/pkg/apiserver/statement/models.go b/pkg/apiserver/statement/models.go index 5b71416695..48778fd7a6 100644 --- a/pkg/apiserver/statement/models.go +++ b/pkg/apiserver/statement/models.go @@ -5,7 +5,6 @@ package statement import ( "strings" - "github.com/thoas/go-funk" "gorm.io/gorm" "gorm.io/gorm/schema" @@ -156,12 +155,21 @@ func getFieldsAndTags() (stmtFields []Field) { return } -func getVirtualFields(tableFields []string) []string { - fields := getFieldsAndTags() - vFields := funk.Filter(fields, func(f Field) bool { - return len(f.Related) != 0 && utils.IsSubsets(tableFields, f.Related) - }).([]Field) - return funk.Map(vFields, func(f Field) string { - return f.JSONName - }).([]string) +func filterFieldsByColumns(fields []Field, columns []string) []Field { + colMap := map[string]struct{}{} + for _, c := range columns { + colMap[strings.ToLower(c)] = struct{}{} + } + + filteredFields := []Field{} + for _, f := range fields { + // The json name of Statement is currently exactly the same as the table column name + // TODO: use util.VirtualView instead of the convention in the comment + _, ok := colMap[strings.ToLower(f.JSONName)] + if ok || (len(f.Related) != 0 && utils.IsSubsets(columns, f.Related)) { + filteredFields = append(filteredFields, f) + } + } + + return filteredFields } diff --git a/pkg/apiserver/statement/service.go b/pkg/apiserver/statement/service.go index 81a267b9f8..b10ee0c047 100644 --- a/pkg/apiserver/statement/service.go +++ b/pkg/apiserver/statement/service.go @@ -10,7 +10,6 @@ import ( "github.com/gin-gonic/gin" "github.com/joomcode/errorx" - "github.com/thoas/go-funk" "go.uber.org/fx" "github.com/pingcap/tidb-dashboard/pkg/apiserver/user" @@ -57,7 +56,7 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint.POST("/download/token", s.downloadTokenHandler) - endpoint.GET("/table_columns", s.queryTableColumns) + endpoint.GET("/available_fields", s.getAvailableFields) } } } @@ -309,18 +308,25 @@ func (s *Service) downloadHandler(c *gin.Context) { utils.DownloadByToken(token, "statements/download", c) } -// @Summary Query table columns -// @Description Query statements table columns +// @Summary Get available field names +// @Description Get available field names by statements table columns // @Success 200 {array} string // @Failure 401 {object} rest.ErrorResponse // @Security JwtAuth -// @Router /statements/table_columns [get] -func (s *Service) queryTableColumns(c *gin.Context) { +// @Router /statements/available_fields [get] +func (s *Service) getAvailableFields(c *gin.Context) { db := utils.GetTiDBConnection(c) cs, err := s.params.SysSchema.GetTableColumnNames(db, statementsTable) if err != nil { rest.Error(c, err) return } - c.JSON(http.StatusOK, funk.UniqString(append(cs, getVirtualFields(cs)...))) + + fields := filterFieldsByColumns(getFieldsAndTags(), cs) + jsonNames := make([]string, 0, len(fields)) + for _, f := range fields { + jsonNames = append(jsonNames, f.JSONName) + } + + c.JSON(http.StatusOK, jsonNames) } diff --git a/tests/integration/slowquery/compatibility_test.go b/tests/integration/slowquery/compatibility_test.go index 9dad95dc82..2b48512e2c 100644 --- a/tests/integration/slowquery/compatibility_test.go +++ b/tests/integration/slowquery/compatibility_test.go @@ -107,22 +107,89 @@ func (s *testCompatibilitySuite) TestFieldsCompatibility() { func (s *testCompatibilitySuite) TestQueryTableColumns() { if util.CheckTiDBVersion(s.Require(), "< 5.0.0") { - cls, err := slowquery.QueryTableColumns(s.sysSchema, s.dbSession()) + cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession()) s.Require().NoError(err) - s.Require().NotContains(cls, "Rocksdb_delete_skipped_count") - s.Require().NotContains(cls, "Rocksdb_key_skipped_count") - s.Require().NotContains(cls, "Rocksdb_block_cache_hit_count") - s.Require().NotContains(cls, "Rocksdb_block_read_count") - s.Require().NotContains(cls, "Rocksdb_block_read_byte") + s.Require().NotContains(cls, "rocksdb_delete_skipped_count") + s.Require().NotContains(cls, "rocksdb_key_skipped_count") + s.Require().NotContains(cls, "rocksdb_block_cache_hit_count") + s.Require().NotContains(cls, "rocksdb_block_read_count") + s.Require().NotContains(cls, "rocksdb_block_read_byte") } if util.CheckTiDBVersion(s.Require(), ">= 5.0.0") { - cls, err := slowquery.QueryTableColumns(s.sysSchema, s.dbSession()) + cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession()) s.Require().NoError(err) - s.Require().Contains(cls, "Rocksdb_delete_skipped_count") - s.Require().Contains(cls, "Rocksdb_key_skipped_count") - s.Require().Contains(cls, "Rocksdb_block_cache_hit_count") - s.Require().Contains(cls, "Rocksdb_block_read_count") - s.Require().Contains(cls, "Rocksdb_block_read_byte") + s.Require().Contains(cls, "rocksdb_delete_skipped_count") + s.Require().Contains(cls, "rocksdb_key_skipped_count") + s.Require().Contains(cls, "rocksdb_block_cache_hit_count") + s.Require().Contains(cls, "rocksdb_block_read_count") + s.Require().Contains(cls, "rocksdb_block_read_byte") + } +} + +func (s *testCompatibilitySuite) TestAllAvailableFields() { + // TODO: use latest instead of specific versions + if util.CheckTiDBVersion(s.Require(), ">= 5.0.0") { + cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession()) + s.Require().NoError(err) + s.Require().Contains(cls, "digest") + s.Require().Contains(cls, "query") + s.Require().Contains(cls, "instance") + s.Require().Contains(cls, "db") + s.Require().Contains(cls, "connection_id") + s.Require().Contains(cls, "success") + s.Require().Contains(cls, "timestamp") + s.Require().Contains(cls, "query_time") + s.Require().Contains(cls, "parse_time") + s.Require().Contains(cls, "compile_time") + s.Require().Contains(cls, "rewrite_time") + s.Require().Contains(cls, "preproc_subqueries_time") + s.Require().Contains(cls, "optimize_time") + s.Require().Contains(cls, "wait_ts") + s.Require().Contains(cls, "cop_time") + s.Require().Contains(cls, "lock_keys_time") + s.Require().Contains(cls, "write_sql_response_total") + s.Require().Contains(cls, "exec_retry_time") + s.Require().Contains(cls, "memory_max") + s.Require().Contains(cls, "disk_max") + s.Require().Contains(cls, "txn_start_ts") + s.Require().Contains(cls, "prev_stmt") + s.Require().Contains(cls, "plan") + s.Require().Contains(cls, "is_internal") + s.Require().Contains(cls, "index_names") + s.Require().Contains(cls, "stats") + s.Require().Contains(cls, "backoff_types") + s.Require().Contains(cls, "user") + s.Require().Contains(cls, "host") + s.Require().Contains(cls, "process_time") + s.Require().Contains(cls, "wait_time") + s.Require().Contains(cls, "backoff_time") + s.Require().Contains(cls, "get_commit_ts_time") + s.Require().Contains(cls, "local_latch_wait_time") + s.Require().Contains(cls, "resolve_lock_time") + s.Require().Contains(cls, "prewrite_time") + s.Require().Contains(cls, "wait_prewrite_binlog_time") + s.Require().Contains(cls, "commit_time") + s.Require().Contains(cls, "commit_backoff_time") + s.Require().Contains(cls, "cop_proc_avg") + s.Require().Contains(cls, "cop_proc_p90") + s.Require().Contains(cls, "cop_proc_max") + s.Require().Contains(cls, "cop_wait_avg") + s.Require().Contains(cls, "cop_wait_p90") + s.Require().Contains(cls, "cop_wait_max") + s.Require().Contains(cls, "write_keys") + s.Require().Contains(cls, "write_size") + s.Require().Contains(cls, "prewrite_region") + s.Require().Contains(cls, "txn_retry") + s.Require().Contains(cls, "request_count") + s.Require().Contains(cls, "process_keys") + s.Require().Contains(cls, "total_keys") + s.Require().Contains(cls, "cop_proc_addr") + s.Require().Contains(cls, "cop_wait_addr") + s.Require().Contains(cls, "rocksdb_delete_skipped_count") + s.Require().Contains(cls, "rocksdb_key_skipped_count") + s.Require().Contains(cls, "rocksdb_block_cache_hit_count") + s.Require().Contains(cls, "rocksdb_block_read_count") + s.Require().Contains(cls, "rocksdb_block_read_byte") } } diff --git a/ui/cypress/integration/slow_query/list.compat_spec.js b/ui/cypress/integration/slow_query/list.compat_spec.js new file mode 100644 index 0000000000..8022613a30 --- /dev/null +++ b/ui/cypress/integration/slow_query/list.compat_spec.js @@ -0,0 +1,84 @@ +// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0. +import { skipOn } from '@cypress/skip-test' + +describe('SlowQuery list compatibility test', () => { + before(() => { + cy.fixture('uri.json').then(function (uri) { + this.uri = uri + }) + }) + + beforeEach(function () { + cy.login('root') + cy.visit(this.uri.slow_query) + cy.url().should('include', this.uri.slow_query) + }) + + describe('Available fields', () => { + skipOn(Cypress.env('FEATURE_VERSION') !== '6.0.0', () => { + it('Show all available fields', () => { + cy.intercept('/dashboard/api/slow_query/available_fields').as( + 'getAvailableFields' + ) + cy.wait('@getAvailableFields') + + const availableFields = [ + 'query', + 'digest', + 'instance', + 'db', + 'connection_id', + 'timestamp', + + 'query_time', + 'parse_time', + 'compile_time', + 'process_time', + 'memory_max', + 'disk_max', + + 'txn_start_ts', + 'success', + 'is_internal', + 'index_names', + 'stats', + 'backoff_types', + + 'user', + 'host', + + 'wait_time', + 'backoff_time', + 'get_commit_ts_time', + 'local_latch_wait_time', + 'prewrite_time', + 'commit_time', + 'commit_backoff_time', + 'resolve_lock_time', + + 'cop_proc_avg', + 'cop_wait_avg', + 'write_keys', + 'write_size', + 'prewrite_region', + 'txn_retry', + 'request_count', + 'process_keys', + 'total_keys', + 'cop_proc_addr', + 'cop_wait_addr', + 'rocksdb_delete_skipped_count', + 'rocksdb_key_skipped_count', + 'rocksdb_block_cache_hit_count', + 'rocksdb_block_read_count', + 'rocksdb_block_read_byte', + ] + + cy.get('[data-e2e="columns_selector_popover"]').trigger('mouseover') + availableFields.forEach((f) => { + cy.get(`[data-e2e="columns_selector_field_${f}"]`).should('exist') + }) + }) + }) + }) +}) diff --git a/ui/lib/apps/SlowQuery/utils/useSchemaColumns.ts b/ui/lib/apps/SlowQuery/utils/useSchemaColumns.ts index ab61e0296f..abbf44343d 100644 --- a/ui/lib/apps/SlowQuery/utils/useSchemaColumns.ts +++ b/ui/lib/apps/SlowQuery/utils/useSchemaColumns.ts @@ -6,7 +6,7 @@ import { useClientRequest } from '@lib/utils/useClientRequest' export const useSchemaColumns = () => { const [schemaColumns, setSchemaColumns] = useState([]) const { data, isLoading } = useClientRequest((options) => { - return client.getInstance().slowQueryTableColumnsGet(options) + return client.getInstance().slowQueryAvailableFieldsGet(options) }) useEffect(() => { diff --git a/ui/lib/apps/Statement/utils/useSchemaColumns.ts b/ui/lib/apps/Statement/utils/useSchemaColumns.ts index 67a0317f4b..142f2e38d4 100644 --- a/ui/lib/apps/Statement/utils/useSchemaColumns.ts +++ b/ui/lib/apps/Statement/utils/useSchemaColumns.ts @@ -6,7 +6,7 @@ import { useClientRequest } from '@lib/utils/useClientRequest' export const useSchemaColumns = () => { const [schemaColumns, setSchemaColumns] = useState([]) const { data, isLoading } = useClientRequest((options) => { - return client.getInstance().statementsTableColumnsGet(options) + return client.getInstance().statementsAvailableFieldsGet(options) }) useEffect(() => { diff --git a/ui/lib/components/ColumnsSelector/index.tsx b/ui/lib/components/ColumnsSelector/index.tsx index fa243f85e3..83b4a912c6 100644 --- a/ui/lib/components/ColumnsSelector/index.tsx +++ b/ui/lib/components/ColumnsSelector/index.tsx @@ -131,6 +131,7 @@ export default function ColumnsSelector({ > {filteredColumns.map((column) => ( handleCheckChange(e, column)} @@ -145,7 +146,7 @@ export default function ColumnsSelector({ return ( - + {t('component.columnsSelector.trigger_text')} diff --git a/ui/package.json b/ui/package.json index 6602b9bfb8..7fc37776a2 100755 --- a/ui/package.json +++ b/ui/package.json @@ -11,6 +11,7 @@ "dependencies": { "@ant-design/icons": "^4.2.1", "@baurine/grafana-value-formats": "^1.0.0", + "@cypress/skip-test": "^2.6.1", "@duorou_xu/speedscope": "1.14.1", "@elastic/charts": "^20.0.0", "@fortawesome/fontawesome-free": "^5.14.0", diff --git a/ui/yarn.lock b/ui/yarn.lock index cd299ac7ca..c3f29af4c1 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -2988,6 +2988,11 @@ tunnel-agent "^0.6.0" uuid "^8.3.2" +"@cypress/skip-test@^2.6.1": + version "2.6.1" + resolved "https://registry.yarnpkg.com/@cypress/skip-test/-/skip-test-2.6.1.tgz#44a4bc4c2b2e369a7661177c9b38e50d417a36ea" + integrity sha512-X+ibefBiuOmC5gKG91wRIT0/OqXeETYvu7zXktjZ3yLeO186Y8ia0K7/gQUpAwuUi28DuqMd1+7tBQVtPkzbPA== + "@cypress/xvfb@^1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@cypress/xvfb/-/xvfb-1.2.4.tgz#2daf42e8275b39f4aa53c14214e557bd14e7748a"