fix: missing fields in slow query (#1150)

* fix: missing fields in slowquery

* fix: lower case field names in tests

* test: slow query available fields

* test: move available fields tests to compatibility test

* chore: remove test log

* tweak: json names len

pkg/apiserver/slowquery/model.go

@@ -3,7 +3,7 @@
 package slowquery
 
 import (
-	"github.com/thoas/go-funk"
+	"strings"
 
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 	"github.com/pingcap/tidb-dashboard/util/reflectutil"
@@ -112,12 +112,19 @@ func getFieldsAndTags() (slowQueryFields []Field) {
 	return
 }
 
-func getVirtualFields() []string {
-	fields := getFieldsAndTags()
-	vFields := funk.Filter(fields, func(f Field) bool {
-		return f.Projection != ""
-	}).([]Field)
-	return funk.Map(vFields, func(f Field) string {
-		return f.ColumnName
-	}).([]string)
+func filterFieldsByColumns(fields []Field, columns []string) []Field {
+	colMap := map[string]struct{}{}
+	for _, c := range columns {
+		colMap[strings.ToLower(c)] = struct{}{}
+	}
+
+	filteredFields := []Field{}
+	for _, f := range fields {
+		_, ok := colMap[strings.ToLower(f.ColumnName)]
+		if ok || (f.Projection != "") {
+			filteredFields = append(filteredFields, f)
+		}
+	}
+
+	return filteredFields
 }

pkg/apiserver/slowquery/queries.go

@@ -5,7 +5,6 @@ package slowquery
 import (
 	"strings"
 
-	"github.com/thoas/go-funk"
 	"gorm.io/gorm"
 
 	"github.com/pingcap/tidb-dashboard/pkg/utils"
@@ -121,10 +120,17 @@ func QuerySlowLogDetail(req *GetDetailRequest, db *gorm.DB) (*Model, error) {
 	return &result, nil
 }
 
-func QueryTableColumns(sysSchema *utils.SysSchema, db *gorm.DB) ([]string, error) {
+func GetAvailableFields(sysSchema *utils.SysSchema, db *gorm.DB) ([]string, error) {
 	cs, err := sysSchema.GetTableColumnNames(db, SlowQueryTable)
 	if err != nil {
 		return nil, err
 	}
-	return funk.UniqString(append(cs, getVirtualFields()...)), nil
+
+	fields := filterFieldsByColumns(getFieldsAndTags(), cs)
+	jsonNames := make([]string, 0, len(fields))
+	for _, f := range fields {
+		jsonNames = append(jsonNames, f.JSONName)
+	}
+
+	return jsonNames, nil
 }

pkg/apiserver/slowquery/service.go

@@ -51,7 +51,7 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 
 			endpoint.POST("/download/token", s.downloadTokenHandler)
 
-			endpoint.GET("/table_columns", s.queryTableColumns)
+			endpoint.GET("/available_fields", s.getAvailableFields)
 		}
 	}
 }
@@ -165,19 +165,20 @@ func (s *Service) downloadHandler(c *gin.Context) {
 	utils.DownloadByToken(token, "slowquery/download", c)
 }
 
-// @Summary Query table columns
-// @Description Query slowquery table columns
+// @Summary Get available field names
+// @Description Get available field names by slowquery table columns
 // @Success 200 {array} string
 // @Failure 400 {object} rest.ErrorResponse
 // @Failure 401 {object} rest.ErrorResponse
 // @Security JwtAuth
-// @Router /slow_query/table_columns [get]
-func (s *Service) queryTableColumns(c *gin.Context) {
+// @Router /slow_query/available_fields [get]
+func (s *Service) getAvailableFields(c *gin.Context) {
 	db := utils.GetTiDBConnection(c)
-	cs, err := QueryTableColumns(s.params.SysSchema, db)
+	jsonNames, err := GetAvailableFields(s.params.SysSchema, db)
 	if err != nil {
 		rest.Error(c, err)
 		return
 	}
-	c.JSON(http.StatusOK, cs)
+
+	c.JSON(http.StatusOK, jsonNames)
 }

pkg/apiserver/statement/models.go

@@ -5,7 +5,6 @@ package statement
 import (
 	"strings"
 
-	"github.com/thoas/go-funk"
 	"gorm.io/gorm"
 	"gorm.io/gorm/schema"
 
@@ -156,12 +155,21 @@ func getFieldsAndTags() (stmtFields []Field) {
 	return
 }
 
-func getVirtualFields(tableFields []string) []string {
-	fields := getFieldsAndTags()
-	vFields := funk.Filter(fields, func(f Field) bool {
-		return len(f.Related) != 0 && utils.IsSubsets(tableFields, f.Related)
-	}).([]Field)
-	return funk.Map(vFields, func(f Field) string {
-		return f.JSONName
-	}).([]string)
+func filterFieldsByColumns(fields []Field, columns []string) []Field {
+	colMap := map[string]struct{}{}
+	for _, c := range columns {
+		colMap[strings.ToLower(c)] = struct{}{}
+	}
+
+	filteredFields := []Field{}
+	for _, f := range fields {
+		// The json name of Statement is currently exactly the same as the table column name
+		// TODO: use util.VirtualView instead of the convention in the comment
+		_, ok := colMap[strings.ToLower(f.JSONName)]
+		if ok || (len(f.Related) != 0 && utils.IsSubsets(columns, f.Related)) {
+			filteredFields = append(filteredFields, f)
+		}
+	}
+
+	return filteredFields
 }

pkg/apiserver/statement/service.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/joomcode/errorx"
-	"github.com/thoas/go-funk"
 	"go.uber.org/fx"
 
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/user"
@@ -57,7 +56,7 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 
 			endpoint.POST("/download/token", s.downloadTokenHandler)
 
-			endpoint.GET("/table_columns", s.queryTableColumns)
+			endpoint.GET("/available_fields", s.getAvailableFields)
 		}
 	}
 }
@@ -309,18 +308,25 @@ func (s *Service) downloadHandler(c *gin.Context) {
 	utils.DownloadByToken(token, "statements/download", c)
 }
 
-// @Summary Query table columns
-// @Description Query statements table columns
+// @Summary Get available field names
+// @Description Get available field names by statements table columns
 // @Success 200 {array} string
 // @Failure 401 {object} rest.ErrorResponse
 // @Security JwtAuth
-// @Router /statements/table_columns [get]
-func (s *Service) queryTableColumns(c *gin.Context) {
+// @Router /statements/available_fields [get]
+func (s *Service) getAvailableFields(c *gin.Context) {
 	db := utils.GetTiDBConnection(c)
 	cs, err := s.params.SysSchema.GetTableColumnNames(db, statementsTable)
 	if err != nil {
 		rest.Error(c, err)
 		return
 	}
-	c.JSON(http.StatusOK, funk.UniqString(append(cs, getVirtualFields(cs)...)))
+
+	fields := filterFieldsByColumns(getFieldsAndTags(), cs)
+	jsonNames := make([]string, 0, len(fields))
+	for _, f := range fields {
+		jsonNames = append(jsonNames, f.JSONName)
+	}
+
+	c.JSON(http.StatusOK, jsonNames)
 }

tests/integration/slowquery/compatibility_test.go

@@ -107,22 +107,89 @@ func (s *testCompatibilitySuite) TestFieldsCompatibility() {
 
 func (s *testCompatibilitySuite) TestQueryTableColumns() {
 	if util.CheckTiDBVersion(s.Require(), "< 5.0.0") {
-		cls, err := slowquery.QueryTableColumns(s.sysSchema, s.dbSession())
+		cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession())
 		s.Require().NoError(err)
-		s.Require().NotContains(cls, "Rocksdb_delete_skipped_count")
-		s.Require().NotContains(cls, "Rocksdb_key_skipped_count")
-		s.Require().NotContains(cls, "Rocksdb_block_cache_hit_count")
-		s.Require().NotContains(cls, "Rocksdb_block_read_count")
-		s.Require().NotContains(cls, "Rocksdb_block_read_byte")
+		s.Require().NotContains(cls, "rocksdb_delete_skipped_count")
+		s.Require().NotContains(cls, "rocksdb_key_skipped_count")
+		s.Require().NotContains(cls, "rocksdb_block_cache_hit_count")
+		s.Require().NotContains(cls, "rocksdb_block_read_count")
+		s.Require().NotContains(cls, "rocksdb_block_read_byte")
 	}
 
 	if util.CheckTiDBVersion(s.Require(), ">= 5.0.0") {
-		cls, err := slowquery.QueryTableColumns(s.sysSchema, s.dbSession())
+		cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession())
 		s.Require().NoError(err)
-		s.Require().Contains(cls, "Rocksdb_delete_skipped_count")
-		s.Require().Contains(cls, "Rocksdb_key_skipped_count")
-		s.Require().Contains(cls, "Rocksdb_block_cache_hit_count")
-		s.Require().Contains(cls, "Rocksdb_block_read_count")
-		s.Require().Contains(cls, "Rocksdb_block_read_byte")
+		s.Require().Contains(cls, "rocksdb_delete_skipped_count")
+		s.Require().Contains(cls, "rocksdb_key_skipped_count")
+		s.Require().Contains(cls, "rocksdb_block_cache_hit_count")
+		s.Require().Contains(cls, "rocksdb_block_read_count")
+		s.Require().Contains(cls, "rocksdb_block_read_byte")
+	}
+}
+
+func (s *testCompatibilitySuite) TestAllAvailableFields() {
+	// TODO: use latest instead of specific versions
+	if util.CheckTiDBVersion(s.Require(), ">= 5.0.0") {
+		cls, err := slowquery.GetAvailableFields(s.sysSchema, s.dbSession())
+		s.Require().NoError(err)
+		s.Require().Contains(cls, "digest")
+		s.Require().Contains(cls, "query")
+		s.Require().Contains(cls, "instance")
+		s.Require().Contains(cls, "db")
+		s.Require().Contains(cls, "connection_id")
+		s.Require().Contains(cls, "success")
+		s.Require().Contains(cls, "timestamp")
+		s.Require().Contains(cls, "query_time")
+		s.Require().Contains(cls, "parse_time")
+		s.Require().Contains(cls, "compile_time")
+		s.Require().Contains(cls, "rewrite_time")
+		s.Require().Contains(cls, "preproc_subqueries_time")
+		s.Require().Contains(cls, "optimize_time")
+		s.Require().Contains(cls, "wait_ts")
+		s.Require().Contains(cls, "cop_time")
+		s.Require().Contains(cls, "lock_keys_time")
+		s.Require().Contains(cls, "write_sql_response_total")
+		s.Require().Contains(cls, "exec_retry_time")
+		s.Require().Contains(cls, "memory_max")
+		s.Require().Contains(cls, "disk_max")
+		s.Require().Contains(cls, "txn_start_ts")
+		s.Require().Contains(cls, "prev_stmt")
+		s.Require().Contains(cls, "plan")
+		s.Require().Contains(cls, "is_internal")
+		s.Require().Contains(cls, "index_names")
+		s.Require().Contains(cls, "stats")
+		s.Require().Contains(cls, "backoff_types")
+		s.Require().Contains(cls, "user")
+		s.Require().Contains(cls, "host")
+		s.Require().Contains(cls, "process_time")
+		s.Require().Contains(cls, "wait_time")
+		s.Require().Contains(cls, "backoff_time")
+		s.Require().Contains(cls, "get_commit_ts_time")
+		s.Require().Contains(cls, "local_latch_wait_time")
+		s.Require().Contains(cls, "resolve_lock_time")
+		s.Require().Contains(cls, "prewrite_time")
+		s.Require().Contains(cls, "wait_prewrite_binlog_time")
+		s.Require().Contains(cls, "commit_time")
+		s.Require().Contains(cls, "commit_backoff_time")
+		s.Require().Contains(cls, "cop_proc_avg")
+		s.Require().Contains(cls, "cop_proc_p90")
+		s.Require().Contains(cls, "cop_proc_max")
+		s.Require().Contains(cls, "cop_wait_avg")
+		s.Require().Contains(cls, "cop_wait_p90")
+		s.Require().Contains(cls, "cop_wait_max")
+		s.Require().Contains(cls, "write_keys")
+		s.Require().Contains(cls, "write_size")
+		s.Require().Contains(cls, "prewrite_region")
+		s.Require().Contains(cls, "txn_retry")
+		s.Require().Contains(cls, "request_count")
+		s.Require().Contains(cls, "process_keys")
+		s.Require().Contains(cls, "total_keys")
+		s.Require().Contains(cls, "cop_proc_addr")
+		s.Require().Contains(cls, "cop_wait_addr")
+		s.Require().Contains(cls, "rocksdb_delete_skipped_count")
+		s.Require().Contains(cls, "rocksdb_key_skipped_count")
+		s.Require().Contains(cls, "rocksdb_block_cache_hit_count")
+		s.Require().Contains(cls, "rocksdb_block_read_count")
+		s.Require().Contains(cls, "rocksdb_block_read_byte")
 	}
 }

ui/cypress/integration/slow_query/list.compat_spec.js

@@ -0,0 +1,84 @@
+// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
+import { skipOn } from '@cypress/skip-test'
+
+describe('SlowQuery list compatibility test', () => {
+  before(() => {
+    cy.fixture('uri.json').then(function (uri) {
+      this.uri = uri
+    })
+  })
+
+  beforeEach(function () {
+    cy.login('root')
+    cy.visit(this.uri.slow_query)
+    cy.url().should('include', this.uri.slow_query)
+  })
+
+  describe('Available fields', () => {
+    skipOn(Cypress.env('FEATURE_VERSION') !== '6.0.0', () => {
+      it('Show all available fields', () => {
+        cy.intercept('/dashboard/api/slow_query/available_fields').as(
+          'getAvailableFields'
+        )
+        cy.wait('@getAvailableFields')
+
+        const availableFields = [
+          'query',
+          'digest',
+          'instance',
+          'db',
+          'connection_id',
+          'timestamp',
+
+          'query_time',
+          'parse_time',
+          'compile_time',
+          'process_time',
+          'memory_max',
+          'disk_max',
+
+          'txn_start_ts',
+          'success',
+          'is_internal',
+          'index_names',
+          'stats',
+          'backoff_types',
+
+          'user',
+          'host',
+
+          'wait_time',
+          'backoff_time',
+          'get_commit_ts_time',
+          'local_latch_wait_time',
+          'prewrite_time',
+          'commit_time',
+          'commit_backoff_time',
+          'resolve_lock_time',
+
+          'cop_proc_avg',
+          'cop_wait_avg',
+          'write_keys',
+          'write_size',
+          'prewrite_region',
+          'txn_retry',
+          'request_count',
+          'process_keys',
+          'total_keys',
+          'cop_proc_addr',
+          'cop_wait_addr',
+          'rocksdb_delete_skipped_count',
+          'rocksdb_key_skipped_count',
+          'rocksdb_block_cache_hit_count',
+          'rocksdb_block_read_count',
+          'rocksdb_block_read_byte',
+        ]
+
+        cy.get('[data-e2e="columns_selector_popover"]').trigger('mouseover')
+        availableFields.forEach((f) => {
+          cy.get(`[data-e2e="columns_selector_field_${f}"]`).should('exist')
+        })
+      })
+    })
+  })
+})

ui/lib/apps/SlowQuery/utils/useSchemaColumns.ts

@@ -6,7 +6,7 @@ import { useClientRequest } from '@lib/utils/useClientRequest'
 export const useSchemaColumns = () => {
   const [schemaColumns, setSchemaColumns] = useState<string[]>([])
   const { data, isLoading } = useClientRequest((options) => {
-    return client.getInstance().slowQueryTableColumnsGet(options)
+    return client.getInstance().slowQueryAvailableFieldsGet(options)
   })
 
   useEffect(() => {

ui/lib/apps/Statement/utils/useSchemaColumns.ts

@@ -6,7 +6,7 @@ import { useClientRequest } from '@lib/utils/useClientRequest'
 export const useSchemaColumns = () => {
   const [schemaColumns, setSchemaColumns] = useState<string[]>([])
   const { data, isLoading } = useClientRequest((options) => {
-    return client.getInstance().statementsTableColumnsGet(options)
+    return client.getInstance().statementsAvailableFieldsGet(options)
   })
 
   useEffect(() => {