From d9cf63ea88d5dd3e1d20ced3c586af47b52835da Mon Sep 17 00:00:00 2001 From: Tzu-Chiao Yeh Date: Sun, 30 Aug 2020 09:25:02 +0800 Subject: [PATCH] Support optional resultset metadata Allow optional resultset metadata. Can potentially improve the performance in many scenario. Issue #1105 --- AUTHORS | 1 + README.md | 11 +++++++++++ connection.go | 48 ++++++++++++++++++++++++++++++++++-------------- connector.go | 18 ++++++++++++++++++ const.go | 14 ++++++++++++++ driver_test.go | 33 +++++++++++++++++++++++++++++++++ dsn.go | 49 +++++++++++++++++++++++++++++++++---------------- dsn_test.go | 3 +++ errors.go | 3 ++- packets.go | 14 ++++++++++++++ 10 files changed, 163 insertions(+), 31 deletions(-) diff --git a/AUTHORS b/AUTHORS index 112e07524..7d51d0794 100644 --- a/AUTHORS +++ b/AUTHORS @@ -90,6 +90,7 @@ Tan Jinhua <312841925 at qq.com> Thomas Wodarek Tim Ruffles Tom Jenkinson +Tzu-Chiao Yeh Vladimir Kovpak Xiangyu Hu Xiaobing Jiang diff --git a/README.md b/README.md index 969aec554..6a69b425a 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,17 @@ Allow multiple statements in one query. While this allows batch queries, it also When `multiStatements` is used, `?` parameters must only be used in the first statement. +##### `resultSetMetadata` + +``` +Type: string +Valid Values: "full", "none" +Default: empty +``` + +Allow resultset metadata being optional. +By making resultset metadata transfer being optional, can potentially improve queries performance. + ##### `parseTime` ``` diff --git a/connection.go b/connection.go index 90aec6439..e6ea4b196 100644 --- a/connection.go +++ b/connection.go @@ -21,20 +21,21 @@ import ( ) type mysqlConn struct { - buf buffer - netConn net.Conn - rawConn net.Conn // underlying connection when netConn is TLS connection. - affectedRows uint64 - insertId uint64 - cfg *Config - maxAllowedPacket int - maxWriteSize int - writeTimeout time.Duration - flags clientFlag - status statusFlag - sequence uint8 - parseTime bool - reset bool // set when the Go SQL package calls ResetSession + buf buffer + netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. + affectedRows uint64 + insertId uint64 + cfg *Config + maxAllowedPacket int + maxWriteSize int + writeTimeout time.Duration + flags clientFlag + status statusFlag + sequence uint8 + parseTime bool + reset bool // set when the Go SQL package calls ResetSession + resultSetMetadata uint8 // for context support (Go 1.8+) watching bool @@ -392,6 +393,10 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) } } + if mc.resultSetMetadata == resultSetMetadataNone { + return mc.readIgnoreColumns(rows, resLen) + } + // Columns rows.rs.columns, err = mc.readColumns(resLen) return rows, err @@ -400,6 +405,21 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) return nil, mc.markBadConn(err) } +func (mc *mysqlConn) readIgnoreColumns(rows *textRows, resLen int) (*textRows, error) { + data, err := mc.readPacket() + if err != nil { + errLog.Print(err) + return nil, err + } + // Expected an EOF packet + if data[0] == iEOF && (len(data) == 5 || len(data) == 1) { + // Set empty columnNames, we will first read these columnNames via rows.Columns(). + rows.rs.columnNames = make([]string, resLen) + return rows, nil + } + return nil, ErrOptionalResultSet +} + // Gets the value of the given MySQL System Variable // The returned byte slice is only valid until the next read func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { diff --git a/connector.go b/connector.go index d567b4e4f..666f088de 100644 --- a/connector.go +++ b/connector.go @@ -129,6 +129,24 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { mc.maxWriteSize = mc.maxAllowedPacket } + // Additional handling for result set optional metadata + if mc.cfg.ResultSetMetadata != "" { + err = mc.exec("SET resultset_metadata=" + mc.cfg.ResultSetMetadata) + if err != nil { + mc.Close() + return nil, err + } + switch mc.cfg.ResultSetMetadata { + case resultSetMetadataSysVarNone: + mc.resultSetMetadata = resultSetMetadataNone + case resultSetMetadataSysVarFull: + mc.resultSetMetadata = resultSetMetadataFull + default: + mc.Close() + return nil, ErrOptionalResultSet + } + } + // Handle DSN Params err = mc.handleParams() if err != nil { diff --git a/const.go b/const.go index b1e6b85ef..b4de1eaa1 100644 --- a/const.go +++ b/const.go @@ -56,6 +56,7 @@ const ( clientCanHandleExpiredPasswords clientSessionTrack clientDeprecateEOF + clientOptionalResultSetMetadata ) const ( @@ -172,3 +173,16 @@ const ( cachingSha2PasswordFastAuthSuccess = 3 cachingSha2PasswordPerformFullAuthentication = 4 ) + +const ( + // One-byte metadata flag + // https://dev.mysql.com/worklog/task/?id=8134 + resultSetMetadataNone uint8 = iota + resultSetMetadataFull +) + +const ( + // ResultSet Metadata system var + resultSetMetadataSysVarNone = "NONE" + resultSetMetadataSysVarFull = "FULL" +) diff --git a/driver_test.go b/driver_test.go index aa55d2f55..0d42dcf17 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1344,6 +1344,39 @@ func TestFoundRows(t *testing.T) { }) } +func TestResultSetOptionalMetadata(t *testing.T) { + runTests(t, dsn+"&resultSetMetadata=none", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + row := dbt.db.QueryRow("SELECT id, data FROM test WHERE id = 1") + id, data := 0, 0 + err := row.Scan(&id, &data) + if err != nil { + dbt.Fatal(err) + } + + if id != 1 && data != 0 { + dbt.Fatal("invalid result") + } + }) + runTests(t, dsn+"&resultSetMetadata=full", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + row := dbt.db.QueryRow("SELECT id, data FROM test WHERE id = 1") + id, data := 0, 0 + err := row.Scan(&id, &data) + if err != nil { + dbt.Fatal(err) + } + + if id != 1 && data != 0 { + dbt.Fatal("invalid result") + } + }) +} + func TestTLS(t *testing.T) { tlsTestReq := func(dbt *DBTest) { if err := dbt.db.Ping(); err != nil { diff --git a/dsn.go b/dsn.go index 93f3548cb..699324e68 100644 --- a/dsn.go +++ b/dsn.go @@ -34,22 +34,23 @@ var ( // If a new Config is created instead of being parsed from a DSN string, // the NewConfig function should be used, which sets default values. type Config struct { - User string // Username - Passwd string // Password (requires User) - Net string // Network type - Addr string // Network address (requires Net) - DBName string // Database name - Params map[string]string // Connection parameters - Collation string // Connection collation - Loc *time.Location // Location for time.Time values - MaxAllowedPacket int // Max packet size allowed - ServerPubKey string // Server public key name - pubKey *rsa.PublicKey // Server public key - TLSConfig string // TLS configuration name - tls *tls.Config // TLS configuration - Timeout time.Duration // Dial timeout - ReadTimeout time.Duration // I/O read timeout - WriteTimeout time.Duration // I/O write timeout + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + pubKey *rsa.PublicKey // Server public key + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + ResultSetMetadata string // Allow optional resultset metadata AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE AllowCleartextPasswords bool // Allows the cleartext client side plugin @@ -240,6 +241,10 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "multiStatements", "true") } + if cfg.ResultSetMetadata != "" { + writeDSNParam(&buf, &hasParam, "resultSetMetadata", strings.ToLower(cfg.ResultSetMetadata)) + } + if cfg.ParseTime { writeDSNParam(&buf, &hasParam, "parseTime", "true") } @@ -465,6 +470,18 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // allow resultset metadata being optional + case "resultSetMetadata": + // Pre-check resultSetMetadata. + // Although so far there's only two modes FULL and NONE, in the future it may be extended. + // Because if any potential extensions introduced will force us do the read path change, + // failed earlier when parsing DSN. + upperVal := strings.ToUpper(value) + if upperVal != resultSetMetadataSysVarFull && upperVal != resultSetMetadataSysVarNone { + return errors.New("invalid resultset metadata, allow FULL and NONE only") + } + cfg.ResultSetMetadata = upperVal + // time.Time parsing case "parseTime": var isBool bool diff --git a/dsn_test.go b/dsn_test.go index 89815b341..f7a69bcb2 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -44,6 +44,9 @@ var testDSNs = []struct { }, { "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, +}, { + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0&resultSetMetadata=none", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false, ResultSetMetadata: "NONE"}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, diff --git a/errors.go b/errors.go index 760782ff2..a2b027c1a 100644 --- a/errors.go +++ b/errors.go @@ -21,7 +21,7 @@ var ( ErrMalformPkt = errors.New("malformed packet") ErrNoTLS = errors.New("TLS requested but server does not support TLS") ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") - ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrNativePassword = errors.New("this user requires mysql native password authentication") ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") ErrUnknownPlugin = errors.New("this authentication plugin is not supported") ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") @@ -29,6 +29,7 @@ var ( ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") ErrBusyBuffer = errors.New("busy buffer") + ErrOptionalResultSet = errors.New("malformed optional resultset metadata packets") // errBadConnNoWrite is used for connection errors where nothing was sent to the database yet. // If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn diff --git a/packets.go b/packets.go index 6664e5ae5..22013fd96 100644 --- a/packets.go +++ b/packets.go @@ -301,6 +301,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string clientFlags |= clientMultiStatements } + if mc.cfg.ResultSetMetadata != "" { + clientFlags |= clientOptionalResultSetMetadata + } + // encode length of the auth plugin data var authRespLEIBuf [9]byte authRespLen := len(authResp) @@ -554,6 +558,16 @@ func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { if n-len(data) == 0 { return int(num), nil } + // Sniff one extra byte for resultset metadata if we set capability + // CLIENT_OPTIONAL_RESULTSET_METADTA + // https://dev.mysql.com/worklog/task/?id=8134 + if len(data) == 2 && (mc.flags|clientOptionalResultSetMetadata) != 0 { + // ResultSet metadata flag check + if mc.resultSetMetadata != data[1] { + return 0, ErrOptionalResultSet + } + return int(num), nil + } return 0, ErrMalformPkt }