Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

database/gdb: enhance the underlying data transformation when querying data #3557

Closed
wants to merge 44 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
e9c66ff
只实现类型转换
wln32 Apr 30, 2024
8e51bdf
up
wln32 May 1, 2024
0d9d750
up
wln32 May 1, 2024
9d828ee
对比查询到的表字段和结构体字段,删除不存在的
wln32 May 1, 2024
235dffc
更新测试: Delete时不会报错
wln32 May 1, 2024
ef5d6e7
更新注释
wln32 May 1, 2024
15054e5
更新注释
wln32 May 1, 2024
d5a2f6e
更新注释
wln32 May 1, 2024
7b011c9
更新测试
wln32 May 1, 2024
27b1f0c
test version
wln32 May 1, 2024
e59057e
test date
wln32 May 1, 2024
5019608
更新注释、文件名、格式化代码
wln32 May 1, 2024
9de5f16
test
wln32 May 1, 2024
5503ebc
循环导入
wln32 May 1, 2024
a85bdc2
test
wln32 May 1, 2024
c897225
添加oracle类型number到int的映射
wln32 May 1, 2024
d518b6c
更新测试
wln32 May 1, 2024
1043843
gtime不使用Scan函数来做转换
wln32 May 2, 2024
36d01f9
修复time类型断言失败
wln32 May 2, 2024
05dad62
增加自定义的json.Unmarshaler接口支持
wln32 May 2, 2024
afd179e
修复:sqlite驱动time类型->go的string类型,转换失败
wln32 May 2, 2024
7cfcc27
优化注释
wln32 May 2, 2024
51fe344
up
wln32 May 4, 2024
e621221
use the new logic to do it
wln32 May 7, 2024
3f42451
delete useless files
wln32 May 7, 2024
87bd0b2
update the logic of the detection interface implementation
wln32 May 7, 2024
0b8e111
use arrays to get field information
wln32 May 8, 2024
364da1f
Optimize some function calls
wln32 May 8, 2024
3ec8e0b
update comments
wln32 May 8, 2024
197f4d1
update the logic of the detection interface implementation
wln32 May 8, 2024
a477612
add custom decimal type conversions
wln32 May 8, 2024
6340c18
up format
wln32 May 8, 2024
9f753ce
format
wln32 May 8, 2024
031f1b7
function splitting, variable name optimization
wln32 May 9, 2024
e31ee1f
add conversion of number type
wln32 May 11, 2024
58eb5fb
add some notes
wln32 May 11, 2024
a2a7f88
Merge branch 'master' into Improve/gdb-scan-typeconvert
wln32 May 11, 2024
f58c686
up
wln32 May 22, 2024
3d36959
1.增加两个可以自定义注册的字段转换函数,根据不同场景使用
wln32 Jul 5, 2024
c474c98
使用SetTestEnvironment函数来控制是否删除缓存的结构体信息
wln32 Jul 5, 2024
1f4e061
添加自定义转换的测试
wln32 Jul 5, 2024
c8ccb49
ci
wln32 Jul 5, 2024
342dd6f
修复测试
wln32 Jul 5, 2024
f52440e
1.注册go结构体字段的转换变更为注册go类型的字段转换
wln32 Jul 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions contrib/drivers/clickhouse/clickhouse_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ func init() {
Type: "clickhouse",
Debug: false,
}

gdb.SetTestEnvironment(true)
var err error
db, err = gdb.New(node)
gtest.AssertNil(err)
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/dm/dm_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ func init() {
UpdatedAt: "updated_time",
}

gdb.SetTestEnvironment(true)

// todo
nodeLink := gdb.ConfigNode{
Type: TestDBType,
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/mssql/mssql_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ func init() {
MaxOpenConnCount: 10,
}

gdb.SetTestEnvironment(true)

nodeLink := gdb.ConfigNode{
Type: "mssql",
Name: "test",
Expand Down
1 change: 1 addition & 0 deletions contrib/drivers/mysql/mysql_z_unit_core_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1621,6 +1621,7 @@ func Test_Types(t *testing.T) {
Bit int8
TinyInt bool
}

var obj *T
err = db.Model("types").Scan(&obj)
t.AssertNil(err)
Expand Down
169 changes: 169 additions & 0 deletions contrib/drivers/mysql/mysql_z_unit_feature_custom_convert_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package mysql_test

import (
"database/sql"
"fmt"
"reflect"
"strconv"
"testing"

"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/test/gtest"
)

type sqlScanMyDecimal struct {
v float64
}

func (m *sqlScanMyDecimal) Scan(src any) (err error) {
var v float64
switch sv := src.(type) {
case []byte:
v, err = strconv.ParseFloat(string(sv), 64)
case string:
v, err = strconv.ParseFloat(sv, 64)
case float64:
v = sv
case float32:
v = float64(sv)
default:
err = fmt.Errorf("unknown type: %v(%T)", src, src)
}
if err != nil {
return err
}
m.v = v
return nil
}

func testCustomConvert(t *gtest.T, f func(t *gtest.T)) {
sql := `CREATE TABLE IF NOT EXISTS %s (
id int(10) unsigned NOT NULL AUTO_INCREMENT primary key,
my_decimal1 decimal(5,2) NOT NULL,
my_decimal2 decimal(5,2) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`
tableName := "test_decimal"
_, err := db.Exec(ctx, fmt.Sprintf(sql, tableName))
t.AssertNil(err)
defer dropTable(tableName)
table := db.Model(tableName)
data := g.Map{
"my_decimal1": 777.333,
"my_decimal2": 888.444,
}
_, err = table.Data(data).Insert()
t.AssertNil(err)
f(t)
}

func Test_Custom_Convert_SqlScanner(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
tableName := "test_decimal"
table := db.Model(tableName)
type TestDecimal struct {
MyDecimal1 sqlScanMyDecimal `orm:"my_decimal1"`
MyDecimal2 *sqlScanMyDecimal `orm:"my_decimal2"`
}
testCustomConvert(t, func(t *gtest.T) {
var res *TestDecimal
err := table.Scan(&res)
t.AssertNil(err)
t.Assert(res.MyDecimal1.v, 777.33)
t.Assert(res.MyDecimal2.v, 888.44)
})
})
}

func testCustomFieldConvertFunc(dest reflect.Value, src any) (err error) {
var v float64
switch sv := src.(type) {
case []byte:
v, err = strconv.ParseFloat(string(sv), 64)
case string:
v, err = strconv.ParseFloat(sv, 64)
case float64:
v = sv
case float32:
v = float64(sv)
default:
return fmt.Errorf("unknown type: %v(%T)", src, src)
}
if err != nil {
return err
}

if dest.Kind() == reflect.Ptr {
if dest.IsNil() {
dest.Set(reflect.New(dest.Type().Elem()))
}
dest = dest.Elem()
}
switch dest.Kind() {
case reflect.Float32, reflect.Float64:
dest.SetFloat(v)
case reflect.Int, reflect.Int64 /* reflect.Int16,reflect.Int32 */ :
dest.SetInt(int64(v))
case reflect.Uint, reflect.Uint64 /* reflect.Uint16,reflect.Uint32 */ :
dest.SetUint(uint64(v))
// case other types...:
default:
if dest.Kind() != reflect.Ptr {
dest = dest.Addr()
switch impl := dest.Interface().(type) {
case sql.Scanner:
return impl.Scan(v)
default:
}
}
return fmt.Errorf("Error: mysql_z_unit_feature_custom_convert_test.go:125 unsupported types: %v", dest.Type())
}
return nil
}

func Test_Custom_Convert_RegisterDatabaseConvertFunc(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
gdb.RegisterDatabaseConvertFunc("mysql", "decimal", testCustomFieldConvertFunc)
tableName := "test_decimal"
table := db.Model(tableName)
type MyDecimal float64
type TestDecimal struct {
MyDecimal1 MyDecimal `orm:"my_decimal1"`
MyDecimal2 *MyDecimal `orm:"my_decimal2"`
}
testCustomConvert(t, func(t *gtest.T) {
var res *TestDecimal
err := table.Scan(&res)
t.AssertNil(err)
t.Assert(res.MyDecimal1, 777.33)
t.Assert(res.MyDecimal2, 888.44)
})
})
}

func Test_Custom_Convert_RegisterStructFieldConvertFunc(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
type MyDecimal float64
type TestDecimal struct {
MyDecimal1 MyDecimal `orm:"my_decimal1"`
MyDecimal2 *sqlScanMyDecimal `orm:"my_decimal2"`
}
gdb.RegisterGoTypeConvertFunc(reflect.TypeOf(MyDecimal(0)), testCustomFieldConvertFunc)
tableName := "test_decimal"
table := db.Model(tableName)
testCustomConvert(t, func(t *gtest.T) {
var res *TestDecimal
err := table.Scan(&res)
t.AssertNil(err)
t.Assert(res.MyDecimal1, 777.33)
t.Assert(res.MyDecimal2.v, 888.44)
})
})
}
2 changes: 2 additions & 0 deletions contrib/drivers/mysql/mysql_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ func init() {
gdb.AddConfigNode("partition", partitionDefault)
gdb.AddConfigNode(gdb.DefaultGroupName, nodeDefault)

gdb.SetTestEnvironment(true)

// Default db.
if r, err := gdb.NewByGroup(); err != nil {
gtest.Error(err)
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/oracle/oracle_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ func init() {
Weight: 1,
}

gdb.SetTestEnvironment(true)

gdb.AddConfigNode(gdb.DefaultGroupName, node)
if r, err := gdb.New(node); err != nil {
gtest.Fatal(err)
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/pgsql/pgsql_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ func init() {
Link: `pgsql:postgres:12345678@tcp(127.0.0.1:5432)`,
}

gdb.SetTestEnvironment(true)

//pgsql only permit to connect to the designation database.
//so you need to create the pgsql database before you use orm
gdb.AddConfigNode(gdb.DefaultGroupName, configNode)
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/sqlite/sqlite_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ func init() {

nodeInvalid := configNode

gdb.SetTestEnvironment(true)

gdb.AddConfigNode(DBGroupTest, configNode)
gdb.AddConfigNode(DBGroupPrefix, nodePrefix)
gdb.AddConfigNode(DBGroupInvalid, nodeInvalid)
Expand Down
2 changes: 2 additions & 0 deletions contrib/drivers/sqlitecgo/sqlitecgo_z_unit_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ func init() {

nodeInvalid := configNode

gdb.SetTestEnvironment(true)

gdb.AddConfigNode(DBGroupTest, configNode)
gdb.AddConfigNode(DBGroupPrefix, nodePrefix)
gdb.AddConfigNode(DBGroupInvalid, nodeInvalid)
Expand Down
1 change: 1 addition & 0 deletions database/gdb/gdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,7 @@ type LocalType string
const (
LocalTypeUndefined LocalType = ""
LocalTypeString LocalType = "string"
LocalTypeDecimal LocalType = "decimal"
LocalTypeDate LocalType = "date"
LocalTypeDatetime LocalType = "datetime"
LocalTypeInt LocalType = "int"
Expand Down
48 changes: 48 additions & 0 deletions database/gdb/gdb_core_convert_fieldinfo.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gdb

import (
"database/sql"
"reflect"
)

// dest represents the structural field to be assigned a value to
// src is the field value of the database
type fieldConvertFunc = func(dest reflect.Value, src any) error

type fieldConvertInfo struct {
// table field
ColumnFieldName string
ColumnFieldIndex int
ColumnFieldType *sql.ColumnType
// struct field
StructField reflect.StructField
StructFieldType reflect.Type
// The reason why an index is an []int is because this field may be an anonymous structure
StructFieldIndex []int
convertFunc fieldConvertFunc
}

// GetReflectValue = reflect.Value.FieldByIndex
func (c *fieldConvertInfo) GetReflectValue(structValue reflect.Value) reflect.Value {
if len(c.StructFieldIndex) == 1 {
return structValue.Field(c.StructFieldIndex[0])
}

fieldValue := structValue.Field(c.StructFieldIndex[0])
for i := 1; i < len(c.StructFieldIndex); i++ {
if fieldValue.Kind() == reflect.Pointer {
if fieldValue.IsNil() {
fieldValue.Set(reflect.New(fieldValue.Type().Elem()))
}
fieldValue = fieldValue.Elem()
}
fieldValue = fieldValue.Field(c.StructFieldIndex[i])
}
return fieldValue
}
Loading