Skip to content

Commit

Permalink
Merge upstream
Browse files Browse the repository at this point in the history
  • Loading branch information
@glebarez
glebarez committed Mar 14, 2024
2 parents 084ab1e + 7544227 commit a871805
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 78 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
test:
strategy:
matrix:
go: ['1.20','1.19','1.18']
go: ['1.21', '1.20', '1.19']
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}

Expand Down
43 changes: 18 additions & 25 deletions ddlmod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

var (
sqliteSeparator = "`|\"|'|\t"
uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^CONSTRAINT [%v]?[\w-]+[%v]? UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator))
indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator))
tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator))
separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator))
Expand Down Expand Up @@ -103,11 +104,24 @@ func parseDDL(strs ...string) (*ddl, error) {

for _, f := range result.fields {
fUpper := strings.ToUpper(f)
if strings.HasPrefix(fUpper, "CHECK") ||
strings.HasPrefix(fUpper, "CONSTRAINT") {
if strings.HasPrefix(fUpper, "CHECK") {
continue
}
if strings.HasPrefix(fUpper, "CONSTRAINT") {
matches := uniqueRegexp.FindStringSubmatch(f)
if len(matches) > 0 {
if columns := getAllColumns(matches[1]); len(columns) == 1 {
for idx, column := range result.columns {
if column.NameValue.String == columns[0] {
column.UniqueValue = sql.NullBool{Bool: true, Valid: true}
result.columns[idx] = column
break
}
}
}
}
continue
}

if strings.HasPrefix(fUpper, "PRIMARY KEY") {
for _, name := range getAllColumns(f) {
for idx, column := range result.columns {
Expand Down Expand Up @@ -159,14 +173,7 @@ func parseDDL(strs ...string) (*ddl, error) {
}
}
} else if matches := indexRegexp.FindStringSubmatch(str); len(matches) > 0 {
for _, column := range getAllColumns(matches[1]) {
for idx, c := range result.columns {
if c.NameValue.String == column {
c.UniqueValue = sql.NullBool{Bool: strings.ToUpper(strings.Fields(str)[1]) == "UNIQUE", Valid: true}
result.columns[idx] = c
}
}
}
// don't report Unique by UniqueIndex
} else {
return nil, errors.New("invalid DDL")
}
Expand Down Expand Up @@ -268,20 +275,6 @@ func (d *ddl) getColumns() []string {
return res
}

func (d *ddl) alterColumn(name, sql string) bool {
reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$")

for i := 0; i < len(d.fields); i++ {
if reg.MatchString(d.fields[i]) {
d.fields[i] = sql
return false
}
}

d.fields = append(d.fields, sql)
return true
}

func (d *ddl) removeColumn(name string) bool {
reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$")

Expand Down
63 changes: 54 additions & 9 deletions ddlmod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ func TestParseDDL(t *testing.T) {
columns []migrator.ColumnType
}{
{"with_fk", []string{
"CREATE TABLE `notes` (`id` integer NOT NULL,`text` varchar(500) DEFAULT \"hello\",`age` integer DEFAULT 18,`user_id` integer,PRIMARY KEY (`id`),CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))",
"CREATE TABLE `notes` (" +
"`id` integer NOT NULL,`text` varchar(500) DEFAULT \"hello\",`age` integer DEFAULT 18,`user_id` integer,PRIMARY KEY (`id`),CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))",
"CREATE UNIQUE INDEX `idx_profiles_refer` ON `profiles`(`text`)",
}, 6, []migrator.ColumnType{
{NameValue: sql.NullString{String: "id", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, PrimaryKeyValue: sql.NullBool{Bool: true, Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}},
{NameValue: sql.NullString{String: "text", Valid: true}, DataTypeValue: sql.NullString{String: "varchar", Valid: true}, LengthValue: sql.NullInt64{Int64: 500, Valid: true}, ColumnTypeValue: sql.NullString{String: "varchar(500)", Valid: true}, DefaultValueValue: sql.NullString{String: "hello", Valid: true}, NullableValue: sql.NullBool{Bool: true, Valid: true}, UniqueValue: sql.NullBool{Bool: true, Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}},
{NameValue: sql.NullString{String: "text", Valid: true}, DataTypeValue: sql.NullString{String: "varchar", Valid: true}, LengthValue: sql.NullInt64{Int64: 500, Valid: true}, ColumnTypeValue: sql.NullString{String: "varchar(500)", Valid: true}, DefaultValueValue: sql.NullString{String: "hello", Valid: true}, NullableValue: sql.NullBool{Bool: true, Valid: true}, UniqueValue: sql.NullBool{Bool: false, Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}},
{NameValue: sql.NullString{String: "age", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, DefaultValueValue: sql.NullString{String: "18", Valid: true}, NullableValue: sql.NullBool{Bool: true, Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}},
{NameValue: sql.NullString{String: "user_id", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, NullableValue: sql.NullBool{Bool: true, Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}},
},
Expand Down Expand Up @@ -56,34 +57,78 @@ func TestParseDDL(t *testing.T) {
ColumnTypeValue: sql.NullString{String: "int", Valid: true},
NullableValue: sql.NullBool{Bool: false, Valid: true},
DefaultValueValue: sql.NullString{Valid: false},
UniqueValue: sql.NullBool{Bool: true, Valid: true},
UniqueValue: sql.NullBool{Bool: false, Valid: true},
PrimaryKeyValue: sql.NullBool{Valid: true},
},
},
},
{
}, {
"unique index",
[]string{
"CREATE TABLE `test-b` (`field` integer NOT NULL)",
"CREATE UNIQUE INDEX `idx_uq` ON `test-b`(`field`) WHERE field = 0",
},
1,
[]migrator.ColumnType{{
NameValue: sql.NullString{String: "field", Valid: true},
DataTypeValue: sql.NullString{String: "integer", Valid: true},
ColumnTypeValue: sql.NullString{String: "integer", Valid: true},
PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true},
UniqueValue: sql.NullBool{Bool: false, Valid: true},
NullableValue: sql.NullBool{Bool: false, Valid: true},
}},
}, {
"normal index",
[]string{
"CREATE TABLE `test-c` (`field` integer NOT NULL)",
"CREATE INDEX `idx_uq` ON `test-c`(`field`)",
},
1,
[]migrator.ColumnType{{
NameValue: sql.NullString{String: "field", Valid: true},
DataTypeValue: sql.NullString{String: "integer", Valid: true},
ColumnTypeValue: sql.NullString{String: "integer", Valid: true},
PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true},
UniqueValue: sql.NullBool{Bool: false, Valid: true},
NullableValue: sql.NullBool{Bool: false, Valid: true},
}},
}, {
"unique constraint",
[]string{
"CREATE TABLE `unique_struct` (`name` text,CONSTRAINT `uni_unique_struct_name` UNIQUE (`name`))",
},
2,
[]migrator.ColumnType{{
NameValue: sql.NullString{String: "name", Valid: true},
DataTypeValue: sql.NullString{String: "text", Valid: true},
ColumnTypeValue: sql.NullString{String: "text", Valid: true},
PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true},
UniqueValue: sql.NullBool{Bool: true, Valid: true},
NullableValue: sql.NullBool{Bool: true, Valid: true},
}},
},
{
"non-unique index",
[]string{
"CREATE TABLE `test-c` (`field` integer NOT NULL)",
"CREATE INDEX `idx_uq` ON `test-b`(`field`) WHERE field = 0",
},
1,
[]migrator.ColumnType{
{
NameValue: sql.NullString{String: "field", Valid: true},
DataTypeValue: sql.NullString{String: "integer", Valid: true},
ColumnTypeValue: sql.NullString{String: "integer", Valid: true},
PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true},
UniqueValue: sql.NullBool{Bool: true, Valid: true},
UniqueValue: sql.NullBool{Bool: false, Valid: true},
NullableValue: sql.NullBool{Bool: false, Valid: true},
},
},
},
{
"non-unique index",
"index with \n from .schema sqlite",
[]string{
"CREATE TABLE `test-c` (`field` integer NOT NULL)",
"CREATE INDEX `idx_uq` ON `test-b`(`field`) WHERE field = 0",
"CREATE TABLE `test-d` (`field` integer NOT NULL)",
"CREATE INDEX `idx_uq`\n ON `test-b`(`field`) WHERE field = 0",
},
1,
[]migrator.ColumnType{
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.18

require (
github.com/glebarez/go-sqlite v1.21.2
gorm.io/gorm v1.25.5
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde
modernc.org/sqlite v1.23.1
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg=
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
Expand Down
104 changes: 64 additions & 40 deletions migrator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,28 @@ func (m Migrator) AlterColumn(value interface{}, name string) error {
return m.RunWithoutForeignKey(func() error {
return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) {
if field := stmt.Schema.LookUpField(name); field != nil {
if ddl.alterColumn(field.DBName, fmt.Sprintf("`%s` ?", field.DBName)) {
return nil, nil, fmt.Errorf("field `%s` not found in origin ddl, ddl= '%s'", name, ddl.compile())
var sqlArgs []interface{}
for i, f := range ddl.fields {
if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 1 && matches[1] == field.DBName {
ddl.fields[i] = fmt.Sprintf("`%v` ?", field.DBName)
sqlArgs = []interface{}{m.FullDataTypeOf(field)}
// table created by old version might look like `CREATE TABLE ? (? varchar(10) UNIQUE)`.
// FullDataTypeOf doesn't contain UNIQUE, so we need to add unique constraint.
if strings.Contains(strings.ToUpper(matches[3]), " UNIQUE") {
uniName := m.DB.NamingStrategy.UniqueName(stmt.Table, field.DBName)
uni, _ := m.GuessConstraintInterfaceAndTable(stmt, uniName)
if uni != nil {
uniSQL, uniArgs := uni.Build()
ddl.addConstraint(uniName, uniSQL)
sqlArgs = append(sqlArgs, uniArgs...)
}
}
break
}
}

return ddl, []interface{}{m.FullDataTypeOf(field)}, nil
return ddl, sqlArgs, nil
}

return nil, nil, fmt.Errorf("failed to alter field with name `%s`", name)
return nil, nil, fmt.Errorf("failed to alter field with name %v", name)
})
})
}
Expand Down Expand Up @@ -153,7 +167,7 @@ func (m Migrator) DropColumn(value interface{}, name string) error {

func (m Migrator) CreateConstraint(value interface{}, name string) error {
return m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)

return m.recreateTable(value, &table,
func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) {
Expand All @@ -164,12 +178,8 @@ func (m Migrator) CreateConstraint(value interface{}, name string) error {
)

if constraint != nil {
constraintName = constraint.Name
constraintSql, constraintValues = buildConstraint(constraint)
} else if chk != nil {
constraintName = chk.Name
constraintSql = "CONSTRAINT ? CHECK (?)"
constraintValues = []interface{}{clause.Column{Name: chk.Name}, clause.Expr{SQL: chk.Constraint}}
constraintName = constraint.GetName()
constraintSql, constraintValues = constraint.Build()
} else {
return nil, nil, nil
}
Expand All @@ -182,11 +192,9 @@ func (m Migrator) CreateConstraint(value interface{}, name string) error {

func (m Migrator) DropConstraint(value interface{}, name string) error {
return m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)
if constraint != nil {
name = constraint.Name
} else if chk != nil {
name = chk.Name
name = constraint.GetName()
}

return m.recreateTable(value, &table,
Expand All @@ -200,11 +208,9 @@ func (m Migrator) DropConstraint(value interface{}, name string) error {
func (m Migrator) HasConstraint(value interface{}, name string) bool {
var count int64
m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)
if constraint != nil {
name = constraint.Name
} else if chk != nil {
name = chk.Name
name = constraint.GetName()
}

m.DB.Raw(
Expand Down Expand Up @@ -317,26 +323,44 @@ func (m Migrator) DropIndex(value interface{}, name string) error {
})
}

func buildConstraint(constraint *schema.Constraint) (sql string, results []interface{}) {
sql = "CONSTRAINT ? FOREIGN KEY ? REFERENCES ??"
if constraint.OnDelete != "" {
sql += " ON DELETE " + constraint.OnDelete
}

if constraint.OnUpdate != "" {
sql += " ON UPDATE " + constraint.OnUpdate
}

var foreignKeys, references []interface{}
for _, field := range constraint.ForeignKeys {
foreignKeys = append(foreignKeys, clause.Column{Name: field.DBName})
}
type Index struct {
Seq int
Name string
Unique bool
Origin string
Partial bool
}

for _, field := range constraint.References {
references = append(references, clause.Column{Name: field.DBName})
}
results = append(results, clause.Table{Name: constraint.Name}, foreignKeys, clause.Table{Name: constraint.ReferenceSchema.Table}, references)
return
// GetIndexes return Indexes []gorm.Index and execErr error,
// See the [doc]
//
// [doc]: https://www.sqlite.org/pragma.html#pragma_index_list
func (m Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) {
indexes := make([]gorm.Index, 0)
err := m.RunWithValue(value, func(stmt *gorm.Statement) error {
rst := make([]*Index, 0)
if err := m.DB.Debug().Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)`
return err
}
for _, index := range rst {
if index.Origin == "u" { // skip the index was created by a UNIQUE constraint
continue
}
var columns []string
if err := m.DB.Raw("SELECT name FROM PRAGMA_index_info(?)", index.Name).Scan(&columns).Error; err != nil { // alias `PRAGMA index_info(?)`
return err
}
indexes = append(indexes, &migrator.Index{
TableName: stmt.Table,
NameValue: index.Name,
ColumnList: columns,
PrimaryKeyValue: sql.NullBool{Bool: index.Origin == "pk", Valid: true}, // The exceptions are INTEGER PRIMARY KEY
UniqueValue: sql.NullBool{Bool: index.Unique, Valid: true},
})
}
return nil
})
return indexes, err
}

func (m Migrator) getRawDDL(table string) (string, error) {
Expand Down

0 comments on commit a871805

Please sign in to comment.