diff --git a/ddlmod.go b/ddlmod.go index 9c93e6a..d00f3ef 100644 --- a/ddlmod.go +++ b/ddlmod.go @@ -12,31 +12,23 @@ import ( ) var ( - sqliteSeparator = "`|\"|'|\t" - uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^CONSTRAINT [%v]?[\w-]+[%v]? UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator)) + sqliteSeparator = "`|\"|'" + sqliteColumnQuote = "`" + uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^(?:CONSTRAINT [%v]?[\w-]+[%v]? )?UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator)) indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator)) - tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator)) + tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?(.*)$`, sqliteSeparator, sqliteSeparator)) + checkRegexp = regexp.MustCompile(`^(?i)CHECK[\s]*\(`) + constraintRegexp = regexp.MustCompile(fmt.Sprintf(`^(?i)CONSTRAINT\s+%[1]s[\w\d_]+%[1]s[\s]+`, sqliteColumnQuote)) separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator)) - columnsRegexp = regexp.MustCompile(fmt.Sprintf(`[(,][%v]?(\w+)[%v]?`, sqliteSeparator, sqliteSeparator)) columnRegexp = regexp.MustCompile(fmt.Sprintf(`^[%v]?([\w\d]+)[%v]?\s+([\w\(\)\d]+)(.*)$`, sqliteSeparator, sqliteSeparator)) defaultValueRegexp = regexp.MustCompile(`(?i) DEFAULT \(?(.+)?\)?( |COLLATE|GENERATED|$)`) regRealDataType = regexp.MustCompile(`[^\d](\d+)[^\d]?`) ) -func getAllColumns(s string) []string { - allMatches := columnsRegexp.FindAllStringSubmatch(s, -1) - columns := make([]string, 0, len(allMatches)) - for _, matches := range allMatches { - if len(matches) > 1 { - columns = append(columns, matches[1]) - } - } - return columns -} - type ddl struct { head string fields []string + suffix string // table options after the column list, e.g. WITHOUT ROWID, STRICT columns []migrator.ColumnType } @@ -54,6 +46,7 @@ func parseDDL(strs ...string) (*ddl, error) { ddlBodyRunesLen := len(ddlBodyRunes) result.head = sections[1] + result.suffix = sections[3] for idx := 0; idx < ddlBodyRunesLen; idx++ { var ( @@ -104,15 +97,15 @@ func parseDDL(strs ...string) (*ddl, error) { for _, f := range result.fields { fUpper := strings.ToUpper(f) - if strings.HasPrefix(fUpper, "CHECK") { + if checkRegexp.MatchString(f) || strings.HasPrefix(fUpper, "FOREIGN KEY") { continue } - if strings.HasPrefix(fUpper, "CONSTRAINT") { - matches := uniqueRegexp.FindStringSubmatch(f) + if matches := uniqueRegexp.FindStringSubmatch(f); matches != nil { if len(matches) > 0 { - if columns := getAllColumns(matches[1]); len(columns) == 1 { + cols, err := parseAllColumns(matches[1]) + if err == nil && len(cols) == 1 { for idx, column := range result.columns { - if column.NameValue.String == columns[0] { + if column.NameValue.String == cols[0] { column.UniqueValue = sql.NullBool{Bool: true, Valid: true} result.columns[idx] = column break @@ -122,13 +115,19 @@ func parseDDL(strs ...string) (*ddl, error) { } continue } + if constraintRegexp.MatchString(f) { + continue + } if strings.HasPrefix(fUpper, "PRIMARY KEY") { - for _, name := range getAllColumns(f) { - for idx, column := range result.columns { - if column.NameValue.String == name { - column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} - result.columns[idx] = column - break + cols, err := parseAllColumns(f) + if err == nil { + for _, name := range cols { + for idx, column := range result.columns { + if column.NameValue.String == name { + column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} + result.columns[idx] = column + break + } } } } @@ -196,10 +195,10 @@ func (d *ddl) clone() *ddl { func (d *ddl) compile() string { if len(d.fields) == 0 { - return d.head + return d.head + d.suffix } - return fmt.Sprintf("%s (%s)", d.head, strings.Join(d.fields, ",")) + return fmt.Sprintf("%s (%s)%s", d.head, strings.Join(d.fields, ","), d.suffix) } func (d *ddl) renameTable(dst, src string) error { @@ -217,8 +216,12 @@ func (d *ddl) renameTable(dst, src string) error { return nil } +func compileConstraintRegexp(name string) *regexp.Regexp { + return regexp.MustCompile("^(?i:CONSTRAINT)\\s+[\"`]?" + regexp.QuoteMeta(name) + "[\"`\\s]") +} + func (d *ddl) addConstraint(name string, sql string) { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + reg := compileConstraintRegexp(name) for i := 0; i < len(d.fields); i++ { if reg.MatchString(d.fields[i]) { @@ -231,7 +234,7 @@ func (d *ddl) addConstraint(name string, sql string) { } func (d *ddl) removeConstraint(name string) bool { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + reg := compileConstraintRegexp(name) for i := 0; i < len(d.fields); i++ { if reg.MatchString(d.fields[i]) { @@ -243,7 +246,7 @@ func (d *ddl) removeConstraint(name string) bool { } func (d *ddl) hasConstraint(name string) bool { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + reg := compileConstraintRegexp(name) for _, f := range d.fields { if reg.MatchString(f) { @@ -259,12 +262,15 @@ func (d *ddl) getColumns() []string { for _, f := range d.fields { fUpper := strings.ToUpper(f) if strings.HasPrefix(fUpper, "PRIMARY KEY") || - strings.HasPrefix(fUpper, "CHECK") || - strings.HasPrefix(fUpper, "CONSTRAINT") || + strings.HasPrefix(fUpper, "FOREIGN KEY") || strings.Contains(fUpper, "GENERATED ALWAYS AS") { continue } + if checkRegexp.MatchString(f) || constraintRegexp.MatchString(f) || uniqueRegexp.MatchString(f) { + continue + } + reg := regexp.MustCompile("^[\"`']?([\\w\\d]+)[\"`']?") match := reg.FindStringSubmatch(f) diff --git a/ddlmod_parse_all_columns.go b/ddlmod_parse_all_columns.go new file mode 100644 index 0000000..760acf8 --- /dev/null +++ b/ddlmod_parse_all_columns.go @@ -0,0 +1,117 @@ +package sqlite + +import ( + "errors" + "fmt" +) + +type parseAllColumnsState int + +const ( + parseAllColumnsState_NONE parseAllColumnsState = iota + parseAllColumnsState_Beginning + parseAllColumnsState_ReadingRawName + parseAllColumnsState_ReadingQuotedName + parseAllColumnsState_EndOfName + parseAllColumnsState_State_End +) + +func parseAllColumns(in string) ([]string, error) { + s := []rune(in) + columns := make([]string, 0) + state := parseAllColumnsState_NONE + quote := rune(0) + name := make([]rune, 0) + for i := 0; i < len(s); i++ { + switch state { + case parseAllColumnsState_NONE: + if s[i] == '(' { + state = parseAllColumnsState_Beginning + } + case parseAllColumnsState_Beginning: + if isSpace(s[i]) { + continue + } + if isQuote(s[i]) { + state = parseAllColumnsState_ReadingQuotedName + quote = s[i] + continue + } + if s[i] == '[' { + state = parseAllColumnsState_ReadingQuotedName + quote = ']' + continue + } else if s[i] == ')' { + return columns, fmt.Errorf("unexpected token: %s", string(s[i])) + } + state = parseAllColumnsState_ReadingRawName + name = append(name, s[i]) + case parseAllColumnsState_ReadingRawName: + if isSeparator(s[i]) { + state = parseAllColumnsState_Beginning + columns = append(columns, string(name)) + name = make([]rune, 0) + continue + } + if s[i] == ')' { + state = parseAllColumnsState_State_End + columns = append(columns, string(name)) + } + if isQuote(s[i]) { + return nil, fmt.Errorf("unexpected token: %s", string(s[i])) + } + if isSpace(s[i]) { + state = parseAllColumnsState_EndOfName + columns = append(columns, string(name)) + name = make([]rune, 0) + continue + } + name = append(name, s[i]) + case parseAllColumnsState_ReadingQuotedName: + if s[i] == quote { + // check if quote character is escaped + if i+1 < len(s) && s[i+1] == quote { + name = append(name, quote) + i++ + continue + } + state = parseAllColumnsState_EndOfName + columns = append(columns, string(name)) + name = make([]rune, 0) + continue + } + name = append(name, s[i]) + case parseAllColumnsState_EndOfName: + if isSpace(s[i]) { + continue + } + if isSeparator(s[i]) { + state = parseAllColumnsState_Beginning + continue + } + if s[i] == ')' { + state = parseAllColumnsState_State_End + continue + } + return nil, fmt.Errorf("unexpected token: %s", string(s[i])) + case parseAllColumnsState_State_End: + break + } + } + if state != parseAllColumnsState_State_End { + return nil, errors.New("unexpected end") + } + return columns, nil +} + +func isSpace(r rune) bool { + return r == ' ' || r == '\t' +} + +func isQuote(r rune) bool { + return r == '`' || r == '"' || r == '\'' +} + +func isSeparator(r rune) bool { + return r == ',' +} diff --git a/ddlmod_parse_all_columns_test.go b/ddlmod_parse_all_columns_test.go new file mode 100644 index 0000000..eb70cdd --- /dev/null +++ b/ddlmod_parse_all_columns_test.go @@ -0,0 +1,48 @@ +package sqlite + +import "testing" + +func TestParseAllColumns(t *testing.T) { + tc := []struct { + name string + input string + expected []string + }{ + { + name: "Simple case", + input: "PRIMARY KEY (column1, column2)", + expected: []string{"column1", "column2"}, + }, + { + name: "Quoted column name", + input: "PRIMARY KEY (`column,xxx`, \"column 2\", \"column)3\", 'column''4', \"column\"\"5\")", + expected: []string{"column,xxx", "column 2", "column)3", "column'4", "column\"5"}, + }, + { + name: "Japanese column name", + input: "PRIMARY KEY (カラム1, `カラム2`)", + expected: []string{"カラム1", "カラム2"}, + }, + { + name: "Column name quoted with []", + input: "PRIMARY KEY ([column1], [column2])", + expected: []string{"column1", "column2"}, + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + cols, err := parseAllColumns(tt.input) + if err != nil { + t.Errorf("Failed to parse columns: %s", err) + } + if len(cols) != len(tt.expected) { + t.Errorf("Expected %d columns, got %d", len(tt.expected), len(cols)) + } + for i, col := range cols { + if col != tt.expected[i] { + t.Errorf("Expected %s, got %s", tt.expected[i], col) + } + } + }) + } +} diff --git a/ddlmod_test.go b/ddlmod_test.go index 5b5ec4a..7d4e1a2 100644 --- a/ddlmod_test.go +++ b/ddlmod_test.go @@ -143,15 +143,22 @@ func TestParseDDL(t *testing.T) { }, }, { - "index with \n from .schema sqlite", + "column name and the column type are separated by a single horizontal tab character", []string{ - "CREATE TABLE `test-d` (`field` integer NOT NULL)", - "CREATE INDEX `idx_uq`\n ON `test-b`(`field`) WHERE field = 0", + "CREATE TABLE `test-e` (`field1`\ttext NOT NULL,`field2`\tinteger NOT NULL)", }, - 1, + 2, []migrator.ColumnType{ { - NameValue: sql.NullString{String: "field", Valid: true}, + NameValue: sql.NullString{String: "field1", Valid: true}, + DataTypeValue: sql.NullString{String: "text", Valid: true}, + ColumnTypeValue: sql.NullString{String: "text", Valid: true}, + PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true}, + UniqueValue: sql.NullBool{Bool: false, Valid: true}, + NullableValue: sql.NullBool{Bool: false, Valid: true}, + }, + { + NameValue: sql.NullString{String: "field2", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, PrimaryKeyValue: sql.NullBool{Bool: false, Valid: true}, @@ -160,6 +167,26 @@ func TestParseDDL(t *testing.T) { }, }, }, + {"with a check-like column", []string{"CREATE TABLE Docs (ID int NOT NULL,Checksum text NOT NULL)"}, 2, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "Checksum", Valid: true}, DataTypeValue: sql.NullString{String: "text", Valid: true}, ColumnTypeValue: sql.NullString{String: "text", Valid: true}, NullableValue: sql.NullBool{Bool: false, Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, + {"with a constraint-like column", []string{"CREATE TABLE Docs (ID int NOT NULL,constraints text NOT NULL)"}, 2, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "constraints", Valid: true}, DataTypeValue: sql.NullString{String: "text", Valid: true}, ColumnTypeValue: sql.NullString{String: "text", Valid: true}, NullableValue: sql.NullBool{Bool: false, Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, + {"with a unique-like column", []string{"CREATE TABLE Docs (ID int NOT NULL,unique_code text NOT NULL)"}, 2, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "unique_code", Valid: true}, DataTypeValue: sql.NullString{String: "text", Valid: true}, ColumnTypeValue: sql.NullString{String: "text", Valid: true}, NullableValue: sql.NullBool{Bool: false, Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, + {"with_fk_no_constraint", []string{"CREATE TABLE Docs (ID int NOT NULL,UserID int NOT NULL,FOREIGN KEY (UserID) REFERENCES Users(ID))"}, 3, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "UserID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, + {"with unique without constraint", []string{"CREATE TABLE `users` (`id` text NOT NULL,`email` text NOT NULL,PRIMARY KEY (`id`),UNIQUE (`email`))"}, 4, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "id", Valid: true}, DataTypeValue: sql.NullString{String: "text", Valid: true}, ColumnTypeValue: sql.NullString{String: "text", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true, Bool: true}}, + {NameValue: sql.NullString{String: "email", Valid: true}, DataTypeValue: sql.NullString{String: "text", Valid: true}, ColumnTypeValue: sql.NullString{String: "text", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: false}, UniqueValue: sql.NullBool{Valid: true, Bool: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, } for _, p := range params { @@ -331,6 +358,41 @@ func TestRemoveConstraint(t *testing.T) { success: true, expect: []string{"`id` integer NOT NULL"}, }, + { + name: "lowercase", + fields: []string{"`id` integer NOT NULL", "constraint `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "mixed_case", + fields: []string{"`id` integer NOT NULL", "cOnsTraiNT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "newline", + fields: []string{"`id` integer NOT NULL", "CONSTRAINT `fk_users_notes`\nFOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "lots_of_newlines", + fields: []string{"`id` integer NOT NULL", "constraint \n fk_users_notes \n FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "no_backtick", + fields: []string{"`id` integer NOT NULL", "CONSTRAINT fk_users_notes FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, { name: "check", fields: []string{"CONSTRAINT `name_checker` CHECK (`name` <> 'thetadev')", "`id` integer NOT NULL"}, diff --git a/generated_test.go b/generated_test.go new file mode 100644 index 0000000..70fdfc0 --- /dev/null +++ b/generated_test.go @@ -0,0 +1,56 @@ +package sqlite + +import ( + "testing" + + "gorm.io/gorm/schema" +) + +func TestDataTypeOfGeneratedColumn(t *testing.T) { + dialector := Dialector{} + tests := []struct { + name string + field *schema.Field + want string + }{ + { + name: "computed column renders a STORED generated column", + field: &schema.Field{DataType: schema.Float, TagSettings: map[string]string{"GENERATED": "price * quantity"}}, + want: "real GENERATED ALWAYS AS (price * quantity) STORED", + }, + { + name: "computed expression keeps commas", + field: &schema.Field{DataType: schema.String, TagSettings: map[string]string{"GENERATED": "coalesce(first_name, last_name)"}}, + want: "text GENERATED ALWAYS AS (coalesce(first_name, last_name)) STORED", + }, + { + // `identity` is reserved for identity columns, which SQLite renders + // through its native AUTOINCREMENT rather than a computed column. + name: "identity keyword is not treated as a computed column", + field: &schema.Field{DataType: schema.Int, AutoIncrement: true, TagSettings: map[string]string{"GENERATED": "identity"}}, + want: "integer PRIMARY KEY AUTOINCREMENT", + }, + { + name: "identity with an explicit mode is also reserved", + field: &schema.Field{DataType: schema.Int, AutoIncrement: true, TagSettings: map[string]string{"GENERATED": "identity always"}}, + want: "integer PRIMARY KEY AUTOINCREMENT", + }, + { + name: "a bare generated tag is ignored", + field: &schema.Field{DataType: schema.Float, TagSettings: map[string]string{"GENERATED": "GENERATED"}}, + want: "real", + }, + { + name: "a lowercase generated expression is not mistaken for a bare tag", + field: &schema.Field{DataType: schema.Float, TagSettings: map[string]string{"GENERATED": "generated"}}, + want: "real GENERATED ALWAYS AS (generated) STORED", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dialector.DataTypeOf(tt.field); got != tt.want { + t.Errorf("DataTypeOf() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/migrator.go b/migrator.go index 3256f19..ae15d4a 100644 --- a/migrator.go +++ b/migrator.go @@ -66,8 +66,8 @@ func (m Migrator) HasColumn(value interface{}, name string) bool { if name != "" { m.DB.Raw( - "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", - "table", stmt.Table, `%"`+name+`" %`, `%`+name+` %`, "%`"+name+"`%", "%["+name+"]%", "%\t"+name+"\t%", + "SELECT count(*) FROM pragma_table_info(?) WHERE name = ? COLLATE NOCASE", + stmt.Table, name, ).Row().Scan(&count) } return nil @@ -155,13 +155,15 @@ func (m Migrator) ColumnTypes(value interface{}) ([]gorm.ColumnType, error) { } func (m Migrator) DropColumn(value interface{}, name string) error { - return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { - if field := stmt.Schema.LookUpField(name); field != nil { - name = field.DBName - } + return m.RunWithoutForeignKey(func() error { + return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { + if field := stmt.Schema.LookUpField(name); field != nil { + name = field.DBName + } - ddl.removeColumn(name) - return ddl, nil, nil + ddl.removeColumn(name) + return ddl, nil, nil + }) }) } @@ -339,7 +341,7 @@ func (m Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) { indexes := make([]gorm.Index, 0) err := m.RunWithValue(value, func(stmt *gorm.Statement) error { rst := make([]*Index, 0) - if err := m.DB.Debug().Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)` + if err := m.DB.Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)` return err } for _, index := range rst { @@ -409,6 +411,16 @@ func (m Migrator) recreateTable( columns := createDDL.getColumns() createSQL := createDDL.compile() + // indexes and triggers are dropped together with the old table; save + // their DDL so they can be recreated on the rebuilt table. + var auxDDLs []string + if err := m.DB.Raw( + "SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN (?, ?) AND sql IS NOT NULL", + table, "index", "trigger", + ).Scan(&auxDDLs).Error; err != nil { + return err + } + return m.DB.Transaction(func(tx *gorm.DB) error { if err := tx.Exec(createSQL, sqlArgs...).Error; err != nil { return err @@ -417,13 +429,38 @@ func (m Migrator) recreateTable( queries := []string{ fmt.Sprintf("INSERT INTO `%v`(%v) SELECT %v FROM `%v`", newTableName, strings.Join(columns, ","), strings.Join(columns, ","), table), fmt.Sprintf("DROP TABLE `%v`", table), - fmt.Sprintf("ALTER TABLE `%v` RENAME TO `%v`", newTableName, table), } for _, query := range queries { if err := tx.Exec(query).Error; err != nil { return err } } + + // legacy_alter_table keeps RENAME from re-resolving views that + // reference the table; they point at the original name and become + // valid again right after the rename. + if err := tx.Exec("PRAGMA legacy_alter_table = ON").Error; err != nil { + return err + } + renameErr := tx.Exec(fmt.Sprintf("ALTER TABLE `%v` RENAME TO `%v`", newTableName, table)).Error + if err := tx.Exec("PRAGMA legacy_alter_table = OFF").Error; renameErr == nil { + renameErr = err + } + if renameErr != nil { + return renameErr + } + + // recreate the saved indexes and triggers; ones referencing a + // column that no longer exists cannot apply anymore and are + // skipped, any other failure aborts the migration + for _, aux := range auxDDLs { + if err := tx.Exec(aux).Error; err != nil { + if strings.Contains(err.Error(), "no such column") { + continue + } + return err + } + } return nil }) }) diff --git a/migrator_test.go b/migrator_test.go new file mode 100644 index 0000000..301d3bc --- /dev/null +++ b/migrator_test.go @@ -0,0 +1,180 @@ +package sqlite + +import ( + "strings" + "testing" + + "gorm.io/gorm" +) + +// HasColumn used to match the column name as a LIKE substring against the +// table DDL, which reported columns as present when the name merely appeared +// inside another column name or a string default value. AutoMigrate then +// skipped AddColumn and later queries failed with "no such column". +func TestHasColumnNoFalsePositive(t *testing.T) { + db, err := gorm.Open(Open("file:hascolumn_exact?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("gorm.Open: %v", err) + } + // close the pool so the shared in-memory database is torn down and + // repeated runs (go test -count=N) start from a clean state + t.Cleanup(func() { + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } + }) + + // name is a substring of first_name in an unquoted DDL + if err := db.Exec("CREATE TABLE plaincols (id integer, first_name text)").Error; err != nil { + t.Fatal(err) + } + if db.Migrator().HasColumn("plaincols", "name") { + t.Error(`HasColumn("plaincols", "name") = true, want false (only first_name exists)`) + } + if !db.Migrator().HasColumn("plaincols", "first_name") { + t.Error(`HasColumn("plaincols", "first_name") = false, want true`) + } + + // name appears inside a string default value + if err := db.Exec("CREATE TABLE `defv` (`id` integer, `cfg` text DEFAULT 'name value')").Error; err != nil { + t.Fatal(err) + } + if db.Migrator().HasColumn("defv", "name") { + t.Error(`HasColumn("defv", "name") = true, want false (name only appears in a default value)`) + } + if !db.Migrator().HasColumn("defv", "cfg") { + t.Error(`HasColumn("defv", "cfg") = false, want true`) + } +} + +type RebuildIdxTable struct { + ID int + A string + B string +} + +func (RebuildIdxTable) TableName() string { return "rebuild_idx_table" } + +// recreateTable drops the old table together with its indexes and triggers; +// they must be recreated on the rebuilt table. Indexes on a dropped column +// are the exception: they can no longer apply. +func TestRecreateTablePreservesIndexesAndTriggers(t *testing.T) { + db := openRecreateTestDB(t, "recreate_idx") + stmts := []string{ + "CREATE TABLE `rebuild_idx_table` (`id` integer, `a` text, `b` text)", + "CREATE INDEX `idx_keep` ON `rebuild_idx_table`(`a`)", + "CREATE INDEX `idx_gone` ON `rebuild_idx_table`(`b`)", + "CREATE TABLE `rebuild_audit` (`msg` text)", + "CREATE TRIGGER `trg_keep` AFTER INSERT ON `rebuild_idx_table` BEGIN INSERT INTO `rebuild_audit`(`msg`) VALUES ('x'); END", + } + for _, s := range stmts { + if err := db.Exec(s).Error; err != nil { + t.Fatal(err) + } + } + + if err := db.Migrator().DropColumn(&RebuildIdxTable{}, "b"); err != nil { + t.Fatalf("DropColumn: %v", err) + } + + var names []string + if err := db.Raw("SELECT name FROM sqlite_master WHERE tbl_name = 'rebuild_idx_table' AND type IN ('index','trigger')").Scan(&names).Error; err != nil { + t.Fatalf("querying sqlite_master: %v", err) + } + got := strings.Join(names, ",") + if !strings.Contains(got, "idx_keep") { + t.Errorf("index idx_keep lost after DropColumn, remaining: %v", names) + } + if !strings.Contains(got, "trg_keep") { + t.Errorf("trigger trg_keep lost after DropColumn, remaining: %v", names) + } + if strings.Contains(got, "idx_gone") { + t.Errorf("index idx_gone references the dropped column and must not survive, remaining: %v", names) + } + + // the trigger still works on the rebuilt table + if err := db.Exec("INSERT INTO `rebuild_idx_table`(`id`,`a`) VALUES (1,'v')").Error; err != nil { + t.Fatal(err) + } + var auditCount int + if err := db.Raw("SELECT count(*) FROM rebuild_audit").Scan(&auditCount).Error; err != nil { + t.Fatalf("querying rebuild_audit: %v", err) + } + if auditCount != 1 { + t.Errorf("trigger did not fire after rebuild, audit rows = %d", auditCount) + } +} + +type RebuildOptsTable struct { + ID string `gorm:"primaryKey"` + A string + B string +} + +func (RebuildOptsTable) TableName() string { return "rebuild_opts_table" } + +// Table options after the column list (WITHOUT ROWID, STRICT) must survive a +// table rebuild instead of silently turning the table into a plain rowid one. +func TestRecreateTablePreservesTableOptions(t *testing.T) { + db := openRecreateTestDB(t, "recreate_opts") + if err := db.Exec("CREATE TABLE `rebuild_opts_table` (`id` text PRIMARY KEY, `a` text, `b` text) WITHOUT ROWID").Error; err != nil { + t.Fatal(err) + } + if err := db.Migrator().DropColumn(&RebuildOptsTable{}, "b"); err != nil { + t.Fatalf("DropColumn: %v", err) + } + + var ddl string + if err := db.Raw("SELECT sql FROM sqlite_master WHERE type='table' AND name='rebuild_opts_table'").Scan(&ddl).Error; err != nil { + t.Fatalf("querying sqlite_master: %v", err) + } + if !strings.Contains(ddl, "WITHOUT ROWID") { + t.Errorf("WITHOUT ROWID lost after rebuild: %s", ddl) + } +} + +type RebuildViewedTable struct { + ID int + A string + B string +} + +func (RebuildViewedTable) TableName() string { return "rebuild_viewed" } + +// Rebuilding a table that a view references used to fail with "error in view +// ...: no such table" because the RENAME step re-resolves the view while the +// original table name doesn't exist yet (issue #225). +func TestRecreateTableWithView(t *testing.T) { + db := openRecreateTestDB(t, "recreate_view") + if err := db.Exec("CREATE TABLE `rebuild_viewed` (`id` integer, `a` text, `b` text)").Error; err != nil { + t.Fatal(err) + } + if err := db.Exec("CREATE VIEW `rebuild_viewed_v` AS SELECT `a` FROM `rebuild_viewed`").Error; err != nil { + t.Fatal(err) + } + + if err := db.Migrator().DropColumn(&RebuildViewedTable{}, "b"); err != nil { + t.Fatalf("DropColumn with a referencing view: %v", err) + } + + var n int + if err := db.Raw("SELECT count(*) FROM `rebuild_viewed_v`").Scan(&n).Error; err != nil { + t.Errorf("view is broken after rebuild: %v", err) + } +} + +func openRecreateTestDB(t *testing.T, name string) *gorm.DB { + t.Helper() + db, err := gorm.Open(Open("file:"+name+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("gorm.Open: %v", err) + } + // close the pool so the shared in-memory database is torn down and + // repeated runs (go test -count=N) start from a clean state + t.Cleanup(func() { + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } + }) + return db +} diff --git a/sqlite.go b/sqlite.go index b84147e..caef1b9 100644 --- a/sqlite.go +++ b/sqlite.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "strconv" + "strings" "gorm.io/gorm/callbacks" @@ -26,10 +27,20 @@ type Dialector struct { Conn gorm.ConnPool } +type Config struct { + DriverName string + DSN string + Conn gorm.ConnPool +} + func Open(dsn string) gorm.Dialector { return &Dialector{DSN: dsn} } +func New(config Config) gorm.Dialector { + return &Dialector{DSN: config.DSN, DriverName: config.DriverName, Conn: config.Conn} +} + func (dialector Dialector) Name() string { return "sqlite" } @@ -68,7 +79,9 @@ func (dialector Dialector) Initialize(db *gorm.DB) (err error) { } for k, v := range dialector.ClauseBuilders() { - db.ClauseBuilders[k] = v + if _, ok := db.ClauseBuilders[k]; !ok { + db.ClauseBuilders[k] = v + } } return } @@ -196,6 +209,14 @@ func (dialector Dialector) Explain(sql string, vars ...interface{}) string { } func (dialector Dialector) DataTypeOf(field *schema.Field) string { + if expr, ok := generatedColumnExpr(field); ok { + return dialector.dataTypeOf(field) + " GENERATED ALWAYS AS (" + expr + ") STORED" + } + + return dialector.dataTypeOf(field) +} + +func (dialector Dialector) dataTypeOf(field *schema.Field) string { switch field.DataType { case schema.Bool: return "numeric" @@ -273,3 +294,40 @@ func compareVersion(version1, version2 string) int { } return 0 } + +// generatedColumnExpr returns the expression of a computed (generated) column +// declared via the `generated` tag, if any. The `identity` keyword is reserved +// for identity columns (rendered through the dialect's native auto-increment) +// and is not a computed-column expression. +func generatedColumnExpr(field *schema.Field) (string, bool) { + value, ok := field.TagSettings["GENERATED"] + if !ok { + return "", false + } + // Ignore an empty value or a bare `generated` tag, which the tag parser + // stores as the upper-cased key, rather than treating it as an expression. + if value = strings.TrimSpace(value); value == "" || value == "GENERATED" { + return "", false + } + if isIdentityKeyword(value) { + return "", false + } + return value, true +} + +// isIdentityKeyword reports whether value is the `identity` keyword, optionally +// combined with the generation mode `always` / `by default`. Any other token +// means value is a computed-column expression. +func isIdentityKeyword(value string) bool { + identity := false + for _, token := range strings.Fields(strings.ToLower(value)) { + switch token { + case "identity": + identity = true + case "always", "by", "default": + default: + return false + } + } + return identity +}