From 7a471771057852edf5197c8bacefe030eb534aba Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Wed, 24 Nov 2021 12:42:18 +0100 Subject: [PATCH] Database: Add migrations #319 --- internal/migrate/dialect_mysql.go | 11 +++++-- internal/migrate/dialect_sqlite.go | 11 +++++-- internal/migrate/generate.go | 35 ++++++++++++++++----- internal/migrate/migration.go | 26 ++++++--------- internal/migrate/migrations.go | 24 ++++++++++++-- internal/migrate/mysql/20211121-094727.sql | 2 +- internal/migrate/mysql/20211124-120008.sql | 2 ++ internal/migrate/sqlite/20211121-094727.sql | 2 +- internal/migrate/sqlite/20211124-120008.sql | 2 ++ 9 files changed, 80 insertions(+), 35 deletions(-) create mode 100644 internal/migrate/mysql/20211124-120008.sql create mode 100644 internal/migrate/sqlite/20211124-120008.sql diff --git a/internal/migrate/dialect_mysql.go b/internal/migrate/dialect_mysql.go index 08551606f..19e4e767d 100644 --- a/internal/migrate/dialect_mysql.go +++ b/internal/migrate/dialect_mysql.go @@ -3,8 +3,13 @@ package migrate var DialectMySQL = Migrations{ { - ID: "20211121-094727", - Dialect: "mysql", - Query: "DROP INDEX IF EXISTS uix_places_place_label ON `places`", + ID: "20211121-094727", + Dialect: "mysql", + Statements: []string{"DROP INDEX IF EXISTS uix_places_place_label ON `places`;"}, + }, + { + ID: "20211124-120008", + Dialect: "mysql", + Statements: []string{"DROP INDEX IF EXISTS idx_places_place_label ON `places`;", "DROP INDEX IF EXISTS uix_places_label ON `places`;"}, }, } diff --git a/internal/migrate/dialect_sqlite.go b/internal/migrate/dialect_sqlite.go index c6ed985a7..555607231 100644 --- a/internal/migrate/dialect_sqlite.go +++ b/internal/migrate/dialect_sqlite.go @@ -3,8 +3,13 @@ package migrate var DialectSQLite = Migrations{ { - ID: "20211121-094727", - Dialect: "sqlite", - Query: "DROP INDEX IF EXISTS idx_places_place_label", + ID: "20211121-094727", + Dialect: "sqlite", + Statements: []string{"DROP INDEX IF EXISTS idx_places_place_label;"}, + }, + { + ID: "20211124-120008", + Dialect: "sqlite", + Statements: []string{"DROP INDEX IF EXISTS uix_places_place_label;", "DROP INDEX IF EXISTS uix_places_label;"}, }, } diff --git a/internal/migrate/generate.go b/internal/migrate/generate.go index 4fcc06e47..4f052acab 100644 --- a/internal/migrate/generate.go +++ b/internal/migrate/generate.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "fmt" "os" "path/filepath" @@ -20,9 +21,9 @@ func gen_migrations(name string) { dialect := strings.ToLower(name) type Migration struct { - ID string - Dialect string - Query string + ID string + Dialect string + Statements []string } var migrations []Migration @@ -35,6 +36,23 @@ func gen_migrations(name string) { fmt.Printf("generating %s...", dialect) + strToStmts := func(b []byte) (result []string) { + stmts := bytes.Split(b, []byte(";\n")) + result = make([]string, 0, len(stmts)) + + for i := range stmts { + if s := bytes.TrimSpace(stmts[i]); len(s) > 0 { + if s[len(s)-1] != ';' { + s = append(s, ';') + } + + result = append(result, string(s)) + } + } + + return result + } + // Read migrations from files. for _, file := range files { filePath := filepath.Join(folder, file.Name()) @@ -44,9 +62,10 @@ func gen_migrations(name string) { } else if id := strings.SplitN(filepath.Base(file.Name()), ".", 2)[0]; id == "" { fmt.Printf("e") // Ignore. - } else if query, err := os.ReadFile(filePath); err == nil && len(query) > 0 { + } else if s, err := os.ReadFile(filePath); err == nil && len(s) > 0 { fmt.Printf(".") - migrations = append(migrations, Migration{ID: id, Dialect: dialect, Query: string(query)}) + + migrations = append(migrations, Migration{ID: id, Dialect: dialect, Statements: strToStmts(s)}) } else { fmt.Printf("f") fmt.Println(err.Error()) @@ -85,9 +104,9 @@ package migrate var Dialect{{ print .Name }} = Migrations{ {{- range .Migrations }} { - ID: {{ printf "%q" .ID }}, - Dialect: {{ printf "%q" .Dialect }}, - Query: {{ printf "%q" .Query }}, + ID: {{ printf "%q" .ID }}, + Dialect: {{ printf "%q" .Dialect }}, + Statements: []string{ {{ range $index, $s := .Statements}}{{if $index}},{{end}}{{ printf "%q" $s }}{{end}} }, }, {{- end }} }`)) diff --git a/internal/migrate/migration.go b/internal/migrate/migration.go index 0229933a4..101ba0d7f 100644 --- a/internal/migrate/migration.go +++ b/internal/migrate/migration.go @@ -12,7 +12,7 @@ type Migration struct { Dialect string `gorm:"size:16;" json:"Dialect" yaml:"Dialect,omitempty"` Error string `gorm:"size:255;" json:"Error" yaml:"Error,omitempty"` Source string `gorm:"size:16;" json:"Source" yaml:"Source,omitempty"` - Query string `gorm:"-" json:"Query" yaml:"Query,omitempty"` + Statements []string `gorm:"-" json:"Statements" yaml:"Statements,omitempty"` StartedAt time.Time `json:"StartedAt" yaml:"StartedAt,omitempty"` FinishedAt *time.Time `json:"FinishedAt" yaml:"FinishedAt,omitempty"` } @@ -33,25 +33,17 @@ func (m *Migration) Fail(err error, db *gorm.DB) { } // Finish updates the FinishedAt timestamp when the migration was successful. -func (m *Migration) Finish(db *gorm.DB) { - db.Model(m).Updates(Values{"FinishedAt": time.Now().UTC()}) +func (m *Migration) Finish(db *gorm.DB) error { + return db.Model(m).Updates(Values{"FinishedAt": time.Now().UTC()}).Error } // Execute runs the migration. -func (m *Migration) Execute(db *gorm.DB) { - start := time.Now() - - m.StartedAt = start.UTC().Round(time.Second) - - if err := db.Create(m).Error; err != nil { - return +func (m *Migration) Execute(db *gorm.DB) error { + for _, s := range m.Statements { + if err := db.Exec(s).Error; err != nil { + return err + } } - if err := db.Exec(m.Query).Error; err != nil { - m.Fail(err, db) - log.Errorf("migration %s failed: %s [%s]", m.ID, err, time.Since(start)) - } else { - m.Finish(db) - log.Infof("migration %s successful [%s]", m.ID, time.Since(start)) - } + return nil } diff --git a/internal/migrate/migrations.go b/internal/migrate/migrations.go index b8710f9cb..8537b32b4 100644 --- a/internal/migrate/migrations.go +++ b/internal/migrate/migrations.go @@ -1,6 +1,10 @@ package migrate -import "github.com/jinzhu/gorm" +import ( + "time" + + "github.com/jinzhu/gorm" +) // Migrations represents a sorted list of migrations. type Migrations []Migration @@ -8,6 +12,22 @@ type Migrations []Migration // Start runs all migrations that haven't been executed yet. func (m *Migrations) Start(db *gorm.DB) { for _, migration := range *m { - migration.Execute(db) + start := time.Now() + + migration.StartedAt = start.UTC().Round(time.Second) + + // Continue if already executed. + if err := db.Create(migration).Error; err != nil { + continue + } + + if err := migration.Execute(db); err != nil { + migration.Fail(err, db) + log.Errorf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start)) + } else if err = migration.Finish(db); err != nil { + log.Warnf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start)) + } else { + log.Infof("migration %s successful [%s]", migration.ID, time.Since(start)) + } } } diff --git a/internal/migrate/mysql/20211121-094727.sql b/internal/migrate/mysql/20211121-094727.sql index 5ac866d8d..832c87417 100644 --- a/internal/migrate/mysql/20211121-094727.sql +++ b/internal/migrate/mysql/20211121-094727.sql @@ -1 +1 @@ -DROP INDEX IF EXISTS uix_places_place_label ON `places` \ No newline at end of file +DROP INDEX IF EXISTS uix_places_place_label ON `places`; \ No newline at end of file diff --git a/internal/migrate/mysql/20211124-120008.sql b/internal/migrate/mysql/20211124-120008.sql new file mode 100644 index 000000000..5cef86f6e --- /dev/null +++ b/internal/migrate/mysql/20211124-120008.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_places_place_label ON `places`; +DROP INDEX IF EXISTS uix_places_label ON `places`; \ No newline at end of file diff --git a/internal/migrate/sqlite/20211121-094727.sql b/internal/migrate/sqlite/20211121-094727.sql index bd4204d36..89bc94cbd 100644 --- a/internal/migrate/sqlite/20211121-094727.sql +++ b/internal/migrate/sqlite/20211121-094727.sql @@ -1 +1 @@ -DROP INDEX IF EXISTS idx_places_place_label \ No newline at end of file +DROP INDEX IF EXISTS idx_places_place_label; diff --git a/internal/migrate/sqlite/20211124-120008.sql b/internal/migrate/sqlite/20211124-120008.sql new file mode 100644 index 000000000..7063a0451 --- /dev/null +++ b/internal/migrate/sqlite/20211124-120008.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS uix_places_place_label; +DROP INDEX IF EXISTS uix_places_label; \ No newline at end of file