From 20feb6f0a0e3028e661bd5e2e6f7098258119060 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Dec 2020 12:46:28 +0100 Subject: [PATCH] Database: Add backup command and make config more compatible #460 --- cmd/photoprism/photoprism.go | 1 + internal/commands/backup.go | 120 +++++++++++++++++++++++++++++++++ internal/commands/config.go | 6 ++ internal/config/config_test.go | 14 ---- internal/config/db.go | 109 +++++++++++++++++++++++++++++- internal/config/db_test.go | 68 +++++++++++++++++++ internal/config/filenames.go | 10 +++ internal/config/flags.go | 24 ++++++- internal/config/params.go | 4 ++ 9 files changed, 340 insertions(+), 16 deletions(-) create mode 100644 internal/commands/backup.go create mode 100644 internal/config/db_test.go diff --git a/cmd/photoprism/photoprism.go b/cmd/photoprism/photoprism.go index 08e3567eb..c82874041 100644 --- a/cmd/photoprism/photoprism.go +++ b/cmd/photoprism/photoprism.go @@ -62,6 +62,7 @@ func main() { commands.ConvertCommand, commands.ResampleCommand, commands.MigrateCommand, + commands.BackupCommand, commands.ResetCommand, commands.ConfigCommand, commands.PasswdCommand, diff --git a/internal/commands/backup.go b/internal/commands/backup.go new file mode 100644 index 000000000..6e957503c --- /dev/null +++ b/internal/commands/backup.go @@ -0,0 +1,120 @@ +package commands + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/photoprism/photoprism/pkg/txt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/photoprism/photoprism/internal/config" + "github.com/urfave/cli" +) + +// BackupCommand configures the backup cli command. +var BackupCommand = cli.Command{ + Name: "backup", + Usage: "Creates a database backup", + Flags: backupFlags, + Action: backupAction, +} + +var backupFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "force, f", + Usage: "overwrite existing backup files", + }, +} + +// migrateAction automatically migrates or initializes database +func backupAction(ctx *cli.Context) error { + start := time.Now() + + conf := config.NewConfig(ctx) + + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := conf.Init(); err != nil { + return err + } + + // Use command argument as backup file name. + fileName := ctx.Args().First() + + // If empty, use default backup file name. + if fileName == "" { + backupFile := time.Now().UTC().Format("2006-01-02") + ".sql" + backupPath := filepath.Join(conf.BackupPath(), conf.DatabaseDriver()) + fileName = filepath.Join(backupPath, backupFile) + } + + if _, err := os.Stat(fileName); err == nil && !ctx.Bool("force") { + log.Errorf("backup file already exists: %s", fileName) + return nil + } else if err == nil { + log.Warnf("replacing existing backup file") + } + + // Create backup directory if not exists. + if dir := filepath.Dir(fileName); dir != "." { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + } + + log.Infof("backing up database to %s", txt.Quote(fileName)) + + var cmd *exec.Cmd + + switch conf.DatabaseDriver() { + case config.MySQL: + cmd = exec.Command( + conf.MysqldumpBin(), + "-h", conf.DatabaseHost(), + "-P", conf.DatabasePortString(), + "-u", conf.DatabaseUser(), + "-p"+conf.DatabasePassword(), + conf.DatabaseName(), + ) + case config.SQLite: + cmd = exec.Command( + conf.SqliteBin(), + conf.DatabaseDsn(), + ".dump", + ) + default: + return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver()) + } + + // Fetch command output. + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + // Run backup command. + if err := cmd.Run(); err != nil { + if stderr.String() != "" { + return errors.New(stderr.String()) + } + } + + // Write output to file. + if err := ioutil.WriteFile(fileName, []byte(out.String()), os.ModePerm); err != nil { + return err + } + + elapsed := time.Since(start) + + log.Infof("database backup completed in %s", elapsed) + + conf.Shutdown() + + return nil +} diff --git a/internal/commands/config.go b/internal/commands/config.go index 297c3c8d1..087c5d646 100644 --- a/internal/commands/config.go +++ b/internal/commands/config.go @@ -63,6 +63,12 @@ func configAction(ctx *cli.Context) error { // Database configuration. fmt.Printf("%-25s %s\n", "database-driver", dbDriver) fmt.Printf("%-25s %s\n", "database-dsn", dbDsn) + fmt.Printf("%-25s %s\n", "database-server", conf.DatabaseServer()) + fmt.Printf("%-25s %s\n", "database-host", conf.DatabaseHost()) + fmt.Printf("%-25s %s\n", "database-port", conf.DatabasePortString()) + fmt.Printf("%-25s %s\n", "database-name", conf.DatabaseName()) + fmt.Printf("%-25s %s\n", "database-user", conf.DatabaseUser()) + fmt.Printf("%-25s %s\n", "database-password", conf.DatabasePassword()) fmt.Printf("%-25s %d\n", "database-conns", conf.DatabaseConns()) fmt.Printf("%-25s %d\n", "database-conns-idle", conf.DatabaseConnsIdle()) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 832829576..a36454072 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -162,20 +162,6 @@ func TestConfig_ExifToolBin(t *testing.T) { assert.Equal(t, "/usr/bin/exiftool", bin) } -func TestConfig_DatabaseDriver(t *testing.T) { - c := NewConfig(CliTestContext()) - - driver := c.DatabaseDriver() - assert.Equal(t, SQLite, driver) -} - -func TestConfig_DatabaseDsn(t *testing.T) { - c := NewConfig(CliTestContext()) - - dsn := c.DatabaseDriver() - assert.Equal(t, SQLite, dsn) -} - func TestConfig_CachePath(t *testing.T) { c := NewConfig(CliTestContext()) diff --git a/internal/config/db.go b/internal/config/db.go index 367511f24..444188959 100644 --- a/internal/config/db.go +++ b/internal/config/db.go @@ -2,9 +2,12 @@ package config import ( "errors" + "fmt" "io/ioutil" "path/filepath" + "regexp" "runtime" + "strconv" "strings" "time" @@ -15,6 +18,12 @@ import ( "github.com/photoprism/photoprism/internal/mutex" ) +var dsnPattern = regexp.MustCompile( +`^(?:(?P.*?)(?::(?P.*))?@)?` + +`(?:(?P[^\(]*)(?:\((?P[^\)]*)\))?)?` + +`\/(?P.*?)` + +`(?:\?(?P[^\?]*))?$`) + // DatabaseDriver returns the database driver name. func (c *Config) DatabaseDriver() string { switch strings.ToLower(c.params.DatabaseDriver) { @@ -40,7 +49,13 @@ func (c *Config) DatabaseDsn() string { if c.params.DatabaseDsn == "" { switch c.DatabaseDriver() { case MySQL: - return "photoprism:photoprism@tcp(photoprism-db:3306)/photoprism?parseTime=true" + return fmt.Sprintf( + "%s:%s@tcp(%s)/%s?charset=utf8mb4,utf8&parseTime=true", + c.DatabaseUser(), + c.DatabasePassword(), + c.DatabaseServer(), + c.DatabaseName(), + ) case SQLite: return filepath.Join(c.StoragePath(), "index.db") default: @@ -52,6 +67,98 @@ func (c *Config) DatabaseDsn() string { return c.params.DatabaseDsn } +// ParseDatabaseDsn parses the database dsn and extracts user, password, database server, and name. +func (c *Config) ParseDatabaseDsn() { + if c.params.DatabaseDsn == "" || c.params.DatabaseServer != "" { + return + } + + matches := dsnPattern.FindStringSubmatch(c.params.DatabaseDsn) + names := dsnPattern.SubexpNames() + + for i, match := range matches { + switch names[i] { + case "user": + c.params.DatabaseUser = match + case "password": + c.params.DatabasePassword = match + case "server": + c.params.DatabaseServer = match + case "name": + c.params.DatabaseName = match + } + } +} + +// DatabaseServer the database server. +func (c *Config) DatabaseServer() string { + c.ParseDatabaseDsn() + + if c.params.DatabaseServer == "" { + return "localhost" + } + + return c.params.DatabaseServer +} + +// DatabaseHost the database server host. +func (c *Config) DatabaseHost() string { + if s := strings.Split(c.DatabaseServer(), ":"); len(s) > 0 { + return s[0] + } + + return c.params.DatabaseServer +} + +// DatabasePort the database server port. +func (c *Config) DatabasePort() int { + const defaultPort = 3306 + + if s := strings.Split(c.DatabaseServer(), ":"); len(s) != 2 { + return defaultPort + } else if port, err := strconv.Atoi(s[1]); err != nil { + return defaultPort + } else if port < 1 || port > 65535 { + return defaultPort + } else { + return port + } +} + +// DatabasePortString the database server port as string. +func (c *Config) DatabasePortString() string { + return strconv.Itoa(c.DatabasePort()) +} + +// DatabaseName the database schema name. +func (c *Config) DatabaseName() string { + c.ParseDatabaseDsn() + + if c.params.DatabaseName == "" { + return "photoprism" + } + + return c.params.DatabaseName +} + +// DatabaseUser returns the database user name. +func (c *Config) DatabaseUser() string { + c.ParseDatabaseDsn() + + if c.params.DatabaseUser == "" { + return "photoprism" + } + + return c.params.DatabaseUser +} + +// DatabasePassword returns the database user password. +func (c *Config) DatabasePassword() string { + c.ParseDatabaseDsn() + + return c.params.DatabasePassword +} + // DatabaseConns returns the maximum number of open connections to the database. func (c *Config) DatabaseConns() int { limit := c.params.DatabaseConns diff --git a/internal/config/db_test.go b/internal/config/db_test.go new file mode 100644 index 000000000..2a588e114 --- /dev/null +++ b/internal/config/db_test.go @@ -0,0 +1,68 @@ +package config + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestConfig_DatabaseDriver(t *testing.T) { + c := NewConfig(CliTestContext()) + + driver := c.DatabaseDriver() + assert.Equal(t, SQLite, driver) +} + +func TestConfig_ParseDatabaseDsn(t *testing.T) { + c := NewConfig(CliTestContext()) + c.params.DatabaseDsn ="foo:b@r@tcp(honeypot:1234)/baz?charset=utf8mb4,utf8&parseTime=true" + + assert.Equal(t, "honeypot:1234", c.DatabaseServer()) + assert.Equal(t, "honeypot", c.DatabaseHost()) + assert.Equal(t, 1234, c.DatabasePort()) + assert.Equal(t, "baz", c.DatabaseName()) + assert.Equal(t, "foo", c.DatabaseUser()) + assert.Equal(t, "b@r", c.DatabasePassword()) +} + +func TestConfig_DatabaseServer(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, "localhost", c.DatabaseServer()) +} + +func TestConfig_DatabaseHost(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, "localhost", c.DatabaseHost()) +} + +func TestConfig_DatabasePort(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, 3306, c.DatabasePort()) +} + +func TestConfig_DatabaseName(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, "photoprism", c.DatabaseName()) +} + +func TestConfig_DatabaseUser(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, "photoprism", c.DatabaseUser()) +} + +func TestConfig_DatabasePassword(t *testing.T) { + c := NewConfig(CliTestContext()) + + assert.Equal(t, "", c.DatabasePassword()) +} + +func TestConfig_DatabaseDsn(t *testing.T) { + c := NewConfig(CliTestContext()) + + dsn := c.DatabaseDriver() + assert.Equal(t, SQLite, dsn) +} \ No newline at end of file diff --git a/internal/config/filenames.go b/internal/config/filenames.go index 8cd2d4059..9d028379d 100644 --- a/internal/config/filenames.go +++ b/internal/config/filenames.go @@ -340,3 +340,13 @@ func (c *Config) ExamplesPath() string { func (c *Config) TestdataPath() string { return filepath.Join(c.StoragePath(), "testdata") } + +// MysqldumpBin returns the mysqldump executable file name. +func (c *Config) MysqldumpBin() string { + return findExecutable("", "mysqldump") +} + +// SqliteBin returns the sqlite executable file name. +func (c *Config) SqliteBin() string { + return findExecutable("", "sqlite3") +} \ No newline at end of file diff --git a/internal/config/flags.go b/internal/config/flags.go index 1fc0ce959..8445c9bf9 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -103,9 +103,31 @@ var GlobalFlags = []cli.Flag{ }, cli.StringFlag{ Name: "database-dsn", - Usage: "data source or file name (`DSN`)", + Usage: "sqlite file name, `DSN` is optional for mysql", EnvVar: "PHOTOPRISM_DATABASE_DSN", }, + cli.StringFlag{ + Name: "database-server", + Usage: "database server `HOST`, port is optional", + EnvVar: "PHOTOPRISM_DATABASE_SERVER", + }, + cli.StringFlag{ + Name: "database-name", + Value: "photoprism", + Usage: "database `NAME`", + EnvVar: "PHOTOPRISM_DATABASE_NAME", + }, + cli.StringFlag{ + Name: "database-user", + Value: "photoprism", + Usage: "database user `NAME`", + EnvVar: "PHOTOPRISM_DATABASE_USER", + }, + cli.StringFlag{ + Name: "database-password", + Usage: "database user `PASSWORD``", + EnvVar: "PHOTOPRISM_DATABASE_PASSWORD", + }, cli.IntFlag{ Name: "database-conns", Usage: "max `NUMBER` of open connections to the database", diff --git a/internal/config/params.go b/internal/config/params.go index 2651ae0cd..017457448 100644 --- a/internal/config/params.go +++ b/internal/config/params.go @@ -60,6 +60,10 @@ type Params struct { CachePath string `yaml:"cache-path" flag:"cache-path"` DatabaseDriver string `yaml:"database-driver" flag:"database-driver"` DatabaseDsn string `yaml:"database-dsn" flag:"database-dsn"` + DatabaseServer string `yaml:"database-server" flag:"database-server"` + DatabaseName string `yaml:"database-name" flag:"database-name"` + DatabaseUser string `yaml:"database-user" flag:"database-user"` + DatabasePassword string `yaml:"database-password" flag:"database-password"` DatabaseConns int `yaml:"database-conns" flag:"database-conns"` DatabaseConnsIdle int `yaml:"database-conns-idle" flag:"database-conns-idle"` HttpServerHost string `yaml:"http-host" flag:"http-host"`