From 449fb7a2c1c303563cffb88896e365f3ba1c60c5 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Thu, 17 Dec 2020 18:24:55 +0100 Subject: [PATCH] Backup and restore albums from YAML files #567 --- Makefile | 4 +- internal/api/album.go | 81 +++++-- internal/api/photo.go | 18 +- internal/commands/backup.go | 145 +++++++----- internal/commands/config.go | 20 +- internal/commands/migrate.go | 2 +- internal/commands/restore.go | 208 ++++++++++-------- internal/commands/start.go | 8 + internal/config/config_test.go | 2 +- internal/config/flags.go | 30 +-- internal/config/{filenames.go => fs.go} | 47 ++-- .../config/{filenames_test.go => fs_test.go} | 0 internal/config/params.go | 16 +- internal/config/test.go | 6 +- internal/entity/album.go | 55 ++--- internal/entity/album_fixtures.go | 8 +- internal/entity/album_test.go | 4 +- internal/entity/album_yaml.go | 72 ++++++ internal/entity/album_yaml_test.go | 120 ++++++++++ internal/entity/photo_album.go | 18 +- internal/entity/photo_album_fixtures.go | 3 + internal/entity/photo_label.go | 2 + .../testdata/album/at9lxuqxpoaaaaaa.yml | 16 ++ internal/photoprism/albums.go | 86 ++++++++ internal/photoprism/moments.go | 6 + internal/query/albums.go | 6 + 26 files changed, 715 insertions(+), 268 deletions(-) rename internal/config/{filenames.go => fs.go} (90%) rename internal/config/{filenames_test.go => fs_test.go} (100%) create mode 100644 internal/entity/album_yaml.go create mode 100644 internal/entity/album_yaml_test.go create mode 100755 internal/entity/testdata/album/at9lxuqxpoaaaaaa.yml create mode 100644 internal/photoprism/albums.go diff --git a/Makefile b/Makefile index dc70b293c..4e7741257 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ upgrade: dep-upgrade-js dep-upgrade clean-local: clean-local-config clean-local-share clean-local-cache clean-install: clean-local dep build-js install-bin install-assets acceptance-start: - go run cmd/photoprism/photoprism.go --public --database-driver sqlite --database-dsn ./storage/acceptance/index.db --import-path ./storage/acceptance/import --http-port=2343 --settings-path ./storage/acceptance/settings --originals-path ./storage/acceptance/originals --sidecar-json=false --sidecar-yaml=false start -d + go run cmd/photoprism/photoprism.go --public --database-driver sqlite --database-dsn ./storage/acceptance/index.db --import-path ./storage/acceptance/import --http-port=2343 --config-path ./storage/acceptance/config --originals-path ./storage/acceptance/originals --sidecar-json=false --sidecar-yaml=false start -d acceptance-restart: go run cmd/photoprism/photoprism.go stop cp -f storage/acceptance/backup.db storage/acceptance/index.db @@ -40,7 +40,7 @@ acceptance-restart: rm -rf storage/acceptance/originals/2010 rm -rf storage/acceptance/originals/2013 rm -rf storage/acceptance/originals/2017 - go run cmd/photoprism/photoprism.go --public --database-driver sqlite --database-dsn ./storage/acceptance/index.db --import-path ./storage/acceptance/import --http-port=2343 --settings-path ./storage/acceptance/settings --originals-path ./storage/acceptance/originals --sidecar-json=false --sidecar-yaml=false start -d + go run cmd/photoprism/photoprism.go --public --database-driver sqlite --database-dsn ./storage/acceptance/index.db --import-path ./storage/acceptance/import --http-port=2343 --config-path ./storage/acceptance/config --originals-path ./storage/acceptance/originals --sidecar-json=false --sidecar-yaml=false start -d acceptance-restore-db: cp -f storage/acceptance/settings/settingsBackup.yml storage/acceptance/settings/settings.yml cp -f storage/acceptance/backup.db storage/acceptance/index.db diff --git a/internal/api/album.go b/internal/api/album.go index c14b2a6a8..a42129ca0 100644 --- a/internal/api/album.go +++ b/internal/api/album.go @@ -4,6 +4,7 @@ import ( "archive/zip" "fmt" "net/http" + "path/filepath" "strconv" "strings" "time" @@ -25,6 +26,24 @@ import ( "github.com/photoprism/photoprism/pkg/txt" ) +// SaveAlbumAsYaml saves album data as YAML file. +func SaveAlbumAsYaml(a entity.Album) { + c := service.Config() + + // Write YAML sidecar file (optional). + if !c.SidecarYaml() { + return + } + + fileName := a.YamlFileName(c.AlbumsPath()) + + if err := a.SaveAsYaml(fileName); err != nil { + log.Errorf("album: %s (update yaml)", err) + } else { + log.Debugf("album: updated yaml file %s", txt.Quote(filepath.Base(fileName))) + } +} + // ClearAlbumThumbCache removes all cached album covers e.g. after adding or removed photos. func ClearAlbumThumbCache(uid string) { cache := service.Cache() @@ -88,14 +107,14 @@ func GetAlbum(router *gin.RouterGroup) { } id := c.Param("uid") - m, err := query.AlbumByUID(id) + a, err := query.AlbumByUID(id) if err != nil { Abort(c, http.StatusNotFound, i18n.ErrAlbumNotFound) return } - c.JSON(http.StatusOK, m) + c.JSON(http.StatusOK, a) }) } @@ -116,13 +135,13 @@ func CreateAlbum(router *gin.RouterGroup) { return } - m := entity.NewAlbum(f.AlbumTitle, entity.AlbumDefault) - m.AlbumFavorite = f.AlbumFavorite + a := entity.NewAlbum(f.AlbumTitle, entity.AlbumDefault) + a.AlbumFavorite = f.AlbumFavorite - log.Debugf("album: creating %+v %+v", f, m) + log.Debugf("album: creating %+v %+v", f, a) - if res := entity.Db().Create(m); res.Error != nil { - AbortAlreadyExists(c, txt.Quote(m.AlbumTitle)) + if res := entity.Db().Create(a); res.Error != nil { + AbortAlreadyExists(c, txt.Quote(a.AlbumTitle)) return } @@ -130,9 +149,11 @@ func CreateAlbum(router *gin.RouterGroup) { UpdateClientConfig() - PublishAlbumEvent(EntityCreated, m.AlbumUID, c) + PublishAlbumEvent(EntityCreated, a.AlbumUID, c) - c.JSON(http.StatusOK, m) + SaveAlbumAsYaml(*a) + + c.JSON(http.StatusOK, a) }) } @@ -147,14 +168,14 @@ func UpdateAlbum(router *gin.RouterGroup) { } uid := c.Param("uid") - m, err := query.AlbumByUID(uid) + a, err := query.AlbumByUID(uid) if err != nil { Abort(c, http.StatusNotFound, i18n.ErrAlbumNotFound) return } - f, err := form.NewAlbum(m) + f, err := form.NewAlbum(a) if err != nil { log.Error(err) @@ -168,7 +189,7 @@ func UpdateAlbum(router *gin.RouterGroup) { return } - if err := m.SaveForm(f); err != nil { + if err := a.SaveForm(f); err != nil { log.Error(err) AbortSaveFailed(c) return @@ -180,7 +201,9 @@ func UpdateAlbum(router *gin.RouterGroup) { PublishAlbumEvent(EntityUpdated, uid, c) - c.JSON(http.StatusOK, m) + SaveAlbumAsYaml(a) + + c.JSON(http.StatusOK, a) }) } @@ -197,7 +220,7 @@ func DeleteAlbum(router *gin.RouterGroup) { conf := service.Config() id := c.Param("uid") - m, err := query.AlbumByUID(id) + a, err := query.AlbumByUID(id) if err != nil { Abort(c, http.StatusNotFound, i18n.ErrAlbumNotFound) @@ -206,13 +229,15 @@ func DeleteAlbum(router *gin.RouterGroup) { PublishAlbumEvent(EntityDeleted, id, c) - conf.Db().Delete(&m) + conf.Db().Delete(&a) UpdateClientConfig() - event.SuccessMsg(i18n.MsgAlbumDeleted, txt.Quote(m.AlbumTitle)) + SaveAlbumAsYaml(a) - c.JSON(http.StatusOK, m) + event.SuccessMsg(i18n.MsgAlbumDeleted, txt.Quote(a.AlbumTitle)) + + c.JSON(http.StatusOK, a) }) } @@ -230,21 +255,24 @@ func LikeAlbum(router *gin.RouterGroup) { } id := c.Param("uid") - album, err := query.AlbumByUID(id) + a, err := query.AlbumByUID(id) if err != nil { Abort(c, http.StatusNotFound, i18n.ErrAlbumNotFound) return } - if err := album.Update("AlbumFavorite", true); err != nil { + if err := a.Update("AlbumFavorite", true); err != nil { Abort(c, http.StatusInternalServerError, i18n.ErrSaveFailed) return } UpdateClientConfig() + PublishAlbumEvent(EntityUpdated, id, c) + SaveAlbumAsYaml(a) + c.JSON(http.StatusOK, i18n.NewResponse(http.StatusOK, i18n.MsgChangesSaved)) }) } @@ -263,21 +291,24 @@ func DislikeAlbum(router *gin.RouterGroup) { } id := c.Param("uid") - album, err := query.AlbumByUID(id) + a, err := query.AlbumByUID(id) if err != nil { Abort(c, http.StatusNotFound, i18n.ErrAlbumNotFound) return } - if err := album.Update("AlbumFavorite", false); err != nil { + if err := a.Update("AlbumFavorite", false); err != nil { Abort(c, http.StatusInternalServerError, i18n.ErrSaveFailed) return } UpdateClientConfig() + PublishAlbumEvent(EntityUpdated, id, c) + SaveAlbumAsYaml(a) + c.JSON(http.StatusOK, i18n.NewResponse(http.StatusOK, i18n.MsgChangesSaved)) }) } @@ -330,6 +361,8 @@ func CloneAlbums(router *gin.RouterGroup) { event.SuccessMsg(i18n.MsgSelectionAddedTo, txt.Quote(a.Title())) PublishAlbumEvent(EntityUpdated, a.AlbumUID, c) + + SaveAlbumAsYaml(a) } c.JSON(http.StatusOK, gin.H{"code": http.StatusOK, "message": i18n.Msg(i18n.MsgAlbumCloned), "album": a, "added": added}) @@ -379,7 +412,10 @@ func AddPhotosToAlbum(router *gin.RouterGroup) { } ClearAlbumThumbCache(a.AlbumUID) + PublishAlbumEvent(EntityUpdated, a.AlbumUID, c) + + SaveAlbumAsYaml(a) } c.JSON(http.StatusOK, gin.H{"code": http.StatusOK, "message": i18n.Msg(i18n.MsgChangesSaved), "album": a, "photos": photos.UIDs(), "added": added}) @@ -425,7 +461,10 @@ func RemovePhotosFromAlbum(router *gin.RouterGroup) { } ClearAlbumThumbCache(a.AlbumUID) + PublishAlbumEvent(EntityUpdated, a.AlbumUID, c) + + SaveAlbumAsYaml(a) } c.JSON(http.StatusOK, gin.H{"code": http.StatusOK, "message": i18n.Msg(i18n.MsgChangesSaved), "album": a, "photos": f.Photos, "removed": removed}) diff --git a/internal/api/photo.go b/internal/api/photo.go index d79e559fc..0522bbf8c 100644 --- a/internal/api/photo.go +++ b/internal/api/photo.go @@ -20,17 +20,19 @@ import ( // SavePhotoAsYaml saves photo data as YAML file. func SavePhotoAsYaml(p entity.Photo) { - conf := service.Config() + c := service.Config() // Write YAML sidecar file (optional). - if conf.SidecarYaml() { - yamlFile := p.YamlFileName(conf.OriginalsPath(), conf.SidecarPath()) + if !c.SidecarYaml() { + return + } - if err := p.SaveAsYaml(yamlFile); err != nil { - log.Errorf("photo: %s (update yaml)", err) - } else { - log.Debugf("photo: updated yaml file %s", txt.Quote(filepath.Base(yamlFile))) - } + fileName := p.YamlFileName(c.OriginalsPath(), c.SidecarPath()) + + if err := p.SaveAsYaml(fileName); err != nil { + log.Errorf("photo: %s (update yaml)", err) + } else { + log.Debugf("photo: updated yaml file %s", txt.Quote(filepath.Base(fileName))) } } diff --git a/internal/commands/backup.go b/internal/commands/backup.go index b18156376..4cabf43c2 100644 --- a/internal/commands/backup.go +++ b/internal/commands/backup.go @@ -11,6 +11,10 @@ import ( "path/filepath" "time" + "github.com/photoprism/photoprism/internal/service" + + "github.com/photoprism/photoprism/internal/photoprism" + "github.com/photoprism/photoprism/pkg/txt" "github.com/photoprism/photoprism/internal/config" @@ -20,7 +24,7 @@ import ( // BackupCommand configures the backup cli command. var BackupCommand = cli.Command{ Name: "backup", - Usage: "Creates an index database backup", + Usage: "Creates album and index backups", Flags: backupFlags, Action: backupAction, } @@ -30,10 +34,26 @@ var backupFlags = []cli.Flag{ Name: "force, f", Usage: "overwrite existing backup files", }, + cli.BoolFlag{ + Name: "albums, a", + Usage: "create album yaml file backups", + }, + cli.BoolFlag{ + Name: "index, i", + Usage: "create index database backup", + }, } // backupAction creates a database backup. func backupAction(ctx *cli.Context) error { + if !ctx.Bool("index") && !ctx.Bool("albums") { + for _, flag := range backupFlags { + fmt.Println(flag.String()) + } + + return nil + } + start := time.Now() conf := config.NewConfig(ctx) @@ -45,74 +65,87 @@ func backupAction(ctx *cli.Context) error { return err } - // Use command argument as backup file name. - fileName := ctx.Args().First() + if ctx.Bool("index") { + // Use command argument as backup file name. + fileName := ctx.Args().First() - // If empty, use default backup file name. - if fileName == "" { - backupFile := time.Now().UTC().Format("2006-01-02") + ".sql" - backupPath := filepath.Join(conf.BackupPath(), conf.DatabaseDriver()) - fileName = filepath.Join(backupPath, backupFile) - } + // If empty, use default backup file name. + if fileName == "" { + backupFile := time.Now().UTC().Format("2006-01-02") + ".sql" + backupPath := filepath.Join(conf.BackupPath(), conf.DatabaseDriver()) + fileName = filepath.Join(backupPath, backupFile) + } - if _, err := os.Stat(fileName); err == nil && !ctx.Bool("force") { - return fmt.Errorf("backup file already exists: %s", fileName) - } else if err == nil { - log.Warnf("replacing existing backup file") - } + if _, err := os.Stat(fileName); err == nil && !ctx.Bool("force") { + return fmt.Errorf("backup file already exists: %s", fileName) + } else if err == nil { + log.Warnf("replacing existing backup file") + } - // Create backup directory if not exists. - if dir := filepath.Dir(fileName); dir != "." { - if err := os.MkdirAll(dir, os.ModePerm); err != nil { + // Create backup directory if not exists. + if dir := filepath.Dir(fileName); dir != "." { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + } + + log.Infof("backing up database to %s", txt.Quote(fileName)) + + var cmd *exec.Cmd + + switch conf.DatabaseDriver() { + case config.MySQL, config.MariaDB: + cmd = exec.Command( + conf.MysqldumpBin(), + "-h", conf.DatabaseHost(), + "-P", conf.DatabasePortString(), + "-u", conf.DatabaseUser(), + "-p"+conf.DatabasePassword(), + conf.DatabaseName(), + ) + case config.SQLite: + cmd = exec.Command( + conf.SqliteBin(), + conf.DatabaseDsn(), + ".dump", + ) + default: + return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver()) + } + + // Fetch command output. + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + // Run backup command. + if err := cmd.Run(); err != nil { + if stderr.String() != "" { + return errors.New(stderr.String()) + } + } + + // Write output to file. + if err := ioutil.WriteFile(fileName, []byte(out.String()), os.ModePerm); err != nil { return err } } - log.Infof("backing up database to %s", txt.Quote(fileName)) + if ctx.Bool("albums") { + service.SetConfig(conf) + conf.InitDb() - var cmd *exec.Cmd - - switch conf.DatabaseDriver() { - case config.MySQL, config.MariaDB: - cmd = exec.Command( - conf.MysqldumpBin(), - "-h", conf.DatabaseHost(), - "-P", conf.DatabasePortString(), - "-u", conf.DatabaseUser(), - "-p"+conf.DatabasePassword(), - conf.DatabaseName(), - ) - case config.SQLite: - cmd = exec.Command( - conf.SqliteBin(), - conf.DatabaseDsn(), - ".dump", - ) - default: - return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver()) - } - - // Fetch command output. - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - // Run backup command. - if err := cmd.Run(); err != nil { - if stderr.String() != "" { - return errors.New(stderr.String()) + if count, err := photoprism.BackupAlbums(true); err != nil { + return err + } else { + log.Infof("%d albums saved as yaml files", count) } } - // Write output to file. - if err := ioutil.WriteFile(fileName, []byte(out.String()), os.ModePerm); err != nil { - return err - } - elapsed := time.Since(start) - log.Infof("database backup completed in %s", elapsed) + log.Infof("backup completed in %s", elapsed) conf.Shutdown() diff --git a/internal/commands/config.go b/internal/commands/config.go index 087c5d646..d9c9847d7 100644 --- a/internal/commands/config.go +++ b/internal/commands/config.go @@ -30,6 +30,14 @@ func configAction(ctx *cli.Context) error { fmt.Printf("%-25s %t\n", "read-only", conf.ReadOnly()) fmt.Printf("%-25s %t\n", "experimental", conf.Experimental()) + // Config path and main file. + fmt.Printf("%-25s %s\n", "config-path", conf.ConfigPath()) + fmt.Printf("%-25s %s\n", "config-file", conf.ConfigFile()) + fmt.Printf("%-25s %s\n", "settings-file", conf.SettingsFile()) + + // Passwords. + fmt.Printf("%-25s %s\n", "admin-password", conf.AdminPassword()) + // Site information. fmt.Printf("%-25s %s\n", "site-url", conf.SiteUrl()) fmt.Printf("%-25s %s\n", "site-preview", conf.SitePreview()) @@ -39,15 +47,12 @@ func configAction(ctx *cli.Context) error { fmt.Printf("%-25s %s\n", "site-author", conf.SiteAuthor()) // Everything related to TensorFlow. - fmt.Printf("%-25s %t\n", "tf-off", conf.TensorFlowOff()) - fmt.Printf("%-25s %s\n", "tf-version", conf.TensorFlowVersion()) - fmt.Printf("%-25s %s\n", "tf-model-path", conf.TensorFlowModelPath()) + fmt.Printf("%-25s %t\n", "tensorflow-off", conf.TensorFlowOff()) + fmt.Printf("%-25s %s\n", "tensorflow-version", conf.TensorFlowVersion()) + fmt.Printf("%-25s %s\n", "tensorflow-model-path", conf.TensorFlowModelPath()) fmt.Printf("%-25s %t\n", "detect-nsfw", conf.DetectNSFW()) fmt.Printf("%-25s %t\n", "upload-nsfw", conf.UploadNSFW()) - // Passwords. - fmt.Printf("%-25s %s\n", "admin-password", conf.AdminPassword()) - // Background workers and logging. fmt.Printf("%-25s %d\n", "workers", conf.Workers()) fmt.Printf("%-25s %d\n", "wakeup-interval", conf.WakeupInterval()/time.Second) @@ -76,6 +81,7 @@ func configAction(ctx *cli.Context) error { fmt.Printf("%-25s %s\n", "assets-path", conf.AssetsPath()) fmt.Printf("%-25s %s\n", "storage-path", conf.StoragePath()) fmt.Printf("%-25s %s\n", "backup-path", conf.BackupPath()) + fmt.Printf("%-25s %s\n", "albums-path", conf.AlbumsPath()) fmt.Printf("%-25s %s\n", "import-path", conf.ImportPath()) fmt.Printf("%-25s %s\n", "originals-path", conf.OriginalsPath()) fmt.Printf("%-25s %d\n", "originals-limit", conf.OriginalsLimit()) @@ -87,8 +93,6 @@ func configAction(ctx *cli.Context) error { fmt.Printf("%-25s %s\n", "templates-path", conf.TemplatesPath()) fmt.Printf("%-25s %s\n", "cache-path", conf.CachePath()) fmt.Printf("%-25s %s\n", "temp-path", conf.TempPath()) - fmt.Printf("%-25s %s\n", "config-file", conf.ConfigFile()) - fmt.Printf("%-25s %s\n", "settings-path", conf.SettingsPath()) fmt.Printf("%-25s %t\n", "settings-hidden", conf.SettingsHidden()) // External binaries and sidecar configuration. diff --git a/internal/commands/migrate.go b/internal/commands/migrate.go index 60382c448..9f6ad41a4 100644 --- a/internal/commands/migrate.go +++ b/internal/commands/migrate.go @@ -11,7 +11,7 @@ import ( // MigrateCommand is used to register the migrate cli command var MigrateCommand = cli.Command{ Name: "migrate", - Usage: "Initializes and migrates the index database if needed", + Usage: "Initializes the index database if needed", Action: migrateAction, } diff --git a/internal/commands/restore.go b/internal/commands/restore.go index 0dbe31285..9df136192 100644 --- a/internal/commands/restore.go +++ b/internal/commands/restore.go @@ -11,6 +11,10 @@ import ( "regexp" "time" + "github.com/photoprism/photoprism/internal/service" + + "github.com/photoprism/photoprism/internal/photoprism" + "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/pkg/fs" "github.com/photoprism/photoprism/pkg/txt" @@ -22,7 +26,7 @@ import ( // RestoreCommand configures the backup cli command. var RestoreCommand = cli.Command{ Name: "restore", - Usage: "Restores the index from a backup", + Usage: "Restores album and index backups", Flags: restoreFlags, Action: restoreAction, } @@ -32,10 +36,26 @@ var restoreFlags = []cli.Flag{ Name: "force, f", Usage: "overwrite existing index", }, + cli.BoolFlag{ + Name: "albums, a", + Usage: "restore album yaml file backups", + }, + cli.BoolFlag{ + Name: "index, i", + Usage: "restore index database backup", + }, } // restoreAction restores a database backup. func restoreAction(ctx *cli.Context) error { + if !ctx.Bool("index") && !ctx.Bool("albums") { + for _, flag := range restoreFlags { + fmt.Println(flag.String()) + } + + return nil + } + start := time.Now() conf := config.NewConfig(ctx) @@ -47,105 +67,107 @@ func restoreAction(ctx *cli.Context) error { return err } - // Use command argument as backup file name. - fileName := ctx.Args().First() + if ctx.Bool("index") { + // Use command argument as backup file name. + fileName := ctx.Args().First() - // If empty, use default backup file name. - if fileName == "" { - backupPath := filepath.Join(conf.BackupPath(), conf.DatabaseDriver()) + // If empty, use default backup file name. + if fileName == "" { + backupPath := filepath.Join(conf.BackupPath(), conf.DatabaseDriver()) - matches, err := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), "*.sql")) + matches, err := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), "*.sql")) + + if err != nil { + return err + } + + if len(matches) == 0 { + log.Errorf("no backup files found in %s", backupPath) + return nil + } + + fileName = matches[len(matches)-1] + } + + if !fs.FileExists(fileName) { + log.Errorf("backup file not found: %s", fileName) + return nil + } + + counts := struct{ Photos int }{} + + conf.Db().Unscoped().Table("photos"). + Select("COUNT(*) AS photos"). + Take(&counts) + + if counts.Photos == 0 { + // Do nothing; + } else if !ctx.Bool("force") { + return fmt.Errorf("use --force to replace exisisting index with %d photos", counts.Photos) + } else { + log.Warnf("replacing existing index with %d photos", counts.Photos) + } + + log.Infof("restoring index from %s", txt.Quote(fileName)) + + sqlBackup, err := ioutil.ReadFile(fileName) if err != nil { return err } - if len(matches) == 0 { - log.Errorf("no backup files found in %s", backupPath) - return nil + entity.SetDbProvider(conf) + tables := entity.Entities + + var cmd *exec.Cmd + + switch conf.DatabaseDriver() { + case config.MySQL, config.MariaDB: + cmd = exec.Command( + conf.MysqlBin(), + "-h", conf.DatabaseHost(), + "-P", conf.DatabasePortString(), + "-u", conf.DatabaseUser(), + "-p"+conf.DatabasePassword(), + "-f", + conf.DatabaseName(), + ) + case config.SQLite: + log.Infoln("dropping existing tables") + tables.Drop() + cmd = exec.Command( + conf.SqliteBin(), + conf.DatabaseDsn(), + ) + default: + return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver()) } - fileName = matches[len(matches)-1] - } + // Fetch command output. + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr - if !fs.FileExists(fileName) { - log.Errorf("backup file not found: %s", fileName) - return nil - } + stdin, err := cmd.StdinPipe() - counts := struct{ Photos int }{} - - conf.Db().Unscoped().Table("photos"). - Select("COUNT(*) AS photos"). - Take(&counts) - - if counts.Photos == 0 { - // Do nothing; - } else if !ctx.Bool("force") { - return fmt.Errorf("use --force to replace exisisting index with %d photos", counts.Photos) - } else { - log.Warnf("replacing existing index with %d photos", counts.Photos) - } - - log.Infof("restoring index from %s", txt.Quote(fileName)) - - sqlBackup, err := ioutil.ReadFile(fileName) - - if err != nil { - return err - } - - entity.SetDbProvider(conf) - tables := entity.Entities - - var cmd *exec.Cmd - - switch conf.DatabaseDriver() { - case config.MySQL, config.MariaDB: - cmd = exec.Command( - conf.MysqlBin(), - "-h", conf.DatabaseHost(), - "-P", conf.DatabasePortString(), - "-u", conf.DatabaseUser(), - "-p"+conf.DatabasePassword(), - "-f", - conf.DatabaseName(), - ) - case config.SQLite: - log.Infoln("dropping existing tables") - tables.Drop() - cmd = exec.Command( - conf.SqliteBin(), - conf.DatabaseDsn(), - ) - default: - return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver()) - } - - // Fetch command output. - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - stdin, err := cmd.StdinPipe() - - if err != nil { - log.Fatal(err) - } - - go func() { - defer stdin.Close() - if _, err := io.WriteString(stdin, string(sqlBackup)); err != nil { - log.Errorf(err.Error()) + if err != nil { + log.Fatal(err) } - }() - // Run backup command. - if err := cmd.Run(); err != nil { - if stderr.String() != "" { - log.Debugln(stderr.String()) - log.Warnf("index could not be restored completely") + go func() { + defer stdin.Close() + if _, err := io.WriteString(stdin, string(sqlBackup)); err != nil { + log.Errorf(err.Error()) + } + }() + + // Run backup command. + if err := cmd.Run(); err != nil { + if stderr.String() != "" { + log.Debugln(stderr.String()) + log.Warnf("index could not be restored completely") + } } } @@ -153,9 +175,19 @@ func restoreAction(ctx *cli.Context) error { conf.InitDb() + if ctx.Bool("albums") { + service.SetConfig(conf) + + if count, err := photoprism.RestoreAlbums(true); err != nil { + return err + } else { + log.Infof("%d albums restored from yaml files", count) + } + } + elapsed := time.Since(start) - log.Infof("database restored in %s", elapsed) + log.Infof("backup restored in %s", elapsed) conf.Shutdown() diff --git a/internal/commands/start.go b/internal/commands/start.go index 034136dea..4d7f18a01 100644 --- a/internal/commands/start.go +++ b/internal/commands/start.go @@ -9,6 +9,8 @@ import ( "syscall" "time" + "github.com/photoprism/photoprism/internal/photoprism" + "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/server" "github.com/photoprism/photoprism/internal/service" @@ -107,6 +109,12 @@ func startAction(ctx *cli.Context) error { // start web server go server.Start(cctx, conf) + if count, err := photoprism.RestoreAlbums(false); err != nil { + log.Errorf("restore: %s", err) + } else if count > 0 { + log.Infof("%d albums restored", count) + } + // start share & sync workers workers.Start(conf) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a36454072..a0c8b0c28 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -83,7 +83,7 @@ func TestConfig_ConfigFile(t *testing.T) { func TestConfig_SettingsPath(t *testing.T) { c := NewConfig(CliTestContext()) - assert.Contains(t, c.SettingsPath(), "/storage/testdata/settings") + assert.Contains(t, c.ConfigPath(), "/storage/testdata/settings") } func TestConfig_BackupPath(t *testing.T) { diff --git a/internal/config/flags.go b/internal/config/flags.go index 8445c9bf9..52ff295c4 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -21,21 +21,31 @@ var GlobalFlags = []cli.Flag{ Usage: "don't modify originals directory (import and upload disabled)", EnvVar: "PHOTOPRISM_READONLY", }, - cli.BoolFlag{ - Name: "tf-off", - Usage: "don't use TensorFlow for image classification (or anything else)", - EnvVar: "PHOTOPRISM_TENSORFLOW_OFF", - }, cli.BoolFlag{ Name: "experimental, e", Usage: "enable experimental features", EnvVar: "PHOTOPRISM_EXPERIMENTAL", }, + cli.StringFlag{ + Name: "config-path", + Usage: "config `PATH`", + EnvVar: "PHOTOPRISM_CONFIG_PATH", + }, + cli.StringFlag{ + Name: "config-file, c", + Usage: "main config `FILENAME`", + EnvVar: "PHOTOPRISM_CONFIG_FILE", + }, cli.StringFlag{ Name: "admin-password", Usage: "initial admin password", EnvVar: "PHOTOPRISM_ADMIN_PASSWORD", }, + cli.BoolFlag{ + Name: "tensorflow-off", + Usage: "don't use TensorFlow for image classification (or anything else)", + EnvVar: "PHOTOPRISM_TENSORFLOW_OFF", + }, cli.IntFlag{ Name: "workers, w", Usage: "number of workers for indexing", @@ -196,16 +206,6 @@ var GlobalFlags = []cli.Flag{ Usage: "temporary `PATH` for uploads and downloads", EnvVar: "PHOTOPRISM_TEMP_PATH", }, - cli.StringFlag{ - Name: "config-file, c", - Usage: "load configuration from `FILENAME`", - EnvVar: "PHOTOPRISM_CONFIG_FILE", - }, - cli.StringFlag{ - Name: "settings-path", - Usage: "settings `PATH`", - EnvVar: "PHOTOPRISM_SETTINGS_PATH", - }, cli.BoolFlag{ Name: "settings-hidden", Usage: "users can not view or change settings", diff --git a/internal/config/filenames.go b/internal/config/fs.go similarity index 90% rename from internal/config/filenames.go rename to internal/config/fs.go index a923b12c0..fb02a7f23 100644 --- a/internal/config/filenames.go +++ b/internal/config/fs.go @@ -95,10 +95,10 @@ func (c *Config) CreateDirectories() error { return createError(c.ThumbPath(), err) } - if c.SettingsPath() == "" { - return notFoundError("settings") - } else if err := os.MkdirAll(c.SettingsPath(), os.ModePerm); err != nil { - return createError(c.SettingsPath(), err) + if c.ConfigPath() == "" { + return notFoundError("config") + } else if err := os.MkdirAll(c.ConfigPath(), os.ModePerm); err != nil { + return createError(c.ConfigPath(), err) } if c.TempPath() == "" { @@ -107,6 +107,12 @@ func (c *Config) CreateDirectories() error { return createError(c.TempPath(), err) } + if c.AlbumsPath() == "" { + return notFoundError("albums") + } else if err := os.MkdirAll(c.AlbumsPath(), os.ModePerm); err != nil { + return createError(c.AlbumsPath(), err) + } + if c.TensorFlowModelPath() == "" { return notFoundError("tensorflow model") } else if err := os.MkdirAll(c.TensorFlowModelPath(), os.ModePerm); err != nil { @@ -137,29 +143,33 @@ func (c *Config) CreateDirectories() error { // ConfigFile returns the config file name. func (c *Config) ConfigFile() string { if c.params.ConfigFile == "" || !fs.FileExists(c.params.ConfigFile) { - return filepath.Join(c.SettingsPath(), "photoprism.yml") + return filepath.Join(c.ConfigPath(), "photoprism.yml") } return c.params.ConfigFile } +// ConfigPath returns the config path. +func (c *Config) ConfigPath() string { + if c.params.ConfigPath == "" { + if fs.PathExists(filepath.Join(c.StoragePath(), "settings")) { + return filepath.Join(c.StoragePath(), "settings") + } + + return filepath.Join(c.StoragePath(), "config") + } + + return fs.Abs(c.params.ConfigPath) +} + // HubConfigFile returns the backend api config file name. func (c *Config) HubConfigFile() string { - return filepath.Join(c.SettingsPath(), "hub.yml") + return filepath.Join(c.ConfigPath(), "hub.yml") } // SettingsFile returns the user settings file name. func (c *Config) SettingsFile() string { - return filepath.Join(c.SettingsPath(), "settings.yml") -} - -// SettingsPath returns the config path. -func (c *Config) SettingsPath() string { - if c.params.SettingsPath == "" { - return filepath.Join(c.StoragePath(), "settings") - } - - return fs.Abs(c.params.SettingsPath) + return filepath.Join(c.ConfigPath(), "settings.yml") } // PIDFilename returns the filename for storing the server process id (pid). @@ -355,3 +365,8 @@ func (c *Config) MysqldumpBin() string { func (c *Config) SqliteBin() string { return findExecutable("", "sqlite3") } + +// AlbumsPath returns the storage path for album YAML files. +func (c *Config) AlbumsPath() string { + return filepath.Join(c.StoragePath(), "albums") +} diff --git a/internal/config/filenames_test.go b/internal/config/fs_test.go similarity index 100% rename from internal/config/filenames_test.go rename to internal/config/fs_test.go diff --git a/internal/config/params.go b/internal/config/params.go index 063c2b509..91e23035e 100644 --- a/internal/config/params.go +++ b/internal/config/params.go @@ -33,21 +33,22 @@ type Params struct { Name string Version string Copyright string + Debug bool `yaml:"debug" flag:"debug"` + Public bool `yaml:"public" flag:"public"` + ReadOnly bool `yaml:"read-only" flag:"read-only"` + Experimental bool `yaml:"experimental" flag:"experimental"` + ConfigPath string `yaml:"config-path" flag:"config-path"` ConfigFile string + AdminPassword string `yaml:"admin-password" flag:"admin-password"` SiteUrl string `yaml:"site-url" flag:"site-url"` SitePreview string `yaml:"site-preview" flag:"site-preview"` SiteTitle string `yaml:"site-title" flag:"site-title"` SiteCaption string `yaml:"site-caption" flag:"site-caption"` SiteDescription string `yaml:"site-description" flag:"site-description"` SiteAuthor string `yaml:"site-author" flag:"site-author"` - Public bool `yaml:"public" flag:"public"` - Debug bool `yaml:"debug" flag:"debug"` - ReadOnly bool `yaml:"read-only" flag:"read-only"` - Experimental bool `yaml:"experimental" flag:"experimental"` - TensorFlowOff bool `yaml:"tf-off" flag:"tf-off"` + TensorFlowOff bool `yaml:"tensorflow-off" flag:"tensorflow-off"` Workers int `yaml:"workers" flag:"workers"` WakeupInterval int `yaml:"wakeup-interval" flag:"wakeup-interval"` - AdminPassword string `yaml:"admin-password" flag:"admin-password"` LogLevel string `yaml:"log-level" flag:"log-level"` AssetsPath string `yaml:"assets-path" flag:"assets-path"` StoragePath string `yaml:"storage-path" flag:"storage-path"` @@ -55,7 +56,6 @@ type Params struct { ImportPath string `yaml:"import-path" flag:"import-path"` OriginalsPath string `yaml:"originals-path" flag:"originals-path"` OriginalsLimit int64 `yaml:"originals-limit" flag:"originals-limit"` - SettingsPath string `yaml:"settings-path" flag:"settings-path"` SettingsHidden bool `yaml:"settings-hidden" flag:"settings-hidden"` TempPath string `yaml:"temp-path" flag:"temp-path"` CachePath string `yaml:"cache-path" flag:"cache-path"` @@ -124,7 +124,7 @@ func NewParams(ctx *cli.Context) *Params { // expandFilenames converts path in config to absolute path func (c *Params) expandFilenames() { - c.SettingsPath = fs.Abs(c.SettingsPath) + c.ConfigPath = fs.Abs(c.ConfigPath) c.StoragePath = fs.Abs(c.StoragePath) c.BackupPath = fs.Abs(c.BackupPath) c.AssetsPath = fs.Abs(c.AssetsPath) diff --git a/internal/config/test.go b/internal/config/test.go index 917d36b88..de4fb64ad 100644 --- a/internal/config/test.go +++ b/internal/config/test.go @@ -65,7 +65,7 @@ func NewTestParams() *Params { OriginalsPath: testDataPath + "/originals", ImportPath: testDataPath + "/import", TempPath: testDataPath + "/temp", - SettingsPath: testDataPath + "/settings", + ConfigPath: testDataPath + "/settings", SidecarPath: testDataPath + "/sidecar", DatabaseDriver: dbDriver, DatabaseDsn: dbDsn, @@ -120,11 +120,11 @@ func NewTestConfig() *Config { s := NewSettings() - if err := os.MkdirAll(c.SettingsPath(), os.ModePerm); err != nil { + if err := os.MkdirAll(c.ConfigPath(), os.ModePerm); err != nil { log.Fatalf("config: %s", err.Error()) } - if err := s.Save(filepath.Join(c.SettingsPath(), "settings.yml")); err != nil { + if err := s.Save(filepath.Join(c.ConfigPath(), "settings.yml")); err != nil { log.Fatalf("config: %s", err.Error()) } diff --git a/internal/entity/album.go b/internal/entity/album.go index 4de9323e6..9aa6b7b97 100644 --- a/internal/entity/album.go +++ b/internal/entity/album.go @@ -27,31 +27,32 @@ type Albums []Album // Album represents a photo album type Album struct { - ID uint `gorm:"primary_key" json:"ID" yaml:"-"` - AlbumUID string `gorm:"type:VARBINARY(42);unique_index;" json:"UID" yaml:"UID"` - CoverUID string `gorm:"type:VARBINARY(42);" json:"CoverUID" yaml:"CoverUID,omitempty"` - FolderUID string `gorm:"type:VARBINARY(42);index;" json:"FolderUID" yaml:"FolderUID,omitempty"` - AlbumSlug string `gorm:"type:VARBINARY(255);index;" json:"Slug" yaml:"Slug"` - AlbumPath string `gorm:"type:VARBINARY(500);index;" json:"Path" yaml:"-"` - AlbumType string `gorm:"type:VARBINARY(8);default:'album';" json:"Type" yaml:"Type,omitempty"` - AlbumTitle string `gorm:"type:VARCHAR(255);" json:"Title" yaml:"Title"` - AlbumLocation string `gorm:"type:VARCHAR(255);" json:"Location" yaml:"Location,omitempty"` - AlbumCategory string `gorm:"type:VARCHAR(255);index;" json:"Category" yaml:"Category,omitempty"` - AlbumCaption string `gorm:"type:TEXT;" json:"Caption" yaml:"Caption,omitempty"` - AlbumDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"` - AlbumNotes string `gorm:"type:TEXT;" json:"Notes" yaml:"Notes,omitempty"` - AlbumFilter string `gorm:"type:VARBINARY(1024);" json:"Filter" yaml:"Filter,omitempty"` - AlbumOrder string `gorm:"type:VARBINARY(32);" json:"Order" yaml:"Order,omitempty"` - AlbumTemplate string `gorm:"type:VARBINARY(255);" json:"Template" yaml:"Template,omitempty"` - AlbumCountry string `gorm:"type:VARBINARY(2);index:idx_albums_country_year_month;default:'zz'" json:"Country" yaml:"Country,omitempty"` - AlbumYear int `gorm:"index:idx_albums_country_year_month;" json:"Year" yaml:"Year,omitempty"` - AlbumMonth int `gorm:"index:idx_albums_country_year_month;" json:"Month" yaml:"Month,omitempty"` - AlbumDay int `json:"Day" yaml:"Day,omitempty"` - AlbumFavorite bool `json:"Favorite" yaml:"Favorite,omitempty"` - AlbumPrivate bool `json:"Private" yaml:"Private,omitempty"` - CreatedAt time.Time `json:"CreatedAt" yaml:"-"` - UpdatedAt time.Time `json:"UpdatedAt" yaml:"-"` - DeletedAt *time.Time `sql:"index" json:"-" yaml:"-"` + ID uint `gorm:"primary_key" json:"ID" yaml:"-"` + AlbumUID string `gorm:"type:VARBINARY(42);unique_index;" json:"UID" yaml:"UID"` + CoverUID string `gorm:"type:VARBINARY(42);" json:"CoverUID" yaml:"CoverUID,omitempty"` + FolderUID string `gorm:"type:VARBINARY(42);index;" json:"FolderUID" yaml:"FolderUID,omitempty"` + AlbumSlug string `gorm:"type:VARBINARY(255);index;" json:"Slug" yaml:"Slug"` + AlbumPath string `gorm:"type:VARBINARY(500);index;" json:"Path" yaml:"-"` + AlbumType string `gorm:"type:VARBINARY(8);default:'album';" json:"Type" yaml:"Type,omitempty"` + AlbumTitle string `gorm:"type:VARCHAR(255);" json:"Title" yaml:"Title"` + AlbumLocation string `gorm:"type:VARCHAR(255);" json:"Location" yaml:"Location,omitempty"` + AlbumCategory string `gorm:"type:VARCHAR(255);index;" json:"Category" yaml:"Category,omitempty"` + AlbumCaption string `gorm:"type:TEXT;" json:"Caption" yaml:"Caption,omitempty"` + AlbumDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"` + AlbumNotes string `gorm:"type:TEXT;" json:"Notes" yaml:"Notes,omitempty"` + AlbumFilter string `gorm:"type:VARBINARY(1024);" json:"Filter" yaml:"Filter,omitempty"` + AlbumOrder string `gorm:"type:VARBINARY(32);" json:"Order" yaml:"Order,omitempty"` + AlbumTemplate string `gorm:"type:VARBINARY(255);" json:"Template" yaml:"Template,omitempty"` + AlbumCountry string `gorm:"type:VARBINARY(2);index:idx_albums_country_year_month;default:'zz'" json:"Country" yaml:"Country,omitempty"` + AlbumYear int `gorm:"index:idx_albums_country_year_month;" json:"Year" yaml:"Year,omitempty"` + AlbumMonth int `gorm:"index:idx_albums_country_year_month;" json:"Month" yaml:"Month,omitempty"` + AlbumDay int `json:"Day" yaml:"Day,omitempty"` + AlbumFavorite bool `json:"Favorite" yaml:"Favorite,omitempty"` + AlbumPrivate bool `json:"Private" yaml:"Private,omitempty"` + CreatedAt time.Time `json:"CreatedAt" yaml:"CreatedAt,omitempty"` + UpdatedAt time.Time `json:"UpdatedAt" yaml:"UpdatedAt,omitempty"` + DeletedAt *time.Time `sql:"index" json:"DeletedAt" yaml:"DeletedAt,omitempty"` + Photos PhotoAlbums `gorm:"foreignkey:AlbumUID;association_foreignkey:AlbumUID" json:"-" yaml:"Photos,omitempty"` } // AddPhotoToAlbums adds a photo UID to multiple albums and automatically creates them if needed. @@ -382,7 +383,7 @@ func (m *Album) Title() string { } // AddPhotos adds photos to an existing album. -func (m *Album) AddPhotos(UIDs []string) (added []PhotoAlbum) { +func (m *Album) AddPhotos(UIDs []string) (added PhotoAlbums) { for _, uid := range UIDs { entry := PhotoAlbum{AlbumUID: m.AlbumUID, PhotoUID: uid, Hidden: false} @@ -397,7 +398,7 @@ func (m *Album) AddPhotos(UIDs []string) (added []PhotoAlbum) { } // RemovePhotos removes photos from an album. -func (m *Album) RemovePhotos(UIDs []string) (removed []PhotoAlbum) { +func (m *Album) RemovePhotos(UIDs []string) (removed PhotoAlbums) { for _, uid := range UIDs { entry := PhotoAlbum{AlbumUID: m.AlbumUID, PhotoUID: uid, Hidden: true} diff --git a/internal/entity/album_fixtures.go b/internal/entity/album_fixtures.go index bf36d2222..631f8481a 100644 --- a/internal/entity/album_fixtures.go +++ b/internal/entity/album_fixtures.go @@ -30,7 +30,7 @@ var AlbumFixtures = AlbumMap{ AlbumSlug: "christmas2030", AlbumType: AlbumDefault, AlbumTitle: "Christmas2030", - AlbumDescription: "Wonderful christmas", + AlbumDescription: "Wonderful Christmas", AlbumNotes: "", AlbumOrder: "oldest", AlbumTemplate: "", @@ -46,7 +46,7 @@ var AlbumFixtures = AlbumMap{ AlbumSlug: "holiday-2030", AlbumType: AlbumDefault, AlbumTitle: "Holiday2030", - AlbumDescription: "Wonderful christmas", + AlbumDescription: "Wonderful Christmas", AlbumNotes: "", AlbumOrder: "newest", AlbumTemplate: "", @@ -61,8 +61,8 @@ var AlbumFixtures = AlbumMap{ AlbumUID: "at9lxuqxpogaaba9", AlbumSlug: "berlin-2019", AlbumType: AlbumDefault, - AlbumTitle: "Berlin2019", - AlbumDescription: "Wonderful christmas", + AlbumTitle: "Berlin 2019", + AlbumDescription: "We love Berlin!", AlbumNotes: "", AlbumOrder: "oldest", AlbumTemplate: "", diff --git a/internal/entity/album_test.go b/internal/entity/album_test.go index 68eeedc6a..7ec7be034 100644 --- a/internal/entity/album_test.go +++ b/internal/entity/album_test.go @@ -111,7 +111,7 @@ func TestAddPhotoToAlbums(t *testing.T) { t.Fatal(err) } - var entries []PhotoAlbum + var entries PhotoAlbums if err := Db().Where("album_uid = ? AND photo_uid = ?", "at6axuzitogaaiax", "pt9jtxrexxvl0yh0").Find(&entries).Error; err != nil { t.Fatal(err) @@ -149,7 +149,7 @@ func TestAddPhotoToAlbums(t *testing.T) { t.Fatal(err) } - var entries []PhotoAlbum + var entries PhotoAlbums if err := Db().Where("album_uid = ? AND photo_uid = ?", "at6axuzitogaaiax", "pt9jtxrexxvl0yh0").Find(&entries).Error; err != nil { t.Fatal(err) diff --git a/internal/entity/album_yaml.go b/internal/entity/album_yaml.go new file mode 100644 index 000000000..346274b67 --- /dev/null +++ b/internal/entity/album_yaml.go @@ -0,0 +1,72 @@ +package entity + +import ( + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/photoprism/photoprism/pkg/fs" + "gopkg.in/yaml.v2" +) + +var albumYamlMutex = sync.Mutex{} + +// Yaml returns album data as YAML string. +func (m *Album) Yaml() ([]byte, error) { + if err := Db().Model(m).Association("Photos").Find(&m.Photos).Error; err != nil { + log.Errorf("album: %s (yaml)", err) + } + + out, err := yaml.Marshal(m) + + if err != nil { + return []byte{}, err + } + + return out, err +} + +// SaveAsYaml saves album data as YAML file. +func (m *Album) SaveAsYaml(fileName string) error { + data, err := m.Yaml() + + if err != nil { + return err + } + + // Make sure directory exists. + if err := os.MkdirAll(filepath.Dir(fileName), os.ModePerm); err != nil { + return err + } + + albumYamlMutex.Lock() + defer albumYamlMutex.Unlock() + + // Write YAML data to file. + if err := ioutil.WriteFile(fileName, data, os.ModePerm); err != nil { + return err + } + + return nil +} + +// LoadFromYaml photo data from a YAML file. +func (m *Album) LoadFromYaml(fileName string) error { + data, err := ioutil.ReadFile(fileName) + + if err != nil { + return err + } + + if err := yaml.Unmarshal(data, m); err != nil { + return err + } + + return nil +} + +// YamlFileName returns the YAML backup file name. +func (m *Album) YamlFileName(albumsPath string) string { + return filepath.Join(albumsPath, m.AlbumType, m.AlbumUID+fs.YamlExt) +} diff --git a/internal/entity/album_yaml_test.go b/internal/entity/album_yaml_test.go new file mode 100644 index 000000000..e7f950e24 --- /dev/null +++ b/internal/entity/album_yaml_test.go @@ -0,0 +1,120 @@ +package entity + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAlbum_Yaml(t *testing.T) { + t.Run("berlin-2019", func(t *testing.T) { + m := AlbumFixtures.Get("berlin-2019") + + if err := m.Find(); err != nil { + t.Fatal(err) + } + + result, err := m.Yaml() + + if err != nil { + t.Fatal(err) + } + + t.Logf("YAML: %s", result) + }) + t.Run("christmas2030", func(t *testing.T) { + m := AlbumFixtures.Get("christmas2030") + + if err := m.Find(); err != nil { + t.Fatal(err) + } + + result, err := m.Yaml() + + if err != nil { + t.Fatal(err) + } + + t.Logf("YAML: %s", result) + }) +} + +func TestAlbum_SaveAsYaml(t *testing.T) { + t.Run("berlin-2019", func(t *testing.T) { + m := AlbumFixtures.Get("berlin-2019") + + if err := m.Find(); err != nil { + t.Fatal(err) + } + + fileName := m.YamlFileName("testdata") + + if err := m.SaveAsYaml(fileName); err != nil { + t.Fatal(err) + } + + if err := m.LoadFromYaml(fileName); err != nil { + t.Fatal(err) + } + + if err := os.Remove(fileName); err != nil { + t.Fatal(err) + } + }) +} + +func TestAlbum_LoadFromYaml(t *testing.T) { + t.Run("berlin-2020", func(t *testing.T) { + fileName := "testdata/album/at9lxuqxpoaaaaaa.yml" + + m := Album{} + + if err := m.LoadFromYaml(fileName); err != nil { + t.Fatal(err) + } + + if err := m.Save(); err != nil { + t.Fatal(err) + } + + a := Album{AlbumUID: "at9lxuqxpoaaaaaa"} + + if err := a.Find(); err != nil { + t.Fatal(err) + } + + if existingYaml, err := ioutil.ReadFile(fileName); err != nil { + t.Fatal(err) + } else if newYaml, err := a.Yaml(); err != nil { + t.Fatal(err) + } else { + assert.Equal(t, existingYaml[:50], newYaml[:50]) + assert.Equal(t, a.AlbumUID, m.AlbumUID) + assert.Equal(t, a.AlbumSlug, m.AlbumSlug) + assert.Equal(t, a.AlbumType, m.AlbumType) + assert.Equal(t, a.AlbumTitle, m.AlbumTitle) + assert.Equal(t, a.AlbumDescription, m.AlbumDescription) + assert.Equal(t, a.AlbumOrder, m.AlbumOrder) + assert.Equal(t, a.AlbumCountry, m.AlbumCountry) + assert.Equal(t, a.CreatedAt, m.CreatedAt) + assert.Equal(t, a.UpdatedAt, m.UpdatedAt) + assert.Equal(t, len(a.Photos), len(m.Photos)) + } + }) +} + +func TestAlbum_YamlFileName(t *testing.T) { + t.Run("berlin-2019", func(t *testing.T) { + m := AlbumFixtures.Get("berlin-2019") + + if err := m.Find(); err != nil { + t.Fatal(err) + } + + fileName := m.YamlFileName("/foo/bar") + + assert.Equal(t, "/foo/bar/album/at9lxuqxpogaaba9.yml", fileName) + }) +} diff --git a/internal/entity/photo_album.go b/internal/entity/photo_album.go index 7e3d3a131..d26514a40 100644 --- a/internal/entity/photo_album.go +++ b/internal/entity/photo_album.go @@ -4,16 +4,18 @@ import ( "time" ) +type PhotoAlbums []PhotoAlbum + // PhotoAlbum represents the many_to_many relation between Photo and Album type PhotoAlbum struct { - PhotoUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false"` - AlbumUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;index"` - Order int - Hidden bool - CreatedAt time.Time - UpdatedAt time.Time - Photo *Photo `gorm:"PRELOAD:false"` - Album *Album `gorm:"PRELOAD:true"` + PhotoUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false" json:"PhotoUID" yaml:"UID"` + AlbumUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;index" json:"AlbumUID" yaml:"-"` + Order int `json:"Order" yaml:"Order,omitempty"` + Hidden bool `json:"Hidden" yaml:"Hidden,omitempty"` + CreatedAt time.Time `json:"CreatedAt" yaml:"CreatedAt,omitempty"` + UpdatedAt time.Time `json:"UpdatedAt" yaml:"UpdatedAt,omitempty"` + Photo *Photo `gorm:"PRELOAD:false" yaml:"-"` + Album *Album `gorm:"PRELOAD:true" yaml:"-"` } // TableName returns PhotoAlbum table identifier "photos_albums" diff --git a/internal/entity/photo_album_fixtures.go b/internal/entity/photo_album_fixtures.go index 57140db56..673dfbe17 100644 --- a/internal/entity/photo_album_fixtures.go +++ b/internal/entity/photo_album_fixtures.go @@ -24,6 +24,7 @@ var PhotoAlbumFixtures = PhotoAlbumMap{ "1": { PhotoUID: "pt9jtdre2lvl0yh7", AlbumUID: "at9lxuqxpogaaba8", + Hidden: false, Order: 0, CreatedAt: time.Date(2020, 3, 6, 2, 6, 51, 0, time.UTC), UpdatedAt: time.Date(2020, 3, 28, 14, 6, 0, 0, time.UTC), @@ -33,6 +34,7 @@ var PhotoAlbumFixtures = PhotoAlbumMap{ "2": { PhotoUID: "pt9jtdre2lvl0y11", AlbumUID: "at9lxuqxpogaaba9", + Hidden: false, Order: 0, CreatedAt: time.Date(2020, 2, 6, 2, 6, 51, 0, time.UTC), UpdatedAt: time.Date(2020, 4, 28, 14, 6, 0, 0, time.UTC), @@ -42,6 +44,7 @@ var PhotoAlbumFixtures = PhotoAlbumMap{ "3": { PhotoUID: "pt9jtdre2lvl0yh8", AlbumUID: "at9lxuqxpogaaba9", + Hidden: false, Order: 0, CreatedAt: time.Date(2020, 2, 6, 2, 6, 51, 0, time.UTC), UpdatedAt: time.Date(2020, 4, 28, 14, 6, 0, 0, time.UTC), diff --git a/internal/entity/photo_label.go b/internal/entity/photo_label.go index 97140cb84..e1fd2886b 100644 --- a/internal/entity/photo_label.go +++ b/internal/entity/photo_label.go @@ -4,6 +4,8 @@ import ( "github.com/photoprism/photoprism/internal/classify" ) +type PhotoLabels []PhotoLabel + // PhotoLabel represents the many-to-many relation between Photo and label. // Labels are weighted by uncertainty (100 - confidence) type PhotoLabel struct { diff --git a/internal/entity/testdata/album/at9lxuqxpoaaaaaa.yml b/internal/entity/testdata/album/at9lxuqxpoaaaaaa.yml new file mode 100755 index 000000000..889a94835 --- /dev/null +++ b/internal/entity/testdata/album/at9lxuqxpoaaaaaa.yml @@ -0,0 +1,16 @@ +UID: at9lxuqxpoaaaaaa +Slug: berlin-2020 +Type: album +Title: Berlin 2020 +Description: We still love Berlin! +Order: oldest +Country: zz +CreatedAt: 2019-07-01T00:00:00Z +UpdatedAt: 2020-02-01T00:00:00Z +Photos: +- UID: pt9jtdre2lvl0y11 + CreatedAt: 2020-02-06T02:06:51Z + UpdatedAt: 2020-04-28T14:06:00Z +- UID: pt9jtdre2lvl0yh8 + CreatedAt: 2020-02-06T02:06:51Z + UpdatedAt: 2020-04-28T14:06:00Z diff --git a/internal/photoprism/albums.go b/internal/photoprism/albums.go new file mode 100644 index 000000000..89cd3388c --- /dev/null +++ b/internal/photoprism/albums.go @@ -0,0 +1,86 @@ +package photoprism + +import ( + "path/filepath" + "regexp" + + "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/query" + "github.com/photoprism/photoprism/pkg/txt" +) + +// BackupAlbums creates a YAML file backup of all albums. +func BackupAlbums(force bool) (count int, result error) { + c := Config() + + if !c.SidecarYaml() && !force { + log.Debugf("backup: album yaml files disabled") + return count, nil + } + + albums, err := query.GetAlbums(0, 9999) + + if err != nil { + return count, err + } + + for _, a := range albums { + fileName := a.YamlFileName(c.AlbumsPath()) + + if err := a.SaveAsYaml(fileName); err != nil { + log.Errorf("album: %s (update yaml)", err) + result = err + } else { + log.Tracef("backup: saved album yaml file %s", txt.Quote(filepath.Base(fileName))) + count++ + } + } + + return count, result +} + +// RestoreAlbums restores all album YAML file backups. +func RestoreAlbums(force bool) (count int, result error) { + c := Config() + + if !c.SidecarYaml() && !force { + log.Debugf("restore: album yaml files disabled") + return count, nil + } + + existing, err := query.GetAlbums(0, 1) + + if err != nil { + return count, err + } + + if len(existing) > 0 && !force { + log.Debugf("restore: album yaml files disabled") + return count, nil + } + + albums, err := filepath.Glob(regexp.QuoteMeta(c.AlbumsPath()) + "/**/*.yml") + + if err != nil { + return count, err + } + + if len(albums) == 0 { + return count, nil + } + + for _, fileName := range albums { + a := entity.Album{} + + if err := a.LoadFromYaml(fileName); err != nil { + log.Errorf("album: %s (load yaml)", err) + result = err + } else if err := a.Create(); err != nil { + log.Warnf("%s: %s already exists", a.AlbumType, txt.Quote(a.AlbumTitle)) + } else { + count++ + } + } + + return count, result +} diff --git a/internal/photoprism/moments.go b/internal/photoprism/moments.go index 02709bc11..88ce2a64b 100644 --- a/internal/photoprism/moments.go +++ b/internal/photoprism/moments.go @@ -232,6 +232,12 @@ func (m *Moments) Start() (err error) { log.Errorf("moments: %s (update album dates)", err.Error()) } + if count, err := BackupAlbums(false); err != nil { + log.Errorf("moments: %s (backup albums)", err.Error()) + } else if count > 0 { + log.Debugf("moments: %d albums saved as yaml files", count) + } + return nil } diff --git a/internal/query/albums.go b/internal/query/albums.go index 44b5bee72..63e48dcb6 100644 --- a/internal/query/albums.go +++ b/internal/query/albums.go @@ -194,3 +194,9 @@ func UpdateAlbumDates() error { return nil } } + +// GetAlbums returns a slice of albums. +func GetAlbums(offset, limit int) (results entity.Albums, err error) { + err = UnscopedDb().Table("albums").Select("*").Offset(offset).Limit(limit).Find(&results).Error + return results, err +}