From 1df0d9a549b291c2b5453903b62152213acdd22e Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Tue, 26 May 2020 19:27:29 +0200 Subject: [PATCH] Change name of maintenance worker to "prism" #154 See https://twitter.com/browseyourlife/status/1265289044856123393 Signed-off-by: Michael Mayer --- internal/api/photo.go | 8 +- internal/api/photo_label.go | 6 +- internal/config/config.go | 2 +- internal/entity/const.go | 1 + internal/entity/photo.go | 58 ++++++++- internal/mutex/mutex.go | 4 +- internal/photoprism/purge.go | 4 +- internal/query/photo.go | 54 ++++++--- internal/query/photo_test.go | 8 +- internal/workers/groom.go | 42 ------- internal/workers/prism.go | 110 ++++++++++++++++++ .../workers/{groom_test.go => prism_test.go} | 14 ++- internal/workers/workers.go | 10 +- 13 files changed, 235 insertions(+), 86 deletions(-) delete mode 100644 internal/workers/groom.go create mode 100644 internal/workers/prism.go rename internal/workers/{groom_test.go => prism_test.go} (60%) diff --git a/internal/api/photo.go b/internal/api/photo.go index f122e9d8e..68cbc6c31 100644 --- a/internal/api/photo.go +++ b/internal/api/photo.go @@ -40,7 +40,7 @@ func GetPhoto(router *gin.RouterGroup, conf *config.Config) { return } - p, err := query.PreloadPhotoByUID(c.Param("uid")) + p, err := query.PhotoPreloadByUID(c.Param("uid")) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) @@ -95,7 +95,7 @@ func UpdatePhoto(router *gin.RouterGroup, conf *config.Config) { event.Success("photo saved") - p, err := query.PreloadPhotoByUID(uid) + p, err := query.PhotoPreloadByUID(uid) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) @@ -152,7 +152,7 @@ func GetPhotoYaml(router *gin.RouterGroup, conf *config.Config) { return } - p, err := query.PreloadPhotoByUID(c.Param("uid")) + p, err := query.PhotoPreloadByUID(c.Param("uid")) if err != nil { c.AbortWithStatus(http.StatusNotFound) @@ -264,7 +264,7 @@ func SetPhotoPrimary(router *gin.RouterGroup, conf *config.Config) { event.Success("photo saved") - p, err := query.PreloadPhotoByUID(uid) + p, err := query.PhotoPreloadByUID(uid) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) diff --git a/internal/api/photo_label.go b/internal/api/photo_label.go index f90e0aa11..67a264052 100644 --- a/internal/api/photo_label.go +++ b/internal/api/photo_label.go @@ -61,7 +61,7 @@ func AddPhotoLabel(router *gin.RouterGroup, conf *config.Config) { } } - p, err := query.PreloadPhotoByUID(c.Param("uid")) + p, err := query.PhotoPreloadByUID(c.Param("uid")) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) @@ -121,7 +121,7 @@ func RemovePhotoLabel(router *gin.RouterGroup, conf *config.Config) { entity.Db().Save(&label) } - p, err := query.PreloadPhotoByUID(c.Param("uid")) + p, err := query.PhotoPreloadByUID(c.Param("uid")) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) @@ -186,7 +186,7 @@ func UpdatePhotoLabel(router *gin.RouterGroup, conf *config.Config) { return } - p, err := query.PreloadPhotoByUID(c.Param("uid")) + p, err := query.PhotoPreloadByUID(c.Param("uid")) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound) diff --git a/internal/config/config.go b/internal/config/config.go index c27a3f272..c5968326e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -204,7 +204,7 @@ func (c *Config) Shutdown() { mutex.MainWorker.Cancel() mutex.ShareWorker.Cancel() mutex.SyncWorker.Cancel() - mutex.GroomWorker.Cancel() + mutex.PrismWorker.Cancel() if err := c.CloseDb(); err != nil { log.Errorf("could not close database connection: %s", err) diff --git a/internal/entity/const.go b/internal/entity/const.go index 8dc6bc48b..343a1b082 100644 --- a/internal/entity/const.go +++ b/internal/entity/const.go @@ -6,6 +6,7 @@ const ( // data sources SrcAuto = "" SrcManual = "manual" + SrcEstimate = "estimate" SrcName = "name" SrcMeta = "meta" SrcXmp = "xmp" diff --git a/internal/entity/photo.go b/internal/entity/photo.go index 3fbc4ed5d..bbcd3bded 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -64,6 +64,7 @@ type Photo struct { CreatedAt time.Time `yaml:"CreatedAt,omitempty"` UpdatedAt time.Time `yaml:"UpdatedAt,omitempty"` EditedAt *time.Time `yaml:"EditedAt,omitempty"` + MaintainedAt *time.Time `sql:"index" yaml:"-"` DeletedAt *time.Time `sql:"index" yaml:"DeletedAt,omitempty"` } @@ -130,7 +131,6 @@ func (m *Photo) Save() error { return errors.New("photo: can't save to database, id is empty") } - db := Db() labels := m.ClassifyLabels() m.UpdateYearMonth() @@ -151,7 +151,7 @@ func (m *Photo) Save() error { m.PhotoQuality = m.QualityScore() - if err := db.Unscoped().Save(m).Error; err != nil { + if err := UnscopedDb().Save(m).Error; err != nil { return err } @@ -631,3 +631,57 @@ func (m *Photo) SetFavorite(favorite bool) error { return nil } + +// EstimatePosition updates the photo with an estimated geolocation if possible. +func (m *Photo) EstimatePosition() { + var recentPhoto Photo + + if result := UnscopedDb(). + Where("place_uid <> '' && place_uid <> 'zz'"). + Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", m.TakenAt)). + Preload("Place").First(&recentPhoto); result.Error == nil { + if recentPhoto.HasPlace() { + m.Place = recentPhoto.Place + m.PlaceUID = recentPhoto.PlaceUID + m.PhotoCountry = recentPhoto.PhotoCountry + m.LocSrc = SrcEstimate + log.Debugf("prism: approximate location for %s is %s", m.PhotoUID, recentPhoto.PlaceUID) + } + } +} + +// Maintain photo data, improve if possible. +func (m *Photo) Maintain() error { + if !m.HasID() { + return errors.New("photo: can't maintain, id is empty") + } + + maintained := time.Now() + m.MaintainedAt = &maintained + + if m.NoPlace() && (m.LocSrc == SrcAuto || m.LocSrc == SrcEstimate) { + m.EstimatePosition() + } + + labels := m.ClassifyLabels() + + m.UpdateYearMonth() + + if err := m.UpdateTitle(labels); err != nil { + log.Warnf("%s (%s)", err.Error(), m.PhotoUID) + } + + if m.DetailsLoaded() { + w := txt.UniqueKeywords(m.Details.Keywords) + w = append(w, labels.Keywords()...) + m.Details.Keywords = strings.Join(txt.UniqueWords(w), ", ") + } + + if err := m.IndexKeywords(); err != nil { + log.Error(err) + } + + m.PhotoQuality = m.QualityScore() + + return UnscopedDb().Save(m).Error +} diff --git a/internal/mutex/mutex.go b/internal/mutex/mutex.go index ebea1e7a2..f0ad894b9 100644 --- a/internal/mutex/mutex.go +++ b/internal/mutex/mutex.go @@ -9,10 +9,10 @@ var ( MainWorker = Busy{} SyncWorker = Busy{} ShareWorker = Busy{} - GroomWorker = Busy{} + PrismWorker = Busy{} ) // WorkersBusy returns true if any worker is busy. func WorkersBusy() bool { - return MainWorker.Busy() || SyncWorker.Busy() || ShareWorker.Busy() || GroomWorker.Busy() + return MainWorker.Busy() || SyncWorker.Busy() || ShareWorker.Busy() || PrismWorker.Busy() } diff --git a/internal/photoprism/purge.go b/internal/photoprism/purge.go index 621ad7c50..98d4d5a23 100644 --- a/internal/photoprism/purge.go +++ b/internal/photoprism/purge.go @@ -120,7 +120,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh offset = 0 for { - photos, err := query.MissingPhotos(limit, offset) + photos, err := query.PhotosMissing(limit, offset) if err != nil { return purgedFiles, purgedPhotos, err @@ -169,7 +169,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh log.Info("purge: finding hidden photos") - if err := query.ResetPhotosQuality(); err != nil { + if err := query.ResetPhotoQuality(); err != nil { return purgedFiles, purgedPhotos, err } diff --git a/internal/query/photo.go b/internal/query/photo.go index 622355919..ed22b402d 100644 --- a/internal/query/photo.go +++ b/internal/query/photo.go @@ -1,6 +1,8 @@ package query import ( + "time" + "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/entity" ) @@ -8,16 +10,16 @@ import ( // PhotoByID returns a Photo based on the ID. func PhotoByID(photoID uint64) (photo entity.Photo, err error) { if err := UnscopedDb().Where("id = ?", photoID). + Preload("Labels", func(db *gorm.DB) *gorm.DB { + return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") + }). + Preload("Labels.Label"). Preload("Links"). Preload("Camera"). Preload("Lens"). Preload("Details"). Preload("Location"). Preload("Location.Place"). - Preload("Labels", func(db *gorm.DB) *gorm.DB { - return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") - }). - Preload("Labels.Label"). First(&photo).Error; err != nil { return photo, err } @@ -28,16 +30,16 @@ func PhotoByID(photoID uint64) (photo entity.Photo, err error) { // PhotoByUID returns a Photo based on the UID. func PhotoByUID(photoUID string) (photo entity.Photo, err error) { if err := UnscopedDb().Where("photo_uid = ?", photoUID). + Preload("Labels", func(db *gorm.DB) *gorm.DB { + return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") + }). + Preload("Labels.Label"). Preload("Links"). Preload("Camera"). Preload("Lens"). Preload("Details"). Preload("Location"). Preload("Location.Place"). - Preload("Labels", func(db *gorm.DB) *gorm.DB { - return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") - }). - Preload("Labels.Label"). First(&photo).Error; err != nil { return photo, err } @@ -45,19 +47,19 @@ func PhotoByUID(photoUID string) (photo entity.Photo, err error) { return photo, nil } -// PreloadPhotoByUID returns a Photo based on the UID with all dependencies preloaded. -func PreloadPhotoByUID(photoUID string) (photo entity.Photo, err error) { +// PhotoPreloadByUID returns a Photo based on the UID with all dependencies preloaded. +func PhotoPreloadByUID(photoUID string) (photo entity.Photo, err error) { if err := UnscopedDb().Where("photo_uid = ?", photoUID). Preload("Labels", func(db *gorm.DB) *gorm.DB { return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") }). Preload("Labels.Label"). + Preload("Links"). Preload("Camera"). Preload("Lens"). - Preload("Links"). + Preload("Details"). Preload("Location"). Preload("Location.Place"). - Preload("Details"). First(&photo).Error; err != nil { return photo, err } @@ -67,8 +69,8 @@ func PreloadPhotoByUID(photoUID string) (photo entity.Photo, err error) { return photo, nil } -// MissingPhotos returns photo entities without existing files. -func MissingPhotos(limit int, offset int) (entities []entity.Photo, err error) { +// PhotosMissing returns photo entities without existing files. +func PhotosMissing(limit int, offset int) (entities Photos, err error) { err = Db(). Select("photos.*"). Joins("JOIN files a ON photos.id = a.photo_id "). @@ -81,9 +83,29 @@ func MissingPhotos(limit int, offset int) (entities []entity.Photo, err error) { return entities, err } -// ResetPhotosQuality resets the quality of photos without primary file to -1. -func ResetPhotosQuality() error { +// ResetPhotoQuality resets the quality of photos without primary file to -1. +func ResetPhotoQuality() error { return Db().Table("photos"). Where("id IN (SELECT photos.id FROM photos LEFT JOIN files ON photos.id = files.photo_id AND files.file_primary = 1 WHERE files.id IS NULL GROUP BY photos.id)"). Update("photo_quality", -1).Error } + +// PhotosMaintenance returns photos selected for maintenance. +func PhotosMaintenance(limit int, offset int) (entities Photos, err error) { + err = Db(). + Preload("Labels", func(db *gorm.DB) *gorm.DB { + return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC") + }). + Preload("Labels.Label"). + Preload("Links"). + Preload("Camera"). + Preload("Lens"). + Preload("Details"). + Preload("Location"). + Preload("Location.Place"). + Where("maintained_at IS NULL OR maintained_at < ?", time.Now().Add(-1*time.Hour*24*7)). + Where("updated_at < ?", time.Now().Add(-1*time.Hour*36)). + Limit(limit).Offset(offset).Find(&entities).Error + + return entities, err +} diff --git a/internal/query/photo_test.go b/internal/query/photo_test.go index 70dd10a96..28c2bb8af 100644 --- a/internal/query/photo_test.go +++ b/internal/query/photo_test.go @@ -40,7 +40,7 @@ func TestPhotoByUID(t *testing.T) { func TestPreloadPhotoByUID(t *testing.T) { t.Run("photo found", func(t *testing.T) { - result, err := PreloadPhotoByUID("pt9jtdre2lvl0y12") + result, err := PhotoPreloadByUID("pt9jtdre2lvl0y12") if err != nil { t.Fatal(err) } @@ -48,14 +48,14 @@ func TestPreloadPhotoByUID(t *testing.T) { }) t.Run("no photo found", func(t *testing.T) { - result, err := PreloadPhotoByUID("99999") + result, err := PhotoPreloadByUID("99999") assert.Error(t, err, "record not found") t.Log(result) }) } func TestMissingPhotos(t *testing.T) { - r, err := MissingPhotos(15, 0) + r, err := PhotosMissing(15, 0) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestMissingPhotos(t *testing.T) { } func TestResetPhotosQuality(t *testing.T) { - err := ResetPhotosQuality() + err := ResetPhotoQuality() if err != nil { t.Fatal(err) } diff --git a/internal/workers/groom.go b/internal/workers/groom.go deleted file mode 100644 index cd47cf4f4..000000000 --- a/internal/workers/groom.go +++ /dev/null @@ -1,42 +0,0 @@ -package workers - -import ( - "github.com/photoprism/photoprism/internal/config" - "github.com/photoprism/photoprism/internal/mutex" -) - -// Groom represents a groom worker. -type Groom struct { - conf *config.Config -} - -// NewGroom returns a new groom worker. -func NewGroom(conf *config.Config) *Groom { - return &Groom{conf: conf} -} - -// logError logs an error message if err is not nil. -func (worker *Groom) logError(err error) { - if err != nil { - log.Errorf("groom: %s", err.Error()) - } -} - -// logWarn logs a warning message if err is not nil. -func (worker *Groom) logWarn(err error) { - if err != nil { - log.Warnf("groom: %s", err.Error()) - } -} - -// Start starts the groom worker. -func (worker *Groom) Start() (err error) { - if err := mutex.GroomWorker.Start(); err != nil { - worker.logWarn(err) - return err - } - - defer mutex.GroomWorker.Stop() - - return err -} diff --git a/internal/workers/prism.go b/internal/workers/prism.go new file mode 100644 index 000000000..23e9a731e --- /dev/null +++ b/internal/workers/prism.go @@ -0,0 +1,110 @@ +package workers + +import ( + "errors" + "runtime" + "time" + + "github.com/photoprism/photoprism/internal/config" + "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/mutex" + "github.com/photoprism/photoprism/internal/query" +) + +// Prism represents a background maintenance worker. +type Prism struct { + conf *config.Config +} + +// NewPrism returns a new background maintenance worker. +func NewPrism(conf *config.Config) *Prism { + return &Prism{conf: conf} +} + +// logError logs an error message if err is not nil. +func (worker *Prism) logError(err error) { + if err != nil { + log.Errorf("prism: %s", err.Error()) + } +} + +// logWarn logs a warning message if err is not nil. +func (worker *Prism) logWarn(err error) { + if err != nil { + log.Warnf("prism: %s", err.Error()) + } +} + +// originalsPath returns the original media files path as string. +func (worker *Prism) originalsPath() string { + return worker.conf.OriginalsPath() +} + +// Start starts the prism worker. +func (worker *Prism) Start() (err error) { + if err := mutex.PrismWorker.Start(); err != nil { + worker.logWarn(err) + return err + } + + defer func() { + mutex.PrismWorker.Stop() + + if err := recover(); err != nil { + log.Errorf("prism: %s [panic]", err) + } + }() + + done := make(map[string]bool) + + limit := 50 + offset := 0 + + for { + photos, err := query.PhotosMaintenance(limit, offset) + + if err != nil { + return err + } + + if len(photos) == 0 { + break + } else if offset == 0 { + log.Infof("prism: starting photo maintenance") + } + + for _, photo := range photos { + if mutex.PrismWorker.Canceled() { + return errors.New("prism: maintenance canceled") + } + + if done[photo.PhotoUID] { + continue + } + + done[photo.PhotoUID] = true + + worker.logError(photo.Maintain()) + } + + if mutex.PrismWorker.Canceled() { + return errors.New("prism: maintenance canceled") + } + + offset += limit + + time.Sleep(100 * time.Millisecond) + } + + if len(done) > 0 { + log.Infof("prism: maintained %d photos", len(done)) + } + + worker.logError(query.ResetPhotoQuality()) + + worker.logError(entity.UpdatePhotoCounts()) + + runtime.GC() + + return nil +} diff --git a/internal/workers/groom_test.go b/internal/workers/prism_test.go similarity index 60% rename from internal/workers/groom_test.go rename to internal/workers/prism_test.go index 8623a8b02..1061af0e2 100644 --- a/internal/workers/groom_test.go +++ b/internal/workers/prism_test.go @@ -8,14 +8,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGroom_Start(t *testing.T) { +func TestPrism_Start(t *testing.T) { conf := config.TestConfig() - worker := NewGroom(conf) + worker := NewPrism(conf) - assert.IsType(t, &Groom{}, worker) + assert.IsType(t, &Prism{}, worker) - if err := mutex.GroomWorker.Start(); err != nil { + if err := mutex.PrismWorker.Start(); err != nil { t.Fatal(err) } @@ -23,7 +23,11 @@ func TestGroom_Start(t *testing.T) { t.Fatal("error expected") } - mutex.GroomWorker.Stop() + mutex.PrismWorker.Stop() + + if err := worker.Start(); err != nil { + t.Fatal(err) + } if err := worker.Start(); err != nil { t.Fatal(err) diff --git a/internal/workers/workers.go b/internal/workers/workers.go index 3d7d5d46f..55e100d32 100644 --- a/internal/workers/workers.go +++ b/internal/workers/workers.go @@ -21,12 +21,12 @@ func Start(conf *config.Config) { case <-stop: log.Info("shutting down workers") ticker.Stop() - mutex.GroomWorker.Cancel() + mutex.PrismWorker.Cancel() mutex.ShareWorker.Cancel() mutex.SyncWorker.Cancel() return case <-ticker.C: - StartGroom(conf) + StartPrism(conf) StartShare(conf) StartSync(conf) } @@ -39,11 +39,11 @@ func Stop() { stop <- true } -// StartGroom runs the groom worker once. -func StartGroom(conf *config.Config) { +// StartPrism runs the prism worker once. +func StartPrism(conf *config.Config) { if !mutex.WorkersBusy() { go func() { - worker := NewGroom(conf) + worker := NewPrism(conf) if err := worker.Start(); err != nil { log.Error(err) }