luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Change name of maintenance worker to "prism" #154

Browse source 
See https://twitter.com/browseyourlife/status/1265289044856123393

Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-05-26 19:27:29 +02:00
parent 968cd71f34
commit 1df0d9a549
13 changed files with 235 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
internal/api/photo.go
View file

@ -40,7 +40,7 @@ func GetPhoto(router *gin.RouterGroup, conf *config.Config) {
return
}
p, err := query.PreloadPhotoByUID(c.Param("uid"))
p, err := query.PhotoPreloadByUID(c.Param("uid"))
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)
@ -95,7 +95,7 @@ func UpdatePhoto(router *gin.RouterGroup, conf *config.Config) {
event.Success("photo saved")
p, err := query.PreloadPhotoByUID(uid)
p, err := query.PhotoPreloadByUID(uid)
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)
@ -152,7 +152,7 @@ func GetPhotoYaml(router *gin.RouterGroup, conf *config.Config) {
return
}
p, err := query.PreloadPhotoByUID(c.Param("uid"))
p, err := query.PhotoPreloadByUID(c.Param("uid"))
if err != nil {
c.AbortWithStatus(http.StatusNotFound)
@ -264,7 +264,7 @@ func SetPhotoPrimary(router *gin.RouterGroup, conf *config.Config) {
event.Success("photo saved")
p, err := query.PreloadPhotoByUID(uid)
p, err := query.PhotoPreloadByUID(uid)
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)

6
internal/api/photo_label.go
View file

@ -61,7 +61,7 @@ func AddPhotoLabel(router *gin.RouterGroup, conf *config.Config) {
}
}
p, err := query.PreloadPhotoByUID(c.Param("uid"))
p, err := query.PhotoPreloadByUID(c.Param("uid"))
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)
@ -121,7 +121,7 @@ func RemovePhotoLabel(router *gin.RouterGroup, conf *config.Config) {
entity.Db().Save(&label)
}
p, err := query.PreloadPhotoByUID(c.Param("uid"))
p, err := query.PhotoPreloadByUID(c.Param("uid"))
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)
@ -186,7 +186,7 @@ func UpdatePhotoLabel(router *gin.RouterGroup, conf *config.Config) {
return
}
p, err := query.PreloadPhotoByUID(c.Param("uid"))
p, err := query.PhotoPreloadByUID(c.Param("uid"))
if err != nil {
c.AbortWithStatusJSON(http.StatusNotFound, ErrPhotoNotFound)

2
internal/config/config.go
View file

@ -204,7 +204,7 @@ func (c *Config) Shutdown() {
mutex.MainWorker.Cancel()
mutex.ShareWorker.Cancel()
mutex.SyncWorker.Cancel()
mutex.GroomWorker.Cancel()
mutex.PrismWorker.Cancel()
if err := c.CloseDb(); err != nil {
log.Errorf("could not close database connection: %s", err)

1
internal/entity/const.go
View file

@ -6,6 +6,7 @@ const (
// data sources
SrcAuto = ""
SrcManual = "manual"
SrcEstimate = "estimate"
SrcName = "name"
SrcMeta = "meta"
SrcXmp = "xmp"

58
internal/entity/photo.go
View file

@ -64,6 +64,7 @@ type Photo struct {
CreatedAt time.Time `yaml:"CreatedAt,omitempty"`
UpdatedAt time.Time `yaml:"UpdatedAt,omitempty"`
EditedAt *time.Time `yaml:"EditedAt,omitempty"`
MaintainedAt *time.Time `sql:"index" yaml:"-"`
DeletedAt *time.Time `sql:"index" yaml:"DeletedAt,omitempty"`
}
@ -130,7 +131,6 @@ func (m *Photo) Save() error {
return errors.New("photo: can't save to database, id is empty")
}
db := Db()
labels := m.ClassifyLabels()
m.UpdateYearMonth()
@ -151,7 +151,7 @@ func (m *Photo) Save() error {
m.PhotoQuality = m.QualityScore()
if err := db.Unscoped().Save(m).Error; err != nil {
if err := UnscopedDb().Save(m).Error; err != nil {
return err
}
@ -631,3 +631,57 @@ func (m *Photo) SetFavorite(favorite bool) error {
return nil
}
// EstimatePosition updates the photo with an estimated geolocation if possible.
func (m *Photo) EstimatePosition() {
var recentPhoto Photo
if result := UnscopedDb().
Where("place_uid <> '' && place_uid <> 'zz'").
Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", m.TakenAt)).
Preload("Place").First(&recentPhoto); result.Error == nil {
if recentPhoto.HasPlace() {
m.Place = recentPhoto.Place
m.PlaceUID = recentPhoto.PlaceUID
m.PhotoCountry = recentPhoto.PhotoCountry
m.LocSrc = SrcEstimate
log.Debugf("prism: approximate location for %s is %s", m.PhotoUID, recentPhoto.PlaceUID)
}
}
}
// Maintain photo data, improve if possible.
func (m *Photo) Maintain() error {
if !m.HasID() {
return errors.New("photo: can't maintain, id is empty")
}
maintained := time.Now()
m.MaintainedAt = &maintained
if m.NoPlace() && (m.LocSrc == SrcAuto || m.LocSrc == SrcEstimate) {
m.EstimatePosition()
}
labels := m.ClassifyLabels()
m.UpdateYearMonth()
if err := m.UpdateTitle(labels); err != nil {
log.Warnf("%s (%s)", err.Error(), m.PhotoUID)
}
if m.DetailsLoaded() {
w := txt.UniqueKeywords(m.Details.Keywords)
w = append(w, labels.Keywords()...)
m.Details.Keywords = strings.Join(txt.UniqueWords(w), ", ")
}
if err := m.IndexKeywords(); err != nil {
log.Error(err)
}
m.PhotoQuality = m.QualityScore()
return UnscopedDb().Save(m).Error
}

4
internal/mutex/mutex.go
View file

@ -9,10 +9,10 @@ var (
MainWorker = Busy{}
SyncWorker = Busy{}
ShareWorker = Busy{}
GroomWorker = Busy{}
PrismWorker = Busy{}
)
// WorkersBusy returns true if any worker is busy.
func WorkersBusy() bool {
return MainWorker.Busy() || SyncWorker.Busy() || ShareWorker.Busy() || GroomWorker.Busy()
return MainWorker.Busy() || SyncWorker.Busy() || ShareWorker.Busy() || PrismWorker.Busy()
}

4
internal/photoprism/purge.go
View file

@ -120,7 +120,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
offset = 0
for {
photos, err := query.MissingPhotos(limit, offset)
photos, err := query.PhotosMissing(limit, offset)
if err != nil {
return purgedFiles, purgedPhotos, err
@ -169,7 +169,7 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
log.Info("purge: finding hidden photos")
if err := query.ResetPhotosQuality(); err != nil {
if err := query.ResetPhotoQuality(); err != nil {
return purgedFiles, purgedPhotos, err
}

54
internal/query/photo.go
View file

@ -1,6 +1,8 @@
package query
import (
"time"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
)
@ -8,16 +10,16 @@ import (
// PhotoByID returns a Photo based on the ID.
func PhotoByID(photoID uint64) (photo entity.Photo, err error) {
if err := UnscopedDb().Where("id = ?", photoID).
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
Preload("Links").
Preload("Camera").
Preload("Lens").
Preload("Details").
Preload("Location").
Preload("Location.Place").
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
First(&photo).Error; err != nil {
return photo, err
}
@ -28,16 +30,16 @@ func PhotoByID(photoID uint64) (photo entity.Photo, err error) {
// PhotoByUID returns a Photo based on the UID.
func PhotoByUID(photoUID string) (photo entity.Photo, err error) {
if err := UnscopedDb().Where("photo_uid = ?", photoUID).
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
Preload("Links").
Preload("Camera").
Preload("Lens").
Preload("Details").
Preload("Location").
Preload("Location.Place").
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
First(&photo).Error; err != nil {
return photo, err
}
@ -45,19 +47,19 @@ func PhotoByUID(photoUID string) (photo entity.Photo, err error) {
return photo, nil
}
// PreloadPhotoByUID returns a Photo based on the UID with all dependencies preloaded.
func PreloadPhotoByUID(photoUID string) (photo entity.Photo, err error) {
// PhotoPreloadByUID returns a Photo based on the UID with all dependencies preloaded.
func PhotoPreloadByUID(photoUID string) (photo entity.Photo, err error) {
if err := UnscopedDb().Where("photo_uid = ?", photoUID).
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
Preload("Links").
Preload("Camera").
Preload("Lens").
Preload("Links").
Preload("Details").
Preload("Location").
Preload("Location.Place").
Preload("Details").
First(&photo).Error; err != nil {
return photo, err
}
@ -67,8 +69,8 @@ func PreloadPhotoByUID(photoUID string) (photo entity.Photo, err error) {
return photo, nil
}
// MissingPhotos returns photo entities without existing files.
func MissingPhotos(limit int, offset int) (entities []entity.Photo, err error) {
// PhotosMissing returns photo entities without existing files.
func PhotosMissing(limit int, offset int) (entities Photos, err error) {
err = Db().
Select("photos.*").
Joins("JOIN files a ON photos.id = a.photo_id ").
@ -81,9 +83,29 @@ func MissingPhotos(limit int, offset int) (entities []entity.Photo, err error) {
return entities, err
}
// ResetPhotosQuality resets the quality of photos without primary file to -1.
func ResetPhotosQuality() error {
// ResetPhotoQuality resets the quality of photos without primary file to -1.
func ResetPhotoQuality() error {
return Db().Table("photos").
Where("id IN (SELECT photos.id FROM photos LEFT JOIN files ON photos.id = files.photo_id AND files.file_primary = 1 WHERE files.id IS NULL GROUP BY photos.id)").
Update("photo_quality", -1).Error
}
// PhotosMaintenance returns photos selected for maintenance.
func PhotosMaintenance(limit int, offset int) (entities Photos, err error) {
err = Db().
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
}).
Preload("Labels.Label").
Preload("Links").
Preload("Camera").
Preload("Lens").
Preload("Details").
Preload("Location").
Preload("Location.Place").
Where("maintained_at IS NULL OR maintained_at < ?", time.Now().Add(-1*time.Hour*24*7)).
Where("updated_at < ?", time.Now().Add(-1*time.Hour*36)).
Limit(limit).Offset(offset).Find(&entities).Error
return entities, err
}

8
internal/query/photo_test.go
View file

@ -40,7 +40,7 @@ func TestPhotoByUID(t *testing.T) {
func TestPreloadPhotoByUID(t *testing.T) {
t.Run("photo found", func(t *testing.T) {
result, err := PreloadPhotoByUID("pt9jtdre2lvl0y12")
result, err := PhotoPreloadByUID("pt9jtdre2lvl0y12")
if err != nil {
t.Fatal(err)
}
@ -48,14 +48,14 @@ func TestPreloadPhotoByUID(t *testing.T) {
})
t.Run("no photo found", func(t *testing.T) {
result, err := PreloadPhotoByUID("99999")
result, err := PhotoPreloadByUID("99999")
assert.Error(t, err, "record not found")
t.Log(result)
})
}
func TestMissingPhotos(t *testing.T) {
r, err := MissingPhotos(15, 0)
r, err := PhotosMissing(15, 0)
if err != nil {
t.Fatal(err)
}
@ -63,7 +63,7 @@ func TestMissingPhotos(t *testing.T) {
}
func TestResetPhotosQuality(t *testing.T) {
err := ResetPhotosQuality()
err := ResetPhotoQuality()
if err != nil {
t.Fatal(err)
}

42
internal/workers/groom.go
View file

@ -1,42 +0,0 @@
package workers
import (
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/mutex"
)
// Groom represents a groom worker.
type Groom struct {
conf *config.Config
}
// NewGroom returns a new groom worker.
func NewGroom(conf *config.Config) *Groom {
return &Groom{conf: conf}
}
// logError logs an error message if err is not nil.
func (worker *Groom) logError(err error) {
if err != nil {
log.Errorf("groom: %s", err.Error())
}
}
// logWarn logs a warning message if err is not nil.
func (worker *Groom) logWarn(err error) {
if err != nil {
log.Warnf("groom: %s", err.Error())
}
}
// Start starts the groom worker.
func (worker *Groom) Start() (err error) {
if err := mutex.GroomWorker.Start(); err != nil {
worker.logWarn(err)
return err
}
defer mutex.GroomWorker.Stop()
return err
}

110
internal/workers/prism.go Normal file
View file

@ -0,0 +1,110 @@
package workers
import (
"errors"
"runtime"
"time"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/query"
)
// Prism represents a background maintenance worker.
type Prism struct {
conf *config.Config
}
// NewPrism returns a new background maintenance worker.
func NewPrism(conf *config.Config) *Prism {
return &Prism{conf: conf}
}
// logError logs an error message if err is not nil.
func (worker *Prism) logError(err error) {
if err != nil {
log.Errorf("prism: %s", err.Error())
}
}
// logWarn logs a warning message if err is not nil.
func (worker *Prism) logWarn(err error) {
if err != nil {
log.Warnf("prism: %s", err.Error())
}
}
// originalsPath returns the original media files path as string.
func (worker *Prism) originalsPath() string {
return worker.conf.OriginalsPath()
}
// Start starts the prism worker.
func (worker *Prism) Start() (err error) {
if err := mutex.PrismWorker.Start(); err != nil {
worker.logWarn(err)
return err
}
defer func() {
mutex.PrismWorker.Stop()
if err := recover(); err != nil {
log.Errorf("prism: %s [panic]", err)
}
}()
done := make(map[string]bool)
limit := 50
offset := 0
for {
photos, err := query.PhotosMaintenance(limit, offset)
if err != nil {
return err
}
if len(photos) == 0 {
break
} else if offset == 0 {
log.Infof("prism: starting photo maintenance")
}
for _, photo := range photos {
if mutex.PrismWorker.Canceled() {
return errors.New("prism: maintenance canceled")
}
if done[photo.PhotoUID] {
continue
}
done[photo.PhotoUID] = true
worker.logError(photo.Maintain())
}
if mutex.PrismWorker.Canceled() {
return errors.New("prism: maintenance canceled")
}
offset += limit
time.Sleep(100 * time.Millisecond)
}
if len(done) > 0 {
log.Infof("prism: maintained %d photos", len(done))
}
worker.logError(query.ResetPhotoQuality())
worker.logError(entity.UpdatePhotoCounts())
runtime.GC()
return nil
}

14
internal/workers/groom_test.go → internal/workers/prism_test.go
View file

@ -8,14 +8,14 @@ import (
"github.com/stretchr/testify/assert"
)
func TestGroom_Start(t *testing.T) {
func TestPrism_Start(t *testing.T) {
conf := config.TestConfig()
worker := NewGroom(conf)
worker := NewPrism(conf)
assert.IsType(t, &Groom{}, worker)
assert.IsType(t, &Prism{}, worker)
if err := mutex.GroomWorker.Start(); err != nil {
if err := mutex.PrismWorker.Start(); err != nil {
t.Fatal(err)
}
@ -23,7 +23,11 @@ func TestGroom_Start(t *testing.T) {
t.Fatal("error expected")
}
mutex.GroomWorker.Stop()
mutex.PrismWorker.Stop()
if err := worker.Start(); err != nil {
t.Fatal(err)
}
if err := worker.Start(); err != nil {
t.Fatal(err)

10
internal/workers/workers.go
View file

@ -21,12 +21,12 @@ func Start(conf *config.Config) {
case <-stop:
log.Info("shutting down workers")
ticker.Stop()
mutex.GroomWorker.Cancel()
mutex.PrismWorker.Cancel()
mutex.ShareWorker.Cancel()
mutex.SyncWorker.Cancel()
return
case <-ticker.C:
StartGroom(conf)
StartPrism(conf)
StartShare(conf)
StartSync(conf)
}
@ -39,11 +39,11 @@ func Stop() {
stop <- true
}
// StartGroom runs the groom worker once.
func StartGroom(conf *config.Config) {
// StartPrism runs the prism worker once.
func StartPrism(conf *config.Config) {
if !mutex.WorkersBusy() {
go func() {
worker := NewGroom(conf)
worker := NewPrism(conf)
if err := worker.Start(); err != nil {
log.Error(err)
}