luisgulo/photoprism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Index: Refactor optimization worker logs and queries #3124 #3742

Browse Source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2023-10-15 11:02:15 +02:00
parent 05e29170bf
commit 2fed4b549c
5 changed files with 35 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/entity/entity_counts.go
View File

@ -192,7 +192,7 @@ func UpdateCounts() (err error) {
return nil
}
return err
return fmt.Errorf("%s while updating places counts", err)
}
if err = UpdateSubjectCounts(); err != nil {
@ -202,11 +202,11 @@ func UpdateCounts() (err error) {
return nil
}
return err
return fmt.Errorf("%s while updating subject counts", err)
}
if err = UpdateLabelCounts(); err != nil {
return err
return fmt.Errorf("%s while updating label counts", err)
}
/* TODO: Slow with many photos due to missing index.

6
internal/query/covers.go
View File

@ -300,17 +300,17 @@ func UpdateCovers() (err error) {
// Update Albums.
if err = UpdateAlbumCovers(); err != nil {
return err
return fmt.Errorf("%s while updating album covers", err)
}
// Update Labels.
if err = UpdateLabelCovers(); err != nil {
return err
return fmt.Errorf("%s while updating label covers", err)
}
// Update Subjects.
if err = UpdateSubjectCovers(); err != nil {
return err
return fmt.Errorf("%s while updating subject covers", err)
}
return nil

12
internal/query/faces.go
View File

@ -150,7 +150,7 @@ func CountNewFaceMarkers(size, score int) (n int) {
}
// PurgeOrphanFaces removes unused faces from the index.
func PurgeOrphanFaces(faceIds []string, ignored bool) (removed int64, err error) {
func PurgeOrphanFaces(faceIds []string, ignored bool) (affected int, err error) {
// Remove invalid face IDs in batches to be compatible with SQLite.
batchSize := BatchSize()
@ -173,14 +173,16 @@ func PurgeOrphanFaces(faceIds []string, ignored bool) (removed int64, err error)
stmt = stmt.Where("face_kind <= 1")
}
if res := stmt.Delete(&entity.Face{}); res.Error != nil {
return removed, fmt.Errorf("faces: %s while purging orphans", res.Error)
if result := stmt.Delete(&entity.Face{}); result.Error != nil {
return affected, fmt.Errorf("faces: %s while purging orphan faces", result.Error)
} else if result.RowsAffected > 0 {
affected += int(result.RowsAffected)
} else {
removed += res.RowsAffected
affected += len(ids)
}
}
return removed, nil
return affected, nil
}
// MergeFaces returns a new face that replaces multiple others.

26
internal/query/photo.go
View File

@ -1,6 +1,7 @@
package query
import (
"fmt"
"time"
"github.com/dustin/go-humanize/english"
@ -182,8 +183,8 @@ func FlagHiddenPhotos() (err error) {
// IDs of hidden photos.
var hidden []uint
// Number of updated photos.
n := 0
// Number of updated records.
affected := 0
// Find and flag hidden photos.
if err = Db().Table(entity.Photo{}.TableName()).
@ -209,22 +210,23 @@ func FlagHiddenPhotos() (err error) {
ids := hidden[i:j]
// Set photos.photo_quality = -1.
if err = UnscopedDb().Table(entity.Photo{}.TableName()).
if result := UnscopedDb().Table(entity.Photo{}.TableName()).
Where("id IN (?) AND photo_quality > -1", ids).
UpdateColumn("photo_quality", -1).Error; err != nil {
// Failed.
log.Warnf("index: failed to flag %d pictures as hidden", len(ids))
return err
UpdateColumn("photo_quality", -1); result.Error != nil {
// Failed to flag all hidden photos.
log.Warnf("index: failed to flag %d photos as hidden", len(hidden)-affected)
return fmt.Errorf("%s while flagging hidden photos", result.Error)
} else if result.RowsAffected > 0 {
affected += int(result.RowsAffected)
} else {
// Success.
n += len(ids)
affected += len(ids)
}
}
}
// Log number of updated photos, if any.
if n > 0 {
log.Infof("index: flagged %s as hidden [%s]", english.Plural(int(n), "photo", "photos"), time.Since(start))
// Log number of affected rows, if any.
if affected > 0 {
log.Infof("index: flagged %s as hidden [%s]", english.Plural(affected, "photo", "photos"), time.Since(start))
}
return nil

16
internal/workers/meta.go
View File

@ -16,7 +16,7 @@ import (
"github.com/photoprism/photoprism/internal/query"
)
// Meta represents a background metadata optimization worker.
// Meta represents a background index and metadata optimization worker.
type Meta struct {
conf *config.Config
lastRun time.Time
@ -83,7 +83,7 @@ func (w *Meta) Start(delay, interval time.Duration, force bool) (err error) {
for _, photo := range photos {
if mutex.MetaWorker.Canceled() {
return errors.New("index: metadata update canceled")
return errors.New("index: metadata optimization canceled")
}
if done[photo.PhotoUID] {
@ -95,7 +95,7 @@ func (w *Meta) Start(delay, interval time.Duration, force bool) (err error) {
updated, merged, err := photo.Optimize(settings.StackMeta(), settings.StackUUID(), settings.Features.Estimates, force)
if err != nil {
log.Errorf("index: %s (metadata update)", err)
log.Errorf("index: %s in optimization worker", err)
} else if updated {
optimized++
log.Debugf("index: updated photo %s", photo.String())
@ -108,7 +108,7 @@ func (w *Meta) Start(delay, interval time.Duration, force bool) (err error) {
}
if mutex.MetaWorker.Canceled() {
return errors.New("index: metadata update canceled")
return errors.New("index: optimization canceled")
}
offset += limit
@ -123,24 +123,24 @@ func (w *Meta) Start(delay, interval time.Duration, force bool) (err error) {
if updateIndex {
// Set photo quality scores to -1 if files are missing.
if err = query.FlagHiddenPhotos(); err != nil {
log.Warnf("index: %s (reset quality)", err.Error())
log.Warnf("index: %s in optimization worker", err)
}
// Run moments worker.
if moments := photoprism.NewMoments(w.conf); moments == nil {
log.Errorf("index: failed updating moments")
} else if err = moments.Start(); err != nil {
log.Warn(err)
log.Warnf("moments: %s in optimization worker", err)
}
// Update precalculated photo and file counts.
if err = entity.UpdateCounts(); err != nil {
log.Warnf("index: %s (update counts)", err.Error())
log.Warnf("index: %s in optimization worker", err)
}
// Update album, subject, and label cover thumbs.
if err = query.UpdateCovers(); err != nil {
log.Warnf("index: %s (update covers)", err)
log.Warnf("index: %s in optimization worker", err)
}
}