luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Backend: Add index update mutex to reduce database locking

Browse source
This commit is contained in:
Michael Mayer 2021-10-06 11:50:48 +02:00
parent 444c94bf9e
commit 07ae9b83f4
14 changed files with 96 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
internal/entity/counts.go
View file

@ -6,8 +6,9 @@ import (
"time"
"github.com/dustin/go-humanize/english"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/mutex"
)
type LabelPhotoCount struct {
@ -48,6 +49,9 @@ func LabelCounts() LabelPhotoCounts {
// UpdatePlacesCounts updates the places photo counts.
func UpdatePlacesCounts() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
// Update places.
@ -69,6 +73,9 @@ func UpdatePlacesCounts() (err error) {
// UpdateSubjectCounts updates the subject file counts.
func UpdateSubjectCounts() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
@ -121,6 +128,9 @@ func UpdateSubjectCounts() (err error) {
// UpdateLabelCounts updates the label photo counts.
func UpdateLabelCounts() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
if IsDialect(MySQL) {

1
internal/mutex/mutex.go
View file

@ -6,6 +6,7 @@ import (
var (
Db = sync.Mutex{}
IndexUpdate = sync.Mutex{}
MainWorker = Busy{}
SyncWorker = Busy{}
ShareWorker = Busy{}

3
internal/photoprism/albums.go
View file

@ -4,10 +4,9 @@ import (
"path/filepath"
"regexp"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/txt"
)

3
internal/photoprism/cleanup.go
View file

@ -10,10 +10,9 @@ import (
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/pkg/fastwalk"

1
internal/photoprism/faces_audit.go
View file

@ -2,6 +2,7 @@ package photoprism
import (
"github.com/dustin/go-humanize/english"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/query"

2
internal/photoprism/purge.go
View file

@ -246,7 +246,7 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
}
// Set photo quality scores to -1 if files are missing.
if err := query.ResetPhotoQuality(); err != nil {
if err := query.FlagHiddenPhotos(); err != nil {
return purgedFiles, purgedPhotos, err
}

7
internal/query/albums.go
View file

@ -5,6 +5,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/form"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/search"
"github.com/photoprism/photoprism/pkg/txt"
)
@ -71,6 +72,9 @@ func AlbumCoverByUID(uid string) (file entity.File, err error) {
// UpdateAlbumDates updates album year, month and day based on indexed photo metadata.
func UpdateAlbumDates() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
switch DbDialect() {
case MySQL:
return UnscopedDb().Exec(`UPDATE albums
@ -87,6 +91,9 @@ func UpdateAlbumDates() error {
// UpdateMissingAlbumEntries sets a flag for missing photo album entries.
func UpdateMissingAlbumEntries() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
switch DbDialect() {
default:
return UnscopedDb().Exec(`UPDATE photos_albums SET missing = 1 WHERE photo_uid IN

16
internal/query/covers.go
View file

@ -9,10 +9,14 @@ import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
)
// UpdateAlbumDefaultCovers updates default album cover thumbs.
func UpdateAlbumDefaultCovers() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
@ -58,6 +62,9 @@ func UpdateAlbumDefaultCovers() (err error) {
// UpdateAlbumFolderCovers updates folder album cover thumbs.
func UpdateAlbumFolderCovers() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
@ -103,6 +110,9 @@ func UpdateAlbumFolderCovers() (err error) {
// UpdateAlbumMonthCovers updates month album cover thumbs.
func UpdateAlbumMonthCovers() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
@ -168,6 +178,9 @@ func UpdateAlbumCovers() (err error) {
// UpdateLabelCovers updates label cover thumbs.
func UpdateLabelCovers() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB
@ -232,6 +245,9 @@ func UpdateLabelCovers() (err error) {
// UpdateSubjectCovers updates subject cover thumbs.
func UpdateSubjectCovers() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var res *gorm.DB

4
internal/query/faces.go
View file

@ -5,6 +5,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/pkg/txt"
)
@ -212,6 +213,9 @@ func ResolveFaceCollisions() (conflicts, resolved int, err error) {
// RemovePeopleAndFaces permanently removes all people, faces, and face markers.
func RemovePeopleAndFaces() (err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
// Delete people.
if err = UnscopedDb().Delete(entity.Subject{}, "subj_type = ?", entity.SubjPerson).Error; err != nil {
return err

4
internal/query/folders.go
View file

@ -4,6 +4,7 @@ import (
"path/filepath"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/pkg/fs"
)
@ -63,6 +64,9 @@ func AlbumFolders(threshold int) (folders entity.Folders, err error) {
// UpdateFolderDates updates folder year, month and day based on indexed photo metadata.
func UpdateFolderDates() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
switch DbDialect() {
case MySQL:
return UnscopedDb().Exec(`UPDATE folders

39
internal/query/photo.go
View file

@ -4,9 +4,10 @@ import (
"time"
"github.com/dustin/go-humanize/english"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
)
// PhotoByID returns a Photo based on the ID.
@ -83,21 +84,6 @@ func PhotosMissing(limit int, offset int) (entities entity.Photos, err error) {
return entities, err
}
// ResetPhotoQuality sets photo quality scores to -1 if files are missing.
func ResetPhotoQuality() error {
start := time.Now()
res := Db().Table("photos").
Where("id NOT IN (SELECT photo_id FROM files WHERE file_primary = 1 AND file_missing = 0 AND file_error = '' AND deleted_at IS NULL)").
Update("photo_quality", -1)
if res.RowsAffected > 0 {
log.Infof("index: flagged %s as hidden [%s]", english.Plural(int(res.RowsAffected), "broken photo", "broken photos"), time.Since(start))
}
return res.Error
}
// PhotosCheck returns photos selected for maintenance.
func PhotosCheck(limit, offset int, delay time.Duration) (entities entity.Photos, err error) {
err = Db().
@ -132,6 +118,9 @@ func OrphanPhotos() (photos entity.Photos, err error) {
// FixPrimaries tries to set a primary file for photos that have none.
func FixPrimaries() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
var photos entity.Photos
@ -170,3 +159,21 @@ func FixPrimaries() error {
return nil
}
// FlagHiddenPhotos sets the quality score of photos without valid primary file to -1.
func FlagHiddenPhotos() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
start := time.Now()
res := Db().Table("photos").
Where("id NOT IN (SELECT photo_id FROM files WHERE file_primary = 1 AND file_missing = 0 AND file_error = '' AND deleted_at IS NULL)").
Update("photo_quality", -1)
if res.RowsAffected > 0 {
log.Infof("index: flagged %s as hidden [%s]", english.Plural(int(res.RowsAffected), "broken photo", "broken photos"), time.Since(start))
}
return res.Error
}

18
internal/query/photo_test.go
View file

@ -4,9 +4,9 @@ import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/stretchr/testify/assert"
"github.com/photoprism/photoprism/internal/entity"
)
func TestPhotoByID(t *testing.T) {
@ -67,13 +67,6 @@ func TestMissingPhotos(t *testing.T) {
assert.LessOrEqual(t, 1, len(result))
}
func TestResetPhotosQuality(t *testing.T) {
// Set photo quality scores to -1 if files are missing.
if err := ResetPhotoQuality(); err != nil {
t.Fatal(err)
}
}
func TestPhotosCheck(t *testing.T) {
result, err := PhotosCheck(10, 0, time.Second)
@ -103,3 +96,10 @@ func TestFixPrimaries(t *testing.T) {
}
})
}
func TestFlagHiddenPhotos(t *testing.T) {
// Set photo quality scores to -1 if files are missing.
if err := FlagHiddenPhotos(); err != nil {
t.Fatal(err)
}
}

16
internal/query/purge.go
View file

@ -4,6 +4,7 @@ import (
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
)
// PurgeOrphans removes orphan database entries.
@ -43,6 +44,9 @@ func PurgeOrphans() error {
// PurgeOrphanFiles removes files without a photo from the index.
func PurgeOrphanFiles() (count int, err error) {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
files, err := OrphanFiles()
if err != nil {
@ -62,6 +66,9 @@ func PurgeOrphanFiles() (count int, err error) {
// PurgeOrphanDuplicates deletes all files from the duplicates table that don't exist in the files table.
func PurgeOrphanDuplicates() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
return UnscopedDb().Delete(
entity.Duplicate{},
"file_hash NOT IN (SELECT file_hash FROM files WHERE file_missing = 0 AND deleted_at IS NULL)").Error
@ -69,6 +76,9 @@ func PurgeOrphanDuplicates() error {
// PurgeOrphanCountries removes countries without any photos.
func PurgeOrphanCountries() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
entity.FlushCountryCache()
switch DbDialect() {
default:
@ -78,6 +88,9 @@ func PurgeOrphanCountries() error {
// PurgeOrphanCameras removes cameras without any photos.
func PurgeOrphanCameras() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
entity.FlushCameraCache()
switch DbDialect() {
default:
@ -87,6 +100,9 @@ func PurgeOrphanCameras() error {
// PurgeOrphanLenses removes cameras without any photos.
func PurgeOrphanLenses() error {
mutex.IndexUpdate.Lock()
defer mutex.IndexUpdate.Unlock()
entity.FlushLensCache()
switch DbDialect() {
default:

2
internal/workers/meta.go
View file

@ -116,7 +116,7 @@ func (m *Meta) Start(delay time.Duration) (err error) {
}
// Set photo quality scores to -1 if files are missing.
if err := query.ResetPhotoQuality(); err != nil {
if err := query.FlagHiddenPhotos(); err != nil {
log.Warnf("metadata: %s (reset quality)", err.Error())
}