luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Indexer: Refactor photo merge #616

Browse source
This commit is contained in:
Michael Mayer 2020-12-12 22:02:14 +01:00
parent 618525969b
commit ce471de921
12 changed files with 526 additions and 454 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frontend/src/dialog/photo/info.vue
View file

@ -113,7 +113,7 @@
</tr>
<tr>
<td>
<translate>Single</translate>
<translate>Unstacked</translate>
</td>
<td>
<v-switch

BIN
frontend/src/locales/de.mo
View file

Binary file not shown.

340
frontend/src/locales/de.po
View file

File diff suppressed because it is too large Load diff

2
frontend/src/locales/translations.json
View file

File diff suppressed because one or more lines are too long

377
frontend/src/locales/translations.pot
View file

File diff suppressed because it is too large Load diff

4
internal/entity/photo.go
View file

@ -50,8 +50,8 @@ type Photo struct {
TitleSrc string `gorm:"type:VARBINARY(8);" json:"TitleSrc" yaml:"TitleSrc,omitempty"`
PhotoDescription string `gorm:"type:TEXT;" json:"Description" yaml:"Description,omitempty"`
DescriptionSrc string `gorm:"type:VARBINARY(8);" json:"DescriptionSrc" yaml:"DescriptionSrc,omitempty"`
PhotoPath string `gorm:"type:VARBINARY(768);index;" json:"Path" yaml:"-"`
PhotoName string `gorm:"type:VARBINARY(255);" json:"Name" yaml:"-"`
PhotoPath string `gorm:"type:VARBINARY(768);index:idx_photos_path_name;" json:"Path" yaml:"-"`
PhotoName string `gorm:"type:VARBINARY(255);index:idx_photos_path_name;" json:"Name" yaml:"-"`
OriginalName string `gorm:"type:VARBINARY(768);" json:"OriginalName" yaml:"OriginalName,omitempty"`
PhotoFavorite bool `json:"Favorite" yaml:"Favorite,omitempty"`
PhotoSingle bool `json:"Single" yaml:"Single,omitempty"`

5
internal/entity/photo_fixtures.go
View file

@ -962,6 +962,7 @@ var PhotoFixtures = PhotoMap{
},
"Photo19": {
ID: 1000019,
UUID: "123e4567-e89b-12d3-a456-426614174000",
PhotoUID: "pt9jtxrexxvl0yh0",
TakenAt: time.Date(2008, 1, 1, 0, 0, 0, 0, time.UTC),
TakenAtLocal: time.Time{},
@ -969,7 +970,7 @@ var PhotoFixtures = PhotoMap{
PhotoTitle: "",
TitleSrc: "",
PhotoPath: "1990/04",
PhotoName: "Photo03",
PhotoName: "Photo19",
PhotoQuality: -1,
PhotoResolution: 2,
PhotoFavorite: false,
@ -1017,7 +1018,7 @@ var PhotoFixtures = PhotoMap{
PhotoTitle: "",
TitleSrc: "",
PhotoPath: "1990/04",
PhotoName: "Photo03",
PhotoName: "Photo20",
PhotoQuality: 1,
PhotoResolution: 2,
PhotoFavorite: false,

114
internal/entity/photo_merge.go Normal file
View file

@ -0,0 +1,114 @@
package entity
import (
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/pkg/rnd"
)
// ResolvePrimary ensures there is only one primary file for a photo.
func (m *Photo) ResolvePrimary() error {
var file File
if err := Db().Where("file_primary = 1 AND photo_id = ?", m.ID).First(&file).Error; err == nil && file.ID > 0 {
return file.ResolvePrimary()
}
return nil
}
// Identical returns identical photos that can be merged.
func (m *Photo) Identical(includeMeta, includeUuid bool) (identical Photos, err error) {
if m.PhotoSingle || m.PhotoName == "" {
return identical, nil
}
switch {
case includeMeta && includeUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
if err := Db().
Where("(taken_at = ? AND taken_src = 'meta' AND photo_single = 0 AND cell_id = ? AND camera_serial = ? AND camera_id = ?) "+
"OR (uuid = ? AND photo_single = 0)"+
"OR (photo_path = ? AND photo_name = ?)",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID, m.PhotoPath, m.PhotoName).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case includeMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
if err := Db().
Where("(taken_at = ? AND taken_src = 'meta' AND photo_single = 0 AND cell_id = ? AND camera_serial = ? AND camera_id = ?) "+
"OR (photo_path = ? AND photo_name = ?)",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.PhotoPath, m.PhotoName).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case includeUuid && rnd.IsUUID(m.UUID):
if err := Db().
Where("(uuid = ? AND photo_single = 0) OR (photo_path = ? AND photo_name = ?)",
m.UUID, m.PhotoPath, m.PhotoName).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
default:
if err := Db().
Where("photo_path = ? AND photo_name = ?", m.PhotoPath, m.PhotoName).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
}
return identical, nil
}
// Merge photo with identical ones.
func (m *Photo) Merge(mergeMeta, mergeUuid bool) (original Photo, merged Photos, err error) {
identical, err := m.Identical(mergeMeta, mergeUuid)
if len(identical) < 2 || err != nil {
return Photo{}, merged, err
}
logResult := func(res *gorm.DB) {
if res.Error != nil {
log.Errorf("merge: %s", res.Error.Error())
err = res.Error
}
}
for i, merge := range identical {
if i == 0 {
original = merge
log.Debugf("photo: merging id %d with %d identical", original.ID, len(identical)-1)
continue
}
deleted := Timestamp()
logResult(UnscopedDb().Exec("UPDATE `files` SET photo_id = ?, photo_uid = ?, file_primary = 0 WHERE photo_id = ?", original.ID, original.PhotoUID, merge.ID))
logResult(UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", Timestamp(), merge.ID))
switch DbDialect() {
case MySQL:
logResult(UnscopedDb().Exec("UPDATE IGNORE `photos_keywords` SET `photo_id` = ? WHERE photo_id = ?", original.ID, merge.ID))
logResult(UnscopedDb().Exec("UPDATE IGNORE `photos_labels` SET `photo_id` = ? WHERE photo_id = ?", original.ID, merge.ID))
logResult(UnscopedDb().Exec("UPDATE IGNORE `photos_albums` SET `photo_uid` = ? WHERE photo_uid = ?", original.PhotoUID, merge.PhotoUID))
case SQLite:
logResult(UnscopedDb().Exec("UPDATE OR IGNORE `photos_keywords` SET `photo_id` = ? WHERE photo_id = ?", original.ID, merge.ID))
logResult(UnscopedDb().Exec("UPDATE OR IGNORE `photos_labels` SET `photo_id` = ? WHERE photo_id = ?", original.ID, merge.ID))
logResult(UnscopedDb().Exec("UPDATE OR IGNORE `photos_albums` SET `photo_uid` = ? WHERE photo_uid = ?", original.PhotoUID, merge.PhotoUID))
default:
log.Warnf("merge: unknown sql dialect")
}
merge.DeletedAt = &deleted
merge.PhotoQuality = -1
merged = append(merged, merge)
}
if original.ID != m.ID {
deleted := Timestamp()
m.DeletedAt = &deleted
m.PhotoQuality = -1
}
return original, merged, err
}

17
internal/entity/photo_merge_test.go Normal file
View file

@ -0,0 +1,17 @@
package entity
import "testing"
func TestPhoto_IdenticalIdentical(t *testing.T) {
t.Run("success", func(t *testing.T) {
photo := PhotoFixtures.Get("Photo19")
result, err := photo.Identical(true, true)
if err != nil {
t.Fatal(err)
}
t.Logf("result: %#v", result)
})
}

105
internal/entity/photo_optimize.go
View file

@ -2,13 +2,10 @@ package entity
import (
"errors"
"fmt"
"reflect"
"strings"
"time"
"github.com/photoprism/photoprism/pkg/rnd"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/txt"
@ -106,8 +103,12 @@ func (m *Photo) Optimize(mergeMeta, mergeUuid bool) (updated bool, merged Photos
m.UpdateLocation()
}
if merged, err = m.Merge(mergeMeta, mergeUuid, true); err != nil {
log.Errorf("photo: %s (merge)", err)
if original, photos, err := m.Merge(mergeMeta, mergeUuid); err != nil {
return updated, merged, err
} else if len(photos) > 0 && original.ID == m.ID {
merged = photos
} else if len(photos) > 0 && original.ID != m.ID {
return false, photos, nil
}
m.EstimatePlace()
@ -141,97 +142,3 @@ func (m *Photo) Optimize(mergeMeta, mergeUuid bool) (updated bool, merged Photos
return true, merged, m.Save()
}
// ResolvePrimary ensures there is only one primary file for a photo.
func (m *Photo) ResolvePrimary() error {
var file File
if err := Db().Where("file_primary = 1 AND photo_id = ?", m.ID).First(&file).Error; err == nil && file.ID > 0 {
return file.ResolvePrimary()
}
return nil
}
// Identical returns identical photos that can be merged.
func (m *Photo) Identical(findMeta, findUuid, findOlder bool) (identical Photos, err error) {
if !findMeta && !findUuid || m.PhotoSingle || m.DeletedAt != nil {
return identical, nil
}
op := "<>"
if findOlder {
op = "<"
}
switch {
case findMeta && findUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
if err := Db().
Where("(taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?) OR (uuid <> '' AND uuid = ?)",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID).
Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL AND edited_at IS NULL", op), m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case findMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
if err := Db().
Where("taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID).
Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL AND edited_at IS NULL", op), m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case findUuid && rnd.IsUUID(m.UUID):
if err := Db().
Where(fmt.Sprintf("uuid = ? AND id %s ? AND photo_single = 0 AND deleted_at IS NULL AND edited_at IS NULL", op), m.UUID, m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
}
return identical, nil
}
// Merge photo with identical ones.
func (m *Photo) Merge(mergeMeta, mergeUuid, mergeOlder bool) (merged Photos, err error) {
merged, err = m.Identical(mergeMeta, mergeUuid, mergeOlder)
if len(merged) == 0 || err != nil {
return merged, err
}
for _, photo := range merged {
if photo.DeletedAt != nil || photo.ID == m.ID {
continue
}
if err := UnscopedDb().Exec("UPDATE `files` SET photo_id = ?, photo_uid = ?, file_primary = 0 WHERE photo_id = ?", m.ID, m.PhotoUID, photo.ID).Error; err != nil {
return merged, err
}
switch DbDialect() {
case MySQL:
UnscopedDb().Exec("UPDATE IGNORE `photos_keywords` SET `photo_id` = ? WHERE (photo_id = ?)", m.ID, photo.ID)
UnscopedDb().Exec("UPDATE IGNORE `photos_labels` SET `photo_id` = ? WHERE (photo_id = ?)", m.ID, photo.ID)
UnscopedDb().Exec("UPDATE IGNORE `photos_albums` SET `photo_uid` = ? WHERE (photo_uid = ?)", m.PhotoUID, photo.PhotoUID)
case SQLite:
UnscopedDb().Exec("UPDATE OR IGNORE `photos_keywords` SET `photo_id` = ? WHERE (photo_id = ?)", m.ID, photo.ID)
UnscopedDb().Exec("UPDATE OR IGNORE `photos_labels` SET `photo_id` = ? WHERE (photo_id = ?)", m.ID, photo.ID)
UnscopedDb().Exec("UPDATE OR IGNORE `photos_albums` SET `photo_uid` = ? WHERE (photo_uid = ?)", m.PhotoUID, photo.PhotoUID)
default:
log.Warnf("photo: unknown SQL dialect (merge)")
}
deleted := Timestamp()
if err := UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", Timestamp(), photo.ID).Error; err != nil {
return merged, err
}
photo.DeletedAt = &deleted
photo.PhotoQuality = -1
}
return merged, err
}

12
internal/photoprism/index_mediafile.go
View file

@ -96,7 +96,6 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
fileRoot, fileBase, filePath, fileName := m.PathNameInfo(stripSequence)
fullBase := m.BasePrefix(false)
logName := txt.Quote(fileName)
fileSize, modTime, err := m.Stat()
@ -821,11 +820,16 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
if o.Single || photo.PhotoSingle {
// Do nothing.
} else if merged, err := photo.Merge(Config().Settings().StackMeta(), Config().Settings().StackUUID(), true); err != nil {
} else if original, merged, err := photo.Merge(Config().Settings().StackMeta(), Config().Settings().StackUUID()); err != nil {
log.Errorf("index: %s in %s (merge)", err.Error(), logName)
} else if len(merged) > 0 {
log.Infof("index: merged %s with existing photo", logName)
} else if len(merged) == 1 && original.ID == photo.ID {
log.Infof("index: merged one existing photo with %s", logName)
} else if len(merged) > 1 && original.ID == photo.ID {
log.Infof("index: merged %d existing photos with %s", len(merged), logName)
} else if len(merged) > 0 && original.ID != photo.ID {
log.Infof("index: merged %s with existing photo id %d", logName, original.ID)
result.Status = IndexStacked
return result
}
if file.FilePrimary && Config().SidecarYaml() {

2
internal/workers/meta.go
View file

@ -86,7 +86,7 @@ func (worker *Meta) Start(delay time.Duration) (err error) {
}
for _, m := range merged {
log.Infof("metadata: stacked %s", m.PhotoUID)
log.Infof("metadata: merged %s", m.PhotoUID)
done[m.PhotoUID] = true
}
}