diff --git a/internal/entity/face.go b/internal/entity/face.go index d92b01b68..ea42c5175 100644 --- a/internal/entity/face.go +++ b/internal/entity/face.go @@ -122,3 +122,38 @@ func (m *Face) Update(attr string, value interface{}) error { func (m *Face) Updates(values interface{}) error { return UnscopedDb().Model(m).Updates(values).Error } + +// FirstOrCreateFace returns the existing entity, inserts a new entity or nil in case of errors. +func FirstOrCreateFace(m *Face) *Face { + result := Face{} + + if err := UnscopedDb().Where("id = ?", m.ID).First(&result).Error; err == nil { + return &result + } else if createErr := m.Create(); createErr == nil { + return m + } else if err := UnscopedDb().Where("id = ?", m.ID).First(&result).Error; err == nil { + return &result + } else { + log.Errorf("face: %s (find or create %s)", createErr, m.ID) + } + + return nil +} + +// FindFace returns an existing entity if exists. +func FindFace(id string) *Face { + if id == "" { + return nil + } + + result := Face{} + + db := Db() + db = db.Where("id = ?", id) + + if err := db.First(&result).Error; err != nil { + return nil + } + + return &result +} diff --git a/internal/entity/marker.go b/internal/entity/marker.go index 5502eea26..3be13714c 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -31,6 +31,7 @@ type Marker struct { SubjectSrc string `gorm:"type:VARBINARY(8);default:'';" json:"SubjectSrc" yaml:"SubjectSrc,omitempty"` Subject *Subject `gorm:"foreignkey:SubjectUID;association_foreignkey:SubjectUID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Subject,omitempty" yaml:"-"` FaceID string `gorm:"type:VARBINARY(42);index;" json:"FaceID" yaml:"FaceID,omitempty"` + Face *Face `gorm:"foreignkey:FaceID;association_foreignkey:ID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"-" yaml:"-"` EmbeddingsJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"EmbeddingsJSON,omitempty"` embeddings Embeddings `gorm:"-"` LandmarksJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"LandmarksJSON,omitempty"` @@ -198,11 +199,7 @@ func (m *Marker) SyncSubject(updateRelated bool) error { // Create known face for subject? if m.FaceID != "" || m.SubjectSrc != SrcManual { // Do nothing. - } else if f := NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil { - return fmt.Errorf("failed adding known face for subject %s", m.SubjectUID) - } else if err := f.Create(); err != nil { - log.Debugf("marker: %s (add known face)", err) - } else { + } else if f := m.GetFace(); f != nil { m.FaceID = f.ID } @@ -221,7 +218,7 @@ func (m *Marker) SyncSubject(updateRelated bool) error { Updates(Values{"SubjectUID": m.SubjectUID, "SubjectSrc": SrcAuto}).Error; err != nil { return fmt.Errorf("%s (update related markers)", err) } else { - log.Infof("marker: matched %s", subj.SubjectName) + log.Debugf("marker: matched subject %s with face %s", subj.SubjectName, m.FaceID) } return nil @@ -258,7 +255,7 @@ func (m *Marker) Embeddings() Embeddings { return m.embeddings } -// GetSubject returns the matching subject entity, if possible. +// GetSubject returns a subject entity if possible. func (m *Marker) GetSubject() (subj *Subject) { if m.Subject != nil { return m.Subject @@ -283,6 +280,30 @@ func (m *Marker) GetSubject() (subj *Subject) { return m.Subject } +// GetFace returns a matching face entity if possible. +func (m *Marker) GetFace() (f *Face) { + if m.Face != nil { + return m.Face + } + + if m.FaceID == "" && m.SubjectSrc == SrcManual { + if f = NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil { + return nil + } else if f = FirstOrCreateFace(f); f == nil { + log.Debugf("marker: invalid face") + return nil + } + + m.FaceID = f.ID + + return f + } + + m.Face = FindFace(m.FaceID) + + return m.Face +} + // FindMarker returns an existing row if exists. func FindMarker(id uint) *Marker { result := Marker{} diff --git a/internal/entity/subject.go b/internal/entity/subject.go index d18f2964f..d1918a289 100644 --- a/internal/entity/subject.go +++ b/internal/entity/subject.go @@ -140,7 +140,7 @@ func (m *Subject) Updates(values interface{}) error { return UnscopedDb().Model(m).Updates(values).Error } -// FirstOrCreateSubject returns the existing subject, inserts a new subject or nil in case of errors. +// FirstOrCreateSubject returns the existing entity, inserts a new entity or nil in case of errors. func FirstOrCreateSubject(m *Subject) *Subject { result := Subject{} @@ -165,7 +165,7 @@ func FirstOrCreateSubject(m *Subject) *Subject { return nil } -// FindSubject returns an existing row if exists. +// FindSubject returns an existing entity if exists. func FindSubject(s string) *Subject { if s == "" { return nil diff --git a/internal/photoprism/faces.go b/internal/photoprism/faces.go index 3e5722c48..0a3fb9951 100644 --- a/internal/photoprism/faces.go +++ b/internal/photoprism/faces.go @@ -3,7 +3,6 @@ package photoprism import ( "fmt" "runtime/debug" - "time" "github.com/photoprism/photoprism/internal/face" @@ -151,7 +150,13 @@ func (w *Faces) Reset() (err error) { if err := query.ResetFaces(); err != nil { log.Errorf("faces: %s (reset)", err) } else { - log.Infof("faces: reset known faces") + log.Infof("faces: reset faces") + } + + if err := query.ResetSubjects(); err != nil { + log.Errorf("faces: %s (reset)", err) + } else { + log.Infof("faces: reset subjects") } return nil @@ -182,21 +187,30 @@ func (w *Faces) Start(opt FacesOptions) (err error) { defer mutex.MainWorker.Stop() // Skip clustering if index contains no new face markers and force option isn't set. - if n := query.CountNewFaceMarkers(); n < 1 && !opt.Force{ + if n := query.CountNewFaceMarkers(); n < 1 && !opt.Force { log.Debugf("faces: no new samples") - // Match (add and assign) subjects with markers, if possible. - if affected, err := query.MatchMarkersWithSubjects(); err != nil { + var updated int64 + + // Adds and reference known marker subjects. + if affected, err := query.AddMarkerSubjects(); err != nil { log.Errorf("faces: %s (match markers with subjects)", err) - } else if affected > 0 { - log.Infof("faces: matched %d markers with subjects", affected) + } else { + updated += affected } - // Match known faces with markers, if possible. - if matched, err := query.MatchKnownFaces(); err != nil { + // Match markers with known faces. + if affected, err := query.MatchFaceMarkers(); err != nil { return err - } else if matched > 0 { - log.Infof("faces: matched %d markers to faces", matched) + } else { + updated += affected + } + + // Log result. + if updated > 0 { + log.Infof("faces: %d markers updated", updated) + } else { + log.Debug("faces: no changes") } // Clean-up invalid marker data. @@ -250,6 +264,11 @@ func (w *Faces) Start(opt FacesOptions) (err error) { results[n-1] = append(results[n-1], embeddings[i]) } + if err := query.PurgeAnonymousFaces(); err != nil { + dbErrors++ + log.Errorf("faces: %s", err) + } + for _, embedding := range results { if f := entity.NewFace("", entity.SrcAuto, embedding); f == nil { dbErrors++ @@ -264,87 +283,9 @@ func (w *Faces) Start(opt FacesOptions) (err error) { } } - if err := query.PurgeAnonymousFaces(); err != nil { - dbErrors++ - log.Errorf("faces: %s", err) - } - - if faces, err := query.Faces(false); err != nil { + // Match existing markers and faces. + if recognized, unknown, err = w.MatchMarkers(); err != nil { return err - } else { - limit := 500 - offset := 0 - - for { - markers, err := query.Markers(limit, offset, entity.MarkerFace, true, false) - - if err != nil { - return err - } - - if len(markers) == 0 { - break - } - - for _, marker := range markers { - if mutex.MainWorker.Canceled() { - return fmt.Errorf("worker canceled") - } - - // Pointer to the matching face. - var f *entity.Face - - // Distance to the matching face. - var d float64 - - // Find the closest face match for marker. - for _, e := range marker.Embeddings() { - for i, match := range faces { - if dist := clusters.EuclideanDistance(e, match.Embedding()); f == nil || dist < d { - f = &faces[i] - d = dist - } - } - } - - // No match? - if f == nil { - continue - } - - // Too distant? - if d > (f.Radius + face.ClusterRadius) { - continue - } - - if updated, err := marker.SetFace(f); err != nil { - dbErrors++ - log.Errorf("faces: %s", err) - } else if updated { - recognized++ - } - - if marker.SubjectUID == "" { - unknown++ - } - } - - offset += limit - - time.Sleep(50 * time.Millisecond) - } - } - - // Match known faces with markers, if possible. - if m, err := query.MatchKnownFaces(); err != nil { - return err - } else { - recognized += m - } - - // Clean-up invalid marker data. - if err := query.TidyMarkers(); err != nil { - log.Errorf("faces: %s (tidy)", err) } // Log results. diff --git a/internal/photoprism/faces_match.go b/internal/photoprism/faces_match.go new file mode 100644 index 000000000..0fdd56ad9 --- /dev/null +++ b/internal/photoprism/faces_match.go @@ -0,0 +1,98 @@ +package photoprism + +import ( + "fmt" + "time" + + "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/face" + "github.com/photoprism/photoprism/internal/mutex" + "github.com/photoprism/photoprism/internal/query" + "github.com/photoprism/photoprism/pkg/clusters" +) + +// MatchMarkers matches existing markers and faces. +func (w *Faces) MatchMarkers() (recognized, unknown int64, err error) { + if w.Disabled() { + return 0, 0, nil + } + + if faces, err := query.Faces(false); err != nil { + return recognized, unknown, err + } else { + limit := 500 + offset := 0 + + for { + markers, err := query.Markers(limit, offset, entity.MarkerFace, true, false) + + if err != nil { + return recognized, unknown, err + } + + if len(markers) == 0 { + break + } + + for _, marker := range markers { + if mutex.MainWorker.Canceled() { + return recognized, unknown, fmt.Errorf("worker canceled") + } + + // Pointer to the matching face. + var f *entity.Face + + // Distance to the matching face. + var d float64 + + // Find the closest face match for marker. + for _, e := range marker.Embeddings() { + for i, match := range faces { + if dist := clusters.EuclideanDistance(e, match.Embedding()); f == nil || dist < d { + f = &faces[i] + d = dist + } + } + } + + // No match? + if f == nil { + continue + } + + // Too distant? + if d > (f.Radius + face.ClusterRadius) { + continue + } + + if updated, err := marker.SetFace(f); err != nil { + log.Errorf("faces: %s", err) + } else if updated { + recognized++ + } + + if marker.SubjectUID == "" { + unknown++ + } + } + + offset += limit + + time.Sleep(50 * time.Millisecond) + } + } + + // Update remaining markers based on current matches. + if m, err := query.MatchFaceMarkers(); err != nil { + return recognized, unknown, err + } else { + recognized += m + } + + // Reset invalid marker data. + if err := query.TidyMarkers(); err != nil { + return recognized, unknown, err + } + + return recognized, unknown, nil +} diff --git a/internal/photoprism/index.go b/internal/photoprism/index.go index dd9fd262c..33d73fae8 100644 --- a/internal/photoprism/index.go +++ b/internal/photoprism/index.go @@ -230,6 +230,15 @@ func (ind *Index) Start(opt IndexOptions) fs.Done { } if filesIndexed > 0 { + // Match existing faces if facial recognition is enabled. + if w := NewFaces(ind.conf); w.Disabled() { + log.Debugf("index: skipping facial recognition") + } else if recognized, unknown, err := w.MatchMarkers(); err != nil { + log.Errorf("index: %s", err) + } else if recognized > 0 || unknown > 0 { + log.Infof("faces: %d recognized, %d unknown", recognized, unknown) + } + event.Publish("index.updating", event.Data{ "step": "counts", }) @@ -246,7 +255,7 @@ func (ind *Index) Start(opt IndexOptions) fs.Done { return done } -// File indexes a single file and returns the result. +// FileName indexes a single file and returns the result. func (ind *Index) FileName(fileName string, o IndexOptions) (result IndexResult) { file, err := NewMediaFile(fileName) diff --git a/internal/query/faces.go b/internal/query/faces.go index 9708a4d75..0f5bcee00 100644 --- a/internal/query/faces.go +++ b/internal/query/faces.go @@ -4,7 +4,7 @@ import ( "github.com/photoprism/photoprism/internal/entity" ) -// Faces returns (known) faces from the index. +// Faces returns all (known) faces from the index. func Faces(knownOnly bool) (result entity.Faces, err error) { stmt := Db(). Where("face_src <> ?", entity.SrcDefault). @@ -19,8 +19,8 @@ func Faces(knownOnly bool) (result entity.Faces, err error) { return result, err } -// MatchKnownFaces matches known faces with markers, if possible. -func MatchKnownFaces() (affected int64, err error) { +// MatchFaceMarkers matches markers with known faces. +func MatchFaceMarkers() (affected int64, err error) { faces, err := Faces(true) if err != nil { @@ -46,7 +46,7 @@ func MatchKnownFaces() (affected int64, err error) { func PurgeAnonymousFaces() error { return UnscopedDb().Delete( entity.Face{}, - "id <> ? AND subject_uid = '' AND updated_at < ?", entity.UnknownFace.ID, entity.Yesterday()).Error + "face_src = ? AND subject_uid = ''", entity.SrcAuto).Error } // ResetFaces removes all face clusters from the index. @@ -60,11 +60,11 @@ func ResetFaces() error { func CountNewFaceMarkers() (n int) { var f entity.Face - if err := Db().Where("id <> ?", entity.UnknownFace.ID).Order("created_at DESC").Take(&f).Error; err != nil { + if err := Db().Where("face_src = ?", entity.SrcAuto).Order("created_at DESC").Take(&f).Error; err != nil { log.Debugf("faces: no existing clusters") } - q := Db().Model(&entity.Markers{}).Where("marker_type = ? AND embeddings_json <> ''", entity.MarkerFace) + q := Db().Model(&entity.Markers{}).Where("marker_type = ? AND marker_invalid = 0 AND embeddings_json <> ''", entity.MarkerFace) if !f.CreatedAt.IsZero() { q = q.Where("created_at > ?", f.CreatedAt) diff --git a/internal/query/faces_test.go b/internal/query/faces_test.go index 1b1946626..9f14538fd 100644 --- a/internal/query/faces_test.go +++ b/internal/query/faces_test.go @@ -22,7 +22,7 @@ func TestFaces(t *testing.T) { } } -func TestMatchKnownFaces(t *testing.T) { +func TestMatchFaceMarkers(t *testing.T) { const faceFixtureId = uint(6) if m, err := MarkerByID(faceFixtureId); err != nil { @@ -39,7 +39,7 @@ func TestMatchKnownFaces(t *testing.T) { t.Fatal(err) } - affected, err := MatchKnownFaces() + affected, err := MatchFaceMarkers() if err != nil { t.Fatal(err) diff --git a/internal/query/markers.go b/internal/query/markers.go index 155807e6f..57b36fe23 100644 --- a/internal/query/markers.go +++ b/internal/query/markers.go @@ -70,8 +70,8 @@ func Embeddings(single bool) (result entity.Embeddings, err error) { return result, nil } -// MatchMarkersWithSubjects matches (add and assign) subjects with markers, if possible. -func MatchMarkersWithSubjects() (affected int, err error) { +// AddMarkerSubjects adds and references known marker subjects. +func AddMarkerSubjects() (affected int64, err error) { var markers entity.Markers if err := Db(). diff --git a/internal/query/markers_test.go b/internal/query/markers_test.go index dae00ac4e..d09bc3461 100644 --- a/internal/query/markers_test.go +++ b/internal/query/markers_test.go @@ -59,11 +59,11 @@ func TestEmbeddings(t *testing.T) { } } -func TestMatchMarkersWithSubjects(t *testing.T) { - affected, err := MatchMarkersWithSubjects() +func TestAddMarkerSubjects(t *testing.T) { + affected, err := AddMarkerSubjects() assert.NoError(t, err) - assert.GreaterOrEqual(t, affected, 1) + assert.GreaterOrEqual(t, affected, int64(1)) } func TestTidyMarkers(t *testing.T) { diff --git a/internal/query/subjects.go b/internal/query/subjects.go index 32a76b476..911238712 100644 --- a/internal/query/subjects.go +++ b/internal/query/subjects.go @@ -1,6 +1,8 @@ package query import ( + "fmt" + "github.com/photoprism/photoprism/internal/entity" ) @@ -13,3 +15,12 @@ func Subjects(limit, offset int) (result entity.Subjects, err error) { return result, err } + +// ResetSubjects removes all unused subjects from the index. +func ResetSubjects() error { + return UnscopedDb(). + Where("subject_src = ?", entity.SrcMarker). + Where(fmt.Sprintf("subject_uid NOT IN (SELECT subject_uid FROM %s)", entity.Face{}.TableName())). + Delete(&entity.Subject{}). + Error +} diff --git a/internal/query/subjects_test.go b/internal/query/subjects_test.go index 7a622d888..985c31b5b 100644 --- a/internal/query/subjects_test.go +++ b/internal/query/subjects_test.go @@ -21,3 +21,7 @@ func TestSubjects(t *testing.T) { assert.IsType(t, entity.Subject{}, val) } } + +func TestResetSubjects(t *testing.T) { + assert.NoError(t, ResetSubjects()) +}