luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

People: Improve face matching #22

Browse source
This commit is contained in:
Michael Mayer 2021-08-19 23:12:51 +02:00
parent 885024d592
commit 5cec098524
12 changed files with 233 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
internal/entity/face.go
View file

@ -122,3 +122,38 @@ func (m *Face) Update(attr string, value interface{}) error {
func (m *Face) Updates(values interface{}) error {
return UnscopedDb().Model(m).Updates(values).Error
}
// FirstOrCreateFace returns the existing entity, inserts a new entity or nil in case of errors.
func FirstOrCreateFace(m *Face) *Face {
result := Face{}
if err := UnscopedDb().Where("id = ?", m.ID).First(&result).Error; err == nil {
return &result
} else if createErr := m.Create(); createErr == nil {
return m
} else if err := UnscopedDb().Where("id = ?", m.ID).First(&result).Error; err == nil {
return &result
} else {
log.Errorf("face: %s (find or create %s)", createErr, m.ID)
}
return nil
}
// FindFace returns an existing entity if exists.
func FindFace(id string) *Face {
if id == "" {
return nil
}
result := Face{}
db := Db()
db = db.Where("id = ?", id)
if err := db.First(&result).Error; err != nil {
return nil
}
return &result
}

35
internal/entity/marker.go
View file

@ -31,6 +31,7 @@ type Marker struct {
SubjectSrc string `gorm:"type:VARBINARY(8);default:'';" json:"SubjectSrc" yaml:"SubjectSrc,omitempty"`
Subject *Subject `gorm:"foreignkey:SubjectUID;association_foreignkey:SubjectUID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Subject,omitempty" yaml:"-"`
FaceID string `gorm:"type:VARBINARY(42);index;" json:"FaceID" yaml:"FaceID,omitempty"`
Face *Face `gorm:"foreignkey:FaceID;association_foreignkey:ID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"-" yaml:"-"`
EmbeddingsJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"EmbeddingsJSON,omitempty"`
embeddings Embeddings `gorm:"-"`
LandmarksJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"LandmarksJSON,omitempty"`
@ -198,11 +199,7 @@ func (m *Marker) SyncSubject(updateRelated bool) error {
// Create known face for subject?
if m.FaceID != "" || m.SubjectSrc != SrcManual {
// Do nothing.
} else if f := NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil {
return fmt.Errorf("failed adding known face for subject %s", m.SubjectUID)
} else if err := f.Create(); err != nil {
log.Debugf("marker: %s (add known face)", err)
} else {
} else if f := m.GetFace(); f != nil {
m.FaceID = f.ID
}
@ -221,7 +218,7 @@ func (m *Marker) SyncSubject(updateRelated bool) error {
Updates(Values{"SubjectUID": m.SubjectUID, "SubjectSrc": SrcAuto}).Error; err != nil {
return fmt.Errorf("%s (update related markers)", err)
} else {
log.Infof("marker: matched %s", subj.SubjectName)
log.Debugf("marker: matched subject %s with face %s", subj.SubjectName, m.FaceID)
}
return nil
@ -258,7 +255,7 @@ func (m *Marker) Embeddings() Embeddings {
return m.embeddings
}
// GetSubject returns the matching subject entity, if possible.
// GetSubject returns a subject entity if possible.
func (m *Marker) GetSubject() (subj *Subject) {
if m.Subject != nil {
return m.Subject
@ -283,6 +280,30 @@ func (m *Marker) GetSubject() (subj *Subject) {
return m.Subject
}
// GetFace returns a matching face entity if possible.
func (m *Marker) GetFace() (f *Face) {
if m.Face != nil {
return m.Face
}
if m.FaceID == "" && m.SubjectSrc == SrcManual {
if f = NewFace(m.SubjectUID, SrcManual, m.Embeddings()); f == nil {
return nil
} else if f = FirstOrCreateFace(f); f == nil {
log.Debugf("marker: invalid face")
return nil
}
m.FaceID = f.ID
return f
}
m.Face = FindFace(m.FaceID)
return m.Face
}
// FindMarker returns an existing row if exists.
func FindMarker(id uint) *Marker {
result := Marker{}

4
internal/entity/subject.go
View file

@ -140,7 +140,7 @@ func (m *Subject) Updates(values interface{}) error {
return UnscopedDb().Model(m).Updates(values).Error
}
// FirstOrCreateSubject returns the existing subject, inserts a new subject or nil in case of errors.
// FirstOrCreateSubject returns the existing entity, inserts a new entity or nil in case of errors.
func FirstOrCreateSubject(m *Subject) *Subject {
result := Subject{}
@ -165,7 +165,7 @@ func FirstOrCreateSubject(m *Subject) *Subject {
return nil
}
// FindSubject returns an existing row if exists.
// FindSubject returns an existing entity if exists.
func FindSubject(s string) *Subject {
if s == "" {
return nil

123
internal/photoprism/faces.go
View file

@ -3,7 +3,6 @@ package photoprism
import (
"fmt"
"runtime/debug"
"time"
"github.com/photoprism/photoprism/internal/face"
@ -151,7 +150,13 @@ func (w *Faces) Reset() (err error) {
if err := query.ResetFaces(); err != nil {
log.Errorf("faces: %s (reset)", err)
} else {
log.Infof("faces: reset known faces")
log.Infof("faces: reset faces")
}
if err := query.ResetSubjects(); err != nil {
log.Errorf("faces: %s (reset)", err)
} else {
log.Infof("faces: reset subjects")
}
return nil
@ -182,21 +187,30 @@ func (w *Faces) Start(opt FacesOptions) (err error) {
defer mutex.MainWorker.Stop()
// Skip clustering if index contains no new face markers and force option isn't set.
if n := query.CountNewFaceMarkers(); n < 1 && !opt.Force{
if n := query.CountNewFaceMarkers(); n < 1 && !opt.Force {
log.Debugf("faces: no new samples")
// Match (add and assign) subjects with markers, if possible.
if affected, err := query.MatchMarkersWithSubjects(); err != nil {
var updated int64
// Adds and reference known marker subjects.
if affected, err := query.AddMarkerSubjects(); err != nil {
log.Errorf("faces: %s (match markers with subjects)", err)
} else if affected > 0 {
log.Infof("faces: matched %d markers with subjects", affected)
} else {
updated += affected
}
// Match known faces with markers, if possible.
if matched, err := query.MatchKnownFaces(); err != nil {
// Match markers with known faces.
if affected, err := query.MatchFaceMarkers(); err != nil {
return err
} else if matched > 0 {
log.Infof("faces: matched %d markers to faces", matched)
} else {
updated += affected
}
// Log result.
if updated > 0 {
log.Infof("faces: %d markers updated", updated)
} else {
log.Debug("faces: no changes")
}
// Clean-up invalid marker data.
@ -250,6 +264,11 @@ func (w *Faces) Start(opt FacesOptions) (err error) {
results[n-1] = append(results[n-1], embeddings[i])
}
if err := query.PurgeAnonymousFaces(); err != nil {
dbErrors++
log.Errorf("faces: %s", err)
}
for _, embedding := range results {
if f := entity.NewFace("", entity.SrcAuto, embedding); f == nil {
dbErrors++
@ -264,87 +283,9 @@ func (w *Faces) Start(opt FacesOptions) (err error) {
}
}
if err := query.PurgeAnonymousFaces(); err != nil {
dbErrors++
log.Errorf("faces: %s", err)
}
if faces, err := query.Faces(false); err != nil {
// Match existing markers and faces.
if recognized, unknown, err = w.MatchMarkers(); err != nil {
return err
} else {
limit := 500
offset := 0
for {
markers, err := query.Markers(limit, offset, entity.MarkerFace, true, false)
if err != nil {
return err
}
if len(markers) == 0 {
break
}
for _, marker := range markers {
if mutex.MainWorker.Canceled() {
return fmt.Errorf("worker canceled")
}
// Pointer to the matching face.
var f *entity.Face
// Distance to the matching face.
var d float64
// Find the closest face match for marker.
for _, e := range marker.Embeddings() {
for i, match := range faces {
if dist := clusters.EuclideanDistance(e, match.Embedding()); f == nil || dist < d {
f = &faces[i]
d = dist
}
}
}
// No match?
if f == nil {
continue
}
// Too distant?
if d > (f.Radius + face.ClusterRadius) {
continue
}
if updated, err := marker.SetFace(f); err != nil {
dbErrors++
log.Errorf("faces: %s", err)
} else if updated {
recognized++
}
if marker.SubjectUID == "" {
unknown++
}
}
offset += limit
time.Sleep(50 * time.Millisecond)
}
}
// Match known faces with markers, if possible.
if m, err := query.MatchKnownFaces(); err != nil {
return err
} else {
recognized += m
}
// Clean-up invalid marker data.
if err := query.TidyMarkers(); err != nil {
log.Errorf("faces: %s (tidy)", err)
}
// Log results.

98
internal/photoprism/faces_match.go Normal file
View file

@ -0,0 +1,98 @@
package photoprism
import (
"fmt"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/pkg/clusters"
)
// MatchMarkers matches existing markers and faces.
func (w *Faces) MatchMarkers() (recognized, unknown int64, err error) {
if w.Disabled() {
return 0, 0, nil
}
if faces, err := query.Faces(false); err != nil {
return recognized, unknown, err
} else {
limit := 500
offset := 0
for {
markers, err := query.Markers(limit, offset, entity.MarkerFace, true, false)
if err != nil {
return recognized, unknown, err
}
if len(markers) == 0 {
break
}
for _, marker := range markers {
if mutex.MainWorker.Canceled() {
return recognized, unknown, fmt.Errorf("worker canceled")
}
// Pointer to the matching face.
var f *entity.Face
// Distance to the matching face.
var d float64
// Find the closest face match for marker.
for _, e := range marker.Embeddings() {
for i, match := range faces {
if dist := clusters.EuclideanDistance(e, match.Embedding()); f == nil || dist < d {
f = &faces[i]
d = dist
}
}
}
// No match?
if f == nil {
continue
}
// Too distant?
if d > (f.Radius + face.ClusterRadius) {
continue
}
if updated, err := marker.SetFace(f); err != nil {
log.Errorf("faces: %s", err)
} else if updated {
recognized++
}
if marker.SubjectUID == "" {
unknown++
}
}
offset += limit
time.Sleep(50 * time.Millisecond)
}
}
// Update remaining markers based on current matches.
if m, err := query.MatchFaceMarkers(); err != nil {
return recognized, unknown, err
} else {
recognized += m
}
// Reset invalid marker data.
if err := query.TidyMarkers(); err != nil {
return recognized, unknown, err
}
return recognized, unknown, nil
}

11
internal/photoprism/index.go
View file

@ -230,6 +230,15 @@ func (ind *Index) Start(opt IndexOptions) fs.Done {
}
if filesIndexed > 0 {
// Match existing faces if facial recognition is enabled.
if w := NewFaces(ind.conf); w.Disabled() {
log.Debugf("index: skipping facial recognition")
} else if recognized, unknown, err := w.MatchMarkers(); err != nil {
log.Errorf("index: %s", err)
} else if recognized > 0 || unknown > 0 {
log.Infof("faces: %d recognized, %d unknown", recognized, unknown)
}
event.Publish("index.updating", event.Data{
"step": "counts",
})
@ -246,7 +255,7 @@ func (ind *Index) Start(opt IndexOptions) fs.Done {
return done
}
// File indexes a single file and returns the result.
// FileName indexes a single file and returns the result.
func (ind *Index) FileName(fileName string, o IndexOptions) (result IndexResult) {
file, err := NewMediaFile(fileName)

12
internal/query/faces.go
View file

@ -4,7 +4,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
)
// Faces returns (known) faces from the index.
// Faces returns all (known) faces from the index.
func Faces(knownOnly bool) (result entity.Faces, err error) {
stmt := Db().
Where("face_src <> ?", entity.SrcDefault).
@ -19,8 +19,8 @@ func Faces(knownOnly bool) (result entity.Faces, err error) {
return result, err
}
// MatchKnownFaces matches known faces with markers, if possible.
func MatchKnownFaces() (affected int64, err error) {
// MatchFaceMarkers matches markers with known faces.
func MatchFaceMarkers() (affected int64, err error) {
faces, err := Faces(true)
if err != nil {
@ -46,7 +46,7 @@ func MatchKnownFaces() (affected int64, err error) {
func PurgeAnonymousFaces() error {
return UnscopedDb().Delete(
entity.Face{},
"id <> ? AND subject_uid = '' AND updated_at < ?", entity.UnknownFace.ID, entity.Yesterday()).Error
"face_src = ? AND subject_uid = ''", entity.SrcAuto).Error
}
// ResetFaces removes all face clusters from the index.
@ -60,11 +60,11 @@ func ResetFaces() error {
func CountNewFaceMarkers() (n int) {
var f entity.Face
if err := Db().Where("id <> ?", entity.UnknownFace.ID).Order("created_at DESC").Take(&f).Error; err != nil {
if err := Db().Where("face_src = ?", entity.SrcAuto).Order("created_at DESC").Take(&f).Error; err != nil {
log.Debugf("faces: no existing clusters")
}
q := Db().Model(&entity.Markers{}).Where("marker_type = ? AND embeddings_json <> ''", entity.MarkerFace)
q := Db().Model(&entity.Markers{}).Where("marker_type = ? AND marker_invalid = 0 AND embeddings_json <> ''", entity.MarkerFace)
if !f.CreatedAt.IsZero() {
q = q.Where("created_at > ?", f.CreatedAt)

4
internal/query/faces_test.go
View file

@ -22,7 +22,7 @@ func TestFaces(t *testing.T) {
}
}
func TestMatchKnownFaces(t *testing.T) {
func TestMatchFaceMarkers(t *testing.T) {
const faceFixtureId = uint(6)
if m, err := MarkerByID(faceFixtureId); err != nil {
@ -39,7 +39,7 @@ func TestMatchKnownFaces(t *testing.T) {
t.Fatal(err)
}
affected, err := MatchKnownFaces()
affected, err := MatchFaceMarkers()
if err != nil {
t.Fatal(err)

4
internal/query/markers.go
View file

@ -70,8 +70,8 @@ func Embeddings(single bool) (result entity.Embeddings, err error) {
return result, nil
}
// MatchMarkersWithSubjects matches (add and assign) subjects with markers, if possible.
func MatchMarkersWithSubjects() (affected int, err error) {
// AddMarkerSubjects adds and references known marker subjects.
func AddMarkerSubjects() (affected int64, err error) {
var markers entity.Markers
if err := Db().

6
internal/query/markers_test.go
View file

@ -59,11 +59,11 @@ func TestEmbeddings(t *testing.T) {
}
}
func TestMatchMarkersWithSubjects(t *testing.T) {
affected, err := MatchMarkersWithSubjects()
func TestAddMarkerSubjects(t *testing.T) {
affected, err := AddMarkerSubjects()
assert.NoError(t, err)
assert.GreaterOrEqual(t, affected, 1)
assert.GreaterOrEqual(t, affected, int64(1))
}
func TestTidyMarkers(t *testing.T) {

11
internal/query/subjects.go
View file

@ -1,6 +1,8 @@
package query
import (
"fmt"
"github.com/photoprism/photoprism/internal/entity"
)
@ -13,3 +15,12 @@ func Subjects(limit, offset int) (result entity.Subjects, err error) {
return result, err
}
// ResetSubjects removes all unused subjects from the index.
func ResetSubjects() error {
return UnscopedDb().
Where("subject_src = ?", entity.SrcMarker).
Where(fmt.Sprintf("subject_uid NOT IN (SELECT subject_uid FROM %s)", entity.Face{}.TableName())).
Delete(&entity.Subject{}).
Error
}

4
internal/query/subjects_test.go
View file

@ -21,3 +21,7 @@ func TestSubjects(t *testing.T) {
assert.IsType(t, entity.Subject{}, val)
}
}
func TestResetSubjects(t *testing.T) {
assert.NoError(t, ResetSubjects())
}