luisgulo/photoprism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Faces: Skip ambiguous embeddings when matching #1497 #3124

Browse Source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2023-02-21 04:49:06 +01:00
parent 74772aea97
commit 01d5156568
8 changed files with 74 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
internal/entity/face.go
View File

@ -70,7 +70,7 @@ func (m *Face) MatchId(f Face) string {
// SkipMatching checks whether the face should be skipped when matching.
func (m *Face) SkipMatching() bool {
return m.Embedding().SkipMatching()
return m.FaceKind > 1 || m.Embedding().SkipMatching()
}
// SetEmbeddings assigns face embeddings.
@ -100,7 +100,11 @@ func (m *Face) SetEmbeddings(embeddings face.Embeddings) (err error) {
// Update Face ID, Kind, and reset match timestamp,
m.ID = base32.StdEncoding.EncodeToString(s[:])
m.FaceKind = int(m.embedding.Kind())
if k := int(m.embedding.Kind()); k > m.FaceKind {
m.FaceKind = k
}
m.MatchedAt = nil
return nil
@ -183,13 +187,15 @@ func (m *Face) ResolveCollision(embeddings face.Embeddings) (resolved bool, err
// Should never happen.
return false, fmt.Errorf("collision distance must be positive")
} else if dist < 0.02 {
// Ignore if distance is very small as faces may belong to the same person.
log.Warnf("faces: clearing ambiguous subject %s from face %s, similar face at dist %f with source %s", SubjNames.Log(m.SubjUID), m.ID, dist, SrcString(m.FaceSrc))
log.Warnf("faces: ambiguous subject %s from face %s, very similar face at dist %f with source %s", SubjNames.Log(m.SubjUID), m.ID, dist, SrcString(m.FaceSrc))
// Reset subject UID just in case.
m.SubjUID = ""
m.FaceKind = int(face.AmbiguousFace)
m.UpdatedAt = TimeStamp()
m.MatchedAt = &m.UpdatedAt
m.Collisions++
m.CollisionRadius = dist
return true, m.Updates(Values{"SubjUID": m.SubjUID})
return true, m.Updates(Values{"Collisions": m.Collisions, "CollisionRadius": m.CollisionRadius, "FaceKind": m.FaceKind, "UpdatedAt": m.UpdatedAt, "MatchedAt": m.MatchedAt})
} else {
m.MatchedAt = nil
m.Collisions++

1
internal/face/embeddings_random.go
View File

@ -11,6 +11,7 @@ const (
RegularFace Kind = iota + 1
KidsFace
IgnoredFace
AmbiguousFace
)
var r = rand.New(rand.NewSource(time.Now().UnixNano()))

4
internal/photoprism/faces_audit.go
View File

@ -54,7 +54,7 @@ func (w *Faces) Audit(fix bool) (err error) {
conflicts := 0
resolved := 0
faces, ids, err := query.FacesByID(true, false, false)
faces, ids, err := query.FacesByID(true, false, false, false)
if err != nil {
return err
@ -129,7 +129,7 @@ func (w *Faces) Audit(fix bool) (err error) {
if success {
log.Infof("faces: successful conflict resolution for %s, face %s had collisions with other persons", entity.SubjNames.Log(f1.SubjUID), f1.ID)
resolved++
faces, _, err = query.FacesByID(true, false, false)
faces, _, err = query.FacesByID(true, false, false, false)
logErr("faces", "refresh", err)
} else {
log.Infof("faces: conflict resolution for %s not successful, face %s still has collisions with other persons", entity.SubjNames.Log(f1.SubjUID), f1.ID)

4
internal/photoprism/faces_match.go
View File

@ -44,7 +44,7 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) {
matchedAt := entity.TimePointer()
if opt.Force || unmatchedMarkers > 0 {
faces, err := query.Faces(false, false, false)
faces, err := query.Faces(false, false, false, false)
if err != nil {
return result, err
@ -58,7 +58,7 @@ func (w *Faces) Match(opt FacesOptions) (result FacesMatchResult, err error) {
}
// Find unmatched faces.
if unmatchedFaces, err := query.Faces(false, true, false); err != nil {
if unmatchedFaces, err := query.Faces(false, true, false, false); err != nil {
log.Error(err)
} else if len(unmatchedFaces) > 0 {
if r, err := w.MatchFaces(unmatchedFaces, false, matchedAt); err != nil {

4
internal/photoprism/faces_optimize.go
View File

@ -27,7 +27,7 @@ func (w *Faces) Optimize() (result FacesOptimizeResult, err error) {
var faces entity.Faces
// Fetch manually added faces from the database.
if faces, err = query.ManuallyAddedFaces(false, face.RegularFace); err != nil {
if faces, err = query.ManuallyAddedFaces(false, false); err != nil {
return result, err
} else if n = len(faces) - 1; n < 1 {
// Need at least 2 faces to optimize.
@ -41,7 +41,7 @@ func (w *Faces) Optimize() (result FacesOptimizeResult, err error) {
} else if faces[j].SubjUID != merge[len(merge)-1].SubjUID || j == n {
if len(merge) < 2 {
// Nothing to merge.
} else if _, err := query.MergeFaces(merge); err != nil {
} else if _, err := query.MergeFaces(merge, false); err != nil {
log.Errorf("%s (merge)", err)
} else {
result.Merged += len(merge)

2
internal/photoprism/faces_stats.go
View File

@ -54,7 +54,7 @@ func (w *Faces) Stats() (err error) {
log.Infof("faces: max Ø %f < median %f < %f", maxMin, maxMedian, maxMax)
}
if faces, err := query.Faces(true, false, false); err != nil {
if faces, err := query.Faces(true, false, false, false); err != nil {
log.Errorf("faces: %s", err)
} else if samples := len(faces); samples > 0 {
log.Infof("faces: computing distance of faces matching to the same person")

51
internal/query/faces.go
View File

@ -3,6 +3,8 @@ package query
import (
"fmt"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/internal/mutex"
@ -16,8 +18,8 @@ type IDs []string
type FaceMap map[string]entity.Face
// FacesByID retrieves faces from the database and returns a map with the Face ID as key.
func FacesByID(knownOnly, unmatchedOnly, inclHidden bool) (FaceMap, IDs, error) {
faces, err := Faces(knownOnly, unmatchedOnly, inclHidden)
func FacesByID(knownOnly, unmatchedOnly, hidden, ignored bool) (FaceMap, IDs, error) {
faces, err := Faces(knownOnly, unmatchedOnly, hidden, ignored)
if err != nil {
return nil, nil, err
@ -35,7 +37,7 @@ func FacesByID(knownOnly, unmatchedOnly, inclHidden bool) (FaceMap, IDs, error)
}
// Faces returns all (known / unmatched) faces from the index.
func Faces(knownOnly, unmatchedOnly, inclHidden bool) (result entity.Faces, err error) {
func Faces(knownOnly, unmatchedOnly, hidden, ignored bool) (result entity.Faces, err error) {
stmt := Db()
if knownOnly {
@ -46,30 +48,38 @@ func Faces(knownOnly, unmatchedOnly, inclHidden bool) (result entity.Faces, err
stmt = stmt.Where("matched_at IS NULL")
}
if !inclHidden {
if !hidden {
stmt = stmt.Where("face_hidden = ?", false)
}
if !ignored {
stmt = stmt.Where("face_kind <= 1")
}
err = stmt.Order("subj_uid, samples DESC").Find(&result).Error
return result, err
}
// ManuallyAddedFaces returns all manually added face clusters.
func ManuallyAddedFaces(hidden bool, kind face.Kind) (result entity.Faces, err error) {
err = Db().
func ManuallyAddedFaces(hidden, ignored bool) (result entity.Faces, err error) {
stmt := Db().
Where("face_hidden = ?", hidden).
Where("face_kind <= ?", int(kind)).
Where("face_src = ?", entity.SrcManual).
Where("subj_uid <> ''").Order("subj_uid, samples DESC").
Find(&result).Error
Where("subj_uid <> ''")
if !ignored {
stmt = stmt.Where("face_kind <= 1")
}
err = stmt.Order("subj_uid, samples DESC").Find(&result).Error
return result, err
}
// MatchFaceMarkers matches markers with known faces.
func MatchFaceMarkers() (affected int64, err error) {
faces, err := Faces(true, false, false)
faces, err := Faces(true, false, false, false)
if err != nil {
return affected, err
@ -140,12 +150,17 @@ func CountNewFaceMarkers(size, score int) (n int) {
}
// PurgeOrphanFaces removes unused faces from the index.
func PurgeOrphanFaces(faceIds []string) (removed int64, err error) {
func PurgeOrphanFaces(faceIds []string, ignored bool) (removed int64, err error) {
// Remove invalid face IDs.
if res := Db().
stmt := Db().
Where("id IN (?)", faceIds).
Where(fmt.Sprintf("id NOT IN (SELECT face_id FROM %s)", entity.Marker{}.TableName())).
Delete(&entity.Face{}); res.Error != nil {
Where("id NOT IN (SELECT face_id FROM ?)", gorm.Expr(entity.Marker{}.TableName()))
if !ignored {
stmt = stmt.Where("face_kind <= 1")
}
if res := stmt.Delete(&entity.Face{}); res.Error != nil {
return removed, fmt.Errorf("faces: %s while purging orphans", res.Error)
} else {
removed += res.RowsAffected
@ -155,7 +170,7 @@ func PurgeOrphanFaces(faceIds []string) (removed int64, err error) {
}
// MergeFaces returns a new face that replaces multiple others.
func MergeFaces(merge entity.Faces) (merged *entity.Face, err error) {
func MergeFaces(merge entity.Faces, ignored bool) (merged *entity.Face, err error) {
if len(merge) < 2 {
// Nothing to merge.
return merged, fmt.Errorf("faces: two or more clusters required for merging")
@ -180,7 +195,7 @@ func MergeFaces(merge entity.Faces) (merged *entity.Face, err error) {
}
// PurgeOrphanFaces removes unused faces from the index.
if removed, err := PurgeOrphanFaces(merge.IDs()); err != nil {
if removed, err := PurgeOrphanFaces(merge.IDs(), ignored); err != nil {
return merged, err
} else if removed > 0 {
log.Debugf("faces: removed %d orphans for subject %s", removed, clean.Log(subjUID))
@ -193,7 +208,7 @@ func MergeFaces(merge entity.Faces) (merged *entity.Face, err error) {
// ResolveFaceCollisions resolves collisions of different subject's faces.
func ResolveFaceCollisions() (conflicts, resolved int, err error) {
faces, ids, err := FacesByID(true, false, false)
faces, ids, err := FacesByID(true, false, false, false)
if err != nil {
return conflicts, resolved, err
@ -263,7 +278,7 @@ func ResolveFaceCollisions() (conflicts, resolved int, err error) {
if success {
log.Infof("faces: successful conflict resolution for %s, face %s had collisions with other persons", entity.SubjNames.Log(f1.SubjUID), f1.ID)
resolved++
faces, _, err = FacesByID(true, false, false)
faces, _, err = FacesByID(true, false, false, false)
logErr("faces", "refresh", err)
} else {
log.Infof("faces: conflict resolution for %s not successful, face %s still has collisions with other persons", entity.SubjNames.Log(f1.SubjUID), f1.ID)

30
internal/query/faces_test.go
View File

@ -10,8 +10,8 @@ import (
)
func TestFaces(t *testing.T) {
t.Run("known", func(t *testing.T) {
results, err := Faces(true, false, false)
t.Run("Known", func(t *testing.T) {
results, err := Faces(true, false, false, false)
if err != nil {
t.Fatal(err)
@ -25,7 +25,7 @@ func TestFaces(t *testing.T) {
})
t.Run("Hidden", func(t *testing.T) {
results, err := Faces(false, false, true)
results, err := Faces(false, false, true, false)
if err != nil {
t.Fatal(err)
@ -34,8 +34,18 @@ func TestFaces(t *testing.T) {
assert.GreaterOrEqual(t, len(results), 1)
})
t.Run("unmatched", func(t *testing.T) {
results, err := Faces(false, true, false)
t.Run("Ignored", func(t *testing.T) {
results, err := Faces(false, false, true, true)
if err != nil {
t.Fatal(err)
}
assert.GreaterOrEqual(t, len(results), 1)
})
t.Run("Unmatched", func(t *testing.T) {
results, err := Faces(false, true, false, false)
if err != nil {
t.Fatal(err)
@ -51,7 +61,7 @@ func TestFaces(t *testing.T) {
func TestManuallyAddedFaces(t *testing.T) {
t.Run("Ok", func(t *testing.T) {
results, err := ManuallyAddedFaces(false, face.RegularFace)
results, err := ManuallyAddedFaces(false, false)
if err != nil {
t.Fatal(err)
@ -64,7 +74,7 @@ func TestManuallyAddedFaces(t *testing.T) {
}
})
t.Run("Hidden", func(t *testing.T) {
results, err := ManuallyAddedFaces(true, face.RegularFace)
results, err := ManuallyAddedFaces(true, false)
if err != nil {
t.Fatal(err)
@ -164,7 +174,7 @@ func TestMergeFaces(t *testing.T) {
faces := entity.Faces{*face1, *face2}
result, err := MergeFaces(faces)
result, err := MergeFaces(faces, false)
if err != nil {
t.Fatal(err)
@ -200,13 +210,13 @@ func TestMergeFaces(t *testing.T) {
faces := entity.Faces{*face1, *face2}
result, err := MergeFaces(faces)
result, err := MergeFaces(faces, false)
assert.EqualError(t, err, "faces: cannot merge clusters with conflicting subjects jqynvsf28rhn6b0c <> jqynvt925h8c1asv")
assert.Nil(t, result)
})
t.Run("OneSubject", func(t *testing.T) {
result, err := MergeFaces(entity.Faces{entity.Face{ID: "5LH5E35ZGUMF5AYLM42BIZH4DGQHJDAV"}})
result, err := MergeFaces(entity.Faces{entity.Face{ID: "5LH5E35ZGUMF5AYLM42BIZH4DGQHJDAV"}}, false)
assert.EqualError(t, err, "faces: two or more clusters required for merging")
assert.Nil(t, result)