diff --git a/internal/crop/area.go b/internal/crop/area.go index 7d314bb94..c2b216333 100644 --- a/internal/crop/area.go +++ b/internal/crop/area.go @@ -3,6 +3,7 @@ package crop import ( "fmt" "image" + "math" "strconv" "strings" @@ -60,6 +61,66 @@ func (a Area) FileWidth(size Size) int { return int(float32(size.Width) / a.W) } +// Top returns the top Y coordinate as float64. +func (a Area) Top() float64 { + return float64(a.Y) +} + +// Left returns the left X coordinate as float64. +func (a Area) Left() float64 { + return float64(a.X) +} + +// Right returns the right X coordinate as float64. +func (a Area) Right() float64 { + return float64(a.X + a.W) +} + +// Bottom returns the bottom Y coordinate as float64. +func (a Area) Bottom() float64 { + return float64(a.Y + a.H) +} + +// Surface returns the surface area. +func (a Area) Surface() float64 { + return float64(a.W * a.H) +} + +// SurfaceRatio returns the surface ratio. +func (a Area) SurfaceRatio(area float64) float64 { + if area <= 0 { + return 0 + } + + if s := a.Surface(); s <= 0 { + return 0 + } else if area > s { + return s / area + } else { + return area / s + } +} + +// Overlap calculates the overlap of two areas. +func (a Area) Overlap(other Area) (x, y float64) { + x = math.Max(0, math.Min(a.Right(), other.Right())-math.Max(a.Left(), other.Left())) + y = math.Max(0, math.Min(a.Bottom(), other.Bottom())-math.Max(a.Top(), other.Top())) + + return x, y +} + +// OverlapArea calculates the overlap area of two areas. +func (a Area) OverlapArea(other Area) (area float64) { + x, y := a.Overlap(other) + + return x * y +} + +// OverlapPercent calculates the overlap ratio of two areas in percent. +func (a Area) OverlapPercent(other Area) int { + return int(math.Round(other.SurfaceRatio(a.OverlapArea(other)) * 100)) +} + // clipVal ensures the relative size is within a valid range. func clipVal(f float32) float32 { if f > 1 { diff --git a/internal/crop/area_test.go b/internal/crop/area_test.go index 3d0618b38..bbebd2615 100644 --- a/internal/crop/area_test.go +++ b/internal/crop/area_test.go @@ -137,3 +137,45 @@ func TestParseThumb(t *testing.T) { assert.True(t, AreaFromString(a).Empty()) }) } + +func TestArea_SurfaceRatio(t *testing.T) { + var a1 = Area{Name: "face", X: 0.308333, Y: 0.206944, W: 0.355556, H: 0.355556} + var a2 = Area{Name: "face", X: 0.208313, Y: 0.156914, W: 0.655556, H: 0.655556} + var a3 = Area{Name: "face", X: 0.998133, Y: 0.816944, W: 0.0001, H: 0.0001} + var a4 = Area{Name: "face", X: 0.298133, Y: 0.216944, W: 0.255556, H: 0.155556} + + assert.Equal(t, 99, int(a1.SurfaceRatio(a1.OverlapArea(a1))*100)) + assert.Equal(t, 99, int(a1.SurfaceRatio(a1.OverlapArea(a2))*100)) + assert.Equal(t, 29, int(a2.SurfaceRatio(a2.OverlapArea(a1))*100)) + assert.Equal(t, 0, int(a1.SurfaceRatio(a1.OverlapArea(a3))*100)) + assert.Equal(t, 30, int(a1.SurfaceRatio(a1.OverlapArea(a4))*100)) + assert.Equal(t, 0, int(a1.SurfaceRatio(a3.OverlapArea(a1))*100)) + assert.Equal(t, 30, int(a1.SurfaceRatio(a4.OverlapArea(a1))*100)) +} + +func TestArea_OverlapArea(t *testing.T) { + var a1 = Area{Name: "face", X: 0.308333, Y: 0.206944, W: 0.355556, H: 0.355556} + var a2 = Area{Name: "face", X: 0.208313, Y: 0.156914, W: 0.655556, H: 0.655556} + var a3 = Area{Name: "face", X: 0.998133, Y: 0.816944, W: 0.0001, H: 0.0001} + var a4 = Area{Name: "face", X: 0.298133, Y: 0.216944, W: 0.255556, H: 0.155556} + + assert.Equal(t, 0.1264200823986168, a1.OverlapArea(a1)) + assert.Equal(t, int(a1.Surface()*10000), int(a1.OverlapArea(a1)*10000)) + assert.Equal(t, 0.1264200823986168, a1.OverlapArea(a2)) + assert.Equal(t, 0.1264200823986168, a2.OverlapArea(a1)) + assert.Equal(t, 0.0, a1.OverlapArea(a3)) + assert.Equal(t, 0.038166598943088825, a1.OverlapArea(a4)) +} + +func TestArea_OverlapPercent(t *testing.T) { + var a1 = Area{Name: "face", X: 0.308333, Y: 0.206944, W: 0.355556, H: 0.355556} + var a2 = Area{Name: "face", X: 0.208313, Y: 0.156914, W: 0.655556, H: 0.655556} + var a3 = Area{Name: "face", X: 0.998133, Y: 0.816944, W: 0.0001, H: 0.0001} + var a4 = Area{Name: "face", X: 0.298133, Y: 0.216944, W: 0.255556, H: 0.155556} + + assert.Equal(t, 100, a1.OverlapPercent(a1)) + assert.Equal(t, 29, a1.OverlapPercent(a2)) + assert.Equal(t, 100, a2.OverlapPercent(a1)) + assert.Equal(t, 0, a1.OverlapPercent(a3)) + assert.Equal(t, 96, a1.OverlapPercent(a4)) +} diff --git a/internal/entity/marker.go b/internal/entity/marker.go index 758a4d781..0481d7922 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -482,6 +482,11 @@ func (m *Marker) Matched() error { return UnscopedDb().Model(m).UpdateColumns(Values{"MatchedAt": m.MatchedAt}).Error } +// Top returns the top Y coordinate as float64. +func (m *Marker) Top() float64 { + return float64(m.Y) +} + // Left returns the left X coordinate as float64. func (m *Marker) Left() float64 { return float64(m.X) @@ -492,16 +497,31 @@ func (m *Marker) Right() float64 { return float64(m.X + m.W) } -// Top returns the top Y coordinate as float64. -func (m *Marker) Top() float64 { - return float64(m.Y) -} - // Bottom returns the bottom Y coordinate as float64. func (m *Marker) Bottom() float64 { return float64(m.Y + m.H) } +// Surface returns the surface area. +func (m *Marker) Surface() float64 { + return float64(m.W * m.H) +} + +// SurfaceRatio returns the surface ratio. +func (m *Marker) SurfaceRatio(area float64) float64 { + if area <= 0 { + return 0 + } + + if s := m.Surface(); s <= 0 { + return 0 + } else if area > s { + return s / area + } else { + return area / s + } +} + // Overlap calculates the overlap of two markers. func (m *Marker) Overlap(marker Marker) (x, y float64) { x = math.Max(0, math.Min(m.Right(), marker.Right())-math.Max(m.Left(), marker.Left())) @@ -517,6 +537,11 @@ func (m *Marker) OverlapArea(marker Marker) (area float64) { return x * y } +// OverlapPercent calculates the overlap ratio of two markers in percent. +func (m *Marker) OverlapPercent(marker Marker) int { + return int(math.Round(marker.SurfaceRatio(m.OverlapArea(marker)) * 100)) +} + // FindMarker returns an existing row if exists. func FindMarker(markerUid string) *Marker { if markerUid == "" { @@ -552,7 +577,7 @@ func FindFaceMarker(faceId string) *Marker { // UpdateOrCreateMarker updates a marker in the database or creates a new one if needed. func UpdateOrCreateMarker(m *Marker) (*Marker, error) { - const d = 0.07 + const d = 0.0001 result := Marker{} diff --git a/internal/entity/marker_test.go b/internal/entity/marker_test.go index aab2cadef..29decfafe 100644 --- a/internal/entity/marker_test.go +++ b/internal/entity/marker_test.go @@ -532,3 +532,45 @@ func TestMarker_RefreshPhotos(t *testing.T) { t.Fatal(err) } } + +func TestMarker_SurfaceRatio(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea2, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + m4 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + + assert.Equal(t, 99, int(m1.SurfaceRatio(m1.OverlapArea(m1))*100)) + assert.Equal(t, 99, int(m1.SurfaceRatio(m1.OverlapArea(m2))*100)) + assert.Equal(t, 29, int(m2.SurfaceRatio(m2.OverlapArea(m1))*100)) + assert.Equal(t, 0, int(m1.SurfaceRatio(m1.OverlapArea(m3))*100)) + assert.Equal(t, 30, int(m1.SurfaceRatio(m1.OverlapArea(m4))*100)) + assert.Equal(t, 0, int(m1.SurfaceRatio(m3.OverlapArea(m1))*100)) + assert.Equal(t, 30, int(m1.SurfaceRatio(m4.OverlapArea(m1))*100)) +} + +func TestMarker_OverlapArea(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea2, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + m4 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + + assert.Equal(t, 0.1264200823986168, m1.OverlapArea(m1)) + assert.Equal(t, int(m1.Surface()*10000), int(m1.OverlapArea(m1)*10000)) + assert.Equal(t, 0.1264200823986168, m1.OverlapArea(m2)) + assert.Equal(t, 0.1264200823986168, m2.OverlapArea(m1)) + assert.Equal(t, 0.0, m1.OverlapArea(m3)) + assert.Equal(t, 0.038166598943088825, m1.OverlapArea(m4)) +} + +func TestMarker_OverlapPercent(t *testing.T) { + m1 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea1, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 100, 65) + m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea2, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) + m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + m4 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea4, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) + + assert.Equal(t, 100, m1.OverlapPercent(m1)) + assert.Equal(t, 29, m1.OverlapPercent(m2)) + assert.Equal(t, 100, m2.OverlapPercent(m1)) + assert.Equal(t, 0, m1.OverlapPercent(m3)) + assert.Equal(t, 96, m1.OverlapPercent(m4)) +} diff --git a/internal/entity/markers.go b/internal/entity/markers.go index 7360ecf5f..3712d7542 100644 --- a/internal/entity/markers.go +++ b/internal/entity/markers.go @@ -1,6 +1,7 @@ package entity import ( + "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/pkg/txt" ) @@ -24,7 +25,7 @@ func (m Markers) Save(fileUID string) error { // Contains returns true if a marker at the same position already exists. func (m Markers) Contains(other Marker) bool { for _, marker := range m { - if marker.OverlapArea(other) > 0.02 { + if marker.OverlapPercent(other) > face.OverlapThreshold { return true } } diff --git a/internal/entity/markers_test.go b/internal/entity/markers_test.go index 52753acd7..849e76b74 100644 --- a/internal/entity/markers_test.go +++ b/internal/entity/markers_test.go @@ -9,7 +9,7 @@ import ( ) var cropArea1 = crop.Area{Name: "face", X: 0.308333, Y: 0.206944, W: 0.355556, H: 0.355556} -var cropArea2 = crop.Area{Name: "face", X: 0.308313, Y: 0.206914, W: 0.655556, H: 0.655556} +var cropArea2 = crop.Area{Name: "face", X: 0.208313, Y: 0.156914, W: 0.655556, H: 0.655556} var cropArea3 = crop.Area{Name: "face", X: 0.998133, Y: 0.816944, W: 0.0001, H: 0.0001} var cropArea4 = crop.Area{Name: "face", X: 0.298133, Y: 0.216944, W: 0.255556, H: 0.155556} @@ -19,9 +19,12 @@ func TestMarkers_Contains(t *testing.T) { m2 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea2, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 100, 65) m3 := *NewMarker(FileFixtures.Get("exampleFileName.jpg"), cropArea3, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 100, 65) - m := Markers{m1} + assert.Equal(t, 29, m1.OverlapPercent(m2)) + assert.Equal(t, 100, m2.OverlapPercent(m1)) - assert.True(t, m.Contains(m2)) + m := Markers{m2} + + assert.True(t, m.Contains(m1)) assert.False(t, m.Contains(m3)) }) t.Run("Conflicting", func(t *testing.T) { diff --git a/internal/face/area.go b/internal/face/area.go index 34cab03b1..a6158479a 100644 --- a/internal/face/area.go +++ b/internal/face/area.go @@ -60,7 +60,7 @@ func (a Area) Relative(r Area, rows, cols float32) crop.Area { ) } -// TopLeft returns the top left position of the face. +// TopLeft returns the top left position of the area. func (a Area) TopLeft() (int, int) { return a.Row - (a.Scale / 2), a.Col - (a.Scale / 2) } diff --git a/internal/face/area_test.go b/internal/face/area_test.go new file mode 100644 index 000000000..ecf41b455 --- /dev/null +++ b/internal/face/area_test.go @@ -0,0 +1,35 @@ +package face + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var area1 = NewArea("face1", 400, 250, 200) +var area2 = NewArea("face2", 100, 100, 50) +var area3 = NewArea("face3", 900, 500, 25) +var area4 = NewArea("face4", 110, 110, 60) + +func TestArea_TopLeft(t *testing.T) { + t.Run("area1", func(t *testing.T) { + x, y := area1.TopLeft() + assert.Equal(t, 300, x) + assert.Equal(t, 150, y) + }) + t.Run("area2", func(t *testing.T) { + x, y := area2.TopLeft() + assert.Equal(t, 75, x) + assert.Equal(t, 75, y) + }) + t.Run("area3", func(t *testing.T) { + x, y := area3.TopLeft() + assert.Equal(t, 888, x) + assert.Equal(t, 488, y) + }) + t.Run("area4", func(t *testing.T) { + x, y := area4.TopLeft() + assert.Equal(t, 80, x) + assert.Equal(t, 80, y) + }) +} diff --git a/internal/face/detector.go b/internal/face/detector.go index 0017502f9..c109f2fb8 100644 --- a/internal/face/detector.go +++ b/internal/face/detector.go @@ -87,7 +87,7 @@ func Detect(fileName string, findLandmarks bool, minSize int) (faces Faces, err shiftFactor: 0.1, scaleFactor: 1.1, iouThreshold: 0.2, - scoreThreshold: 9.0, + scoreThreshold: ScoreThreshold, perturb: 63, } @@ -154,10 +154,6 @@ func (d *Detector) Detect(fileName string) (faces []pigo.Detection, params pigo. Dim: cols, } - if rows > 800 || cols > 800 { - d.scoreThreshold += 9.0 - } - params = pigo.CascadeParams{ MinSize: d.minSize, MaxSize: maxSize, @@ -180,28 +176,34 @@ func (d *Detector) Detect(fileName string) (faces []pigo.Detection, params pigo. // Faces adds landmark coordinates to detected faces and returns the results. func (d *Detector) Faces(det []pigo.Detection, params pigo.CascadeParams, findLandmarks bool) (results Faces, err error) { - var maxQ float32 - - // Sort by quality. + // Sort by size. sort.Slice(det, func(i, j int) bool { - return det[i].Q > det[j].Q + return det[i].Scale > det[j].Scale }) for _, face := range det { + // Small faces require higher quality. + threshold := d.scoreThreshold + + if face.Scale < 30 { + threshold += 11.5 + } else if face.Scale < 50 { + threshold += 9.0 + } else if face.Scale < 80 { + threshold += 6.5 + } else if face.Scale < 110 { + threshold += 2.5 + } + + // Skip face if quality is too low. + if face.Q < threshold { + continue + } + var eyesCoords []Area var landmarkCoords []Area var puploc *pigo.Puploc - if face.Q < d.scoreThreshold { - continue - } - - if maxQ < face.Q { - maxQ = face.Q - } else if maxQ >= 20 && face.Q < 15 { - continue - } - faceCoord := NewArea( "face", face.Row, @@ -312,14 +314,20 @@ func (d *Detector) Faces(det []pigo.Detection, params pigo.CascadeParams, findLa } } - results = append(results, Face{ + f := Face{ Rows: params.ImageParams.Rows, Cols: params.ImageParams.Cols, Score: int(face.Q), Area: faceCoord, Eyes: eyesCoords, Landmarks: landmarkCoords, - }) + } + + if results.Contains(f) { + // Ignore. + } else { + results.Append(f) + } } return results, nil diff --git a/internal/face/detector_test.go b/internal/face/detector_test.go new file mode 100644 index 000000000..3bee4a4a4 --- /dev/null +++ b/internal/face/detector_test.go @@ -0,0 +1,131 @@ +package face + +import ( + "os" + "path/filepath" + "testing" + + "github.com/photoprism/photoprism/pkg/fastwalk" + "github.com/stretchr/testify/assert" +) + +func TestDetect(t *testing.T) { + expected := map[string]int{ + "1.jpg": 1, + "2.jpg": 1, + "3.jpg": 1, + "4.jpg": 1, + "5.jpg": 1, + "6.jpg": 1, + "7.jpg": 0, + "8.jpg": 0, + "9.jpg": 0, + "10.jpg": 0, + "11.jpg": 0, + "12.jpg": 1, + "13.jpg": 0, + "14.jpg": 0, + "15.jpg": 0, + "16.jpg": 1, + "17.jpg": 1, + "18.jpg": 2, + "19.jpg": 0, + } + + if err := fastwalk.Walk("testdata", func(fileName string, info os.FileMode) error { + if info.IsDir() || filepath.Base(filepath.Dir(fileName)) != "testdata" { + return nil + } + + t.Run(fileName, func(t *testing.T) { + baseName := filepath.Base(fileName) + + faces, err := Detect(fileName, true, 20) + + if err != nil { + t.Fatal(err) + } + + t.Logf("found %d faces in '%s'", len(faces), baseName) + + if len(faces) > 0 { + // t.Logf("results: %#v", faces) + + for i, f := range faces { + t.Logf("marker[%d]: %#v %#v", i, f.CropArea(), f.Area) + t.Logf("landmarks[%d]: %s", i, f.RelativeLandmarksJSON()) + } + } + + if i, ok := expected[baseName]; ok { + assert.Equal(t, i, faces.Count()) + + if faces.Count() == 0 { + assert.Equal(t, 100, faces.Uncertainty()) + } else { + assert.Truef(t, faces.Uncertainty() >= 0 && faces.Uncertainty() <= 50, "uncertainty should be between 0 and 50") + } + t.Logf("uncertainty: %d", faces.Uncertainty()) + } else { + t.Logf("unknown test result for %s", baseName) + } + }) + + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestDetectOverlap(t *testing.T) { + expected := map[string]int{ + "1.jpg": 2, + "2.jpg": 2, + "3.jpg": 2, + "4.jpg": 1, + } + + if err := fastwalk.Walk("testdata/overlap", func(fileName string, info os.FileMode) error { + if info.IsDir() || filepath.Base(filepath.Dir(fileName)) != "overlap" { + return nil + } + + t.Run(fileName, func(t *testing.T) { + baseName := filepath.Base(fileName) + + faces, err := Detect(fileName, true, 20) + + if err != nil { + t.Fatal(err) + } + + t.Logf("found %d faces in '%s'", len(faces), baseName) + + if len(faces) > 0 { + // t.Logf("results: %#v", faces) + + for i, f := range faces { + t.Logf("marker[%d]: %#v %#v", i, f.CropArea(), f.Area) + t.Logf("landmarks[%d]: %s", i, f.RelativeLandmarksJSON()) + } + } + + if i, ok := expected[baseName]; ok { + assert.Equal(t, i, faces.Count()) + + if faces.Count() == 0 { + assert.Equal(t, 100, faces.Uncertainty()) + } else { + assert.Truef(t, faces.Uncertainty() >= 0 && faces.Uncertainty() <= 50, "uncertainty should be between 0 and 50") + } + t.Logf("uncertainty: %d", faces.Uncertainty()) + } else { + t.Logf("unknown test result for %s", baseName) + } + }) + + return nil + }); err != nil { + t.Fatal(err) + } +} diff --git a/internal/face/face.go b/internal/face/face.go index 3acc76531..fdad7a648 100644 --- a/internal/face/face.go +++ b/internal/face/face.go @@ -39,18 +39,39 @@ import ( "github.com/photoprism/photoprism/internal/event" ) +var log = event.Log + var CropSize = crop.Sizes[crop.Tile160] var ClusterCore = 4 var ClusterRadius = 0.6 -var ClusterMinScore = 30 +var ClusterMinScore = 15 var ClusterMinSize = 100 var SampleThreshold = 2 * ClusterCore - -var log = event.Log +var OverlapThreshold = 40 +var OverlapThresholdFloor = OverlapThreshold - 1 +var ScoreThreshold = float32(8.5) // Faces is a list of face detection results. type Faces []Face +// Contains returns true if the face conflicts with existing faces. +func (faces Faces) Contains(other Face) bool { + cropArea := other.CropArea() + + for _, f := range faces { + if f.CropArea().OverlapPercent(cropArea) > OverlapThresholdFloor { + return true + } + } + + return false +} + +// Append adds a face. +func (faces *Faces) Append(f Face) { + *faces = append(*faces, f) +} + // Count returns the number of faces detected. func (faces Faces) Count() int { return len(faces) diff --git a/internal/face/face_test.go b/internal/face/face_test.go index fad078065..f42ab3708 100644 --- a/internal/face/face_test.go +++ b/internal/face/face_test.go @@ -1,159 +1,11 @@ package face import ( - "encoding/json" - "os" - "path/filepath" - "strings" "testing" - "github.com/photoprism/photoprism/internal/crop" - - "github.com/photoprism/photoprism/pkg/fastwalk" "github.com/stretchr/testify/assert" ) -var modelPath, _ = filepath.Abs("../../assets/facenet") - -func TestDetect(t *testing.T) { - expected := map[string]int{ - "1.jpg": 1, - "2.jpg": 1, - "3.jpg": 1, - "4.jpg": 1, - "5.jpg": 1, - "6.jpg": 1, - "7.jpg": 0, - "8.jpg": 0, - "9.jpg": 0, - "10.jpg": 0, - "11.jpg": 0, - "12.jpg": 1, - "13.jpg": 0, - "14.jpg": 0, - "15.jpg": 0, - "16.jpg": 1, - "17.jpg": 1, - "18.jpg": 2, - "19.jpg": 0, - } - - faceindices := map[string][]int{ - "18.jpg": {0, 1}, - "1.jpg": {2}, - "4.jpg": {3}, - "5.jpg": {4}, - "6.jpg": {5}, - "2.jpg": {6}, - "12.jpg": {7}, - "16.jpg": {8}, - "17.jpg": {9}, - "3.jpg": {10}, - } - - faceindexToPersonid := [11]int{ - 0, 1, 1, 1, 2, 0, 1, 0, 0, 1, 0, - } - - var embeddings [11][]float32 - - faceNet := NewNet(modelPath, "testdata/cache", false) - - if err := faceNet.loadModel(); err != nil { - t.Fatal(err) - } - - if err := fastwalk.Walk("testdata", func(fileName string, info os.FileMode) error { - if info.IsDir() || strings.HasPrefix(filepath.Base(fileName), ".") || strings.Contains(fileName, "cache") { - return nil - } - - t.Run(fileName, func(t *testing.T) { - baseName := filepath.Base(fileName) - - faces, err := Detect(fileName, true, 20) - - if err != nil { - t.Fatal(err) - } - - t.Logf("found %d faces in '%s'", len(faces), baseName) - - if len(faces) > 0 { - // t.Logf("results: %#v", faces) - - for i, f := range faces { - t.Logf("marker[%d]: %#v %#v", i, f.CropArea(), f.Area) - t.Logf("landmarks[%d]: %s", i, f.RelativeLandmarksJSON()) - - img, err := crop.ImageFromThumb(fileName, f.CropArea(), CropSize, false) - - if err != nil { - t.Fatal(err) - } - - embedding := faceNet.getEmbeddings(img) - - if b, err := json.Marshal(embedding[0]); err != nil { - t.Fatal(err) - } else { - assert.NotEmpty(t, b) - // t.Logf("embedding: %#v", string(b)) - } - - t.Logf("faces: %d %v", i, faceindices[baseName]) - embeddings[faceindices[baseName][i]] = embedding[0] - } - } - - if i, ok := expected[baseName]; ok { - assert.Equal(t, i, len(faces)) - assert.Equal(t, i, faces.Count()) - if faces.Count() == 0 { - assert.Equal(t, 100, faces.Uncertainty()) - } else { - assert.Truef(t, faces.Uncertainty() >= 0 && faces.Uncertainty() <= 50, "uncertainty should be between 0 and 50") - } - t.Logf("uncertainty: %d", faces.Uncertainty()) - } else { - t.Logf("unknown test result for %s", baseName) - } - }) - - return nil - }); err != nil { - t.Fatal(err) - } - - // Distance Matrix - correct := 0 - - for i := 0; i < len(embeddings); i++ { - for j := 0; j < len(embeddings); j++ { - if i >= j { - continue - } - dist := EuclidianDistance(embeddings[i], embeddings[j]) - t.Logf("Dist for %d %d (faces are %d %d) is %f", i, j, faceindexToPersonid[i], faceindexToPersonid[j], dist) - if faceindexToPersonid[i] == faceindexToPersonid[j] { - if dist < 1.21 { - correct += 1 - } - } else { - if dist >= 1.21 { - correct += 1 - } - } - } - } - - t.Logf("Correct for %d", correct) - - // there are a few incorrect results - // 4 out of 55 with the 1.21 threshold - assert.True(t, correct == 51) -} - func TestFaces_Uncertainty(t *testing.T) { t.Run("maxScore = 310", func(t *testing.T) { f := Faces{Face{Score: 310}, Face{Score: 210}} @@ -269,3 +121,125 @@ func TestFace_CropArea(t *testing.T) { t.Logf("marker: %#v", f.CropArea()) }) } + +func TestFaces_Contains(t *testing.T) { + t.Run("Contained", func(t *testing.T) { + a := Face{ + Cols: 1000, + Rows: 600, + Score: 125, + Area: Area{ + Name: "face", + Col: 400, + Row: 250, + Scale: 200, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + b := Face{ + Cols: 1000, + Rows: 600, + Score: 34, + Area: Area{ + Name: "face", + Col: 100, + Row: 100, + Scale: 50, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + c := Face{ + Cols: 1000, + Rows: 600, + Score: 125, + Area: Area{ + Name: "face", + Col: 125, + Row: 125, + Scale: 25, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + d := Face{ + Cols: 1000, + Rows: 600, + Score: 125, + Area: Area{ + Name: "face", + Col: 110, + Row: 110, + Scale: 50, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + faces := Faces{a, b} + + assert.True(t, faces.Contains(a)) + assert.True(t, faces.Contains(b)) + assert.False(t, faces.Contains(c)) + assert.True(t, faces.Contains(d)) + }) + t.Run("NotContained", func(t *testing.T) { + a := Face{ + Cols: 1000, + Rows: 600, + Score: 125, + Area: Area{ + Name: "face", + Col: 400, + Row: 250, + Scale: 200, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + b := Face{ + Cols: 1000, + Rows: 600, + Score: 34, + Area: Area{ + Name: "face", + Col: 100, + Row: 100, + Scale: 50, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + c := Face{ + Cols: 1000, + Rows: 600, + Score: 125, + Area: Area{ + Name: "face", + Col: 900, + Row: 500, + Scale: 25, + }, + Eyes: nil, + Landmarks: nil, + Embeddings: nil, + } + + faces := Faces{a} + + assert.False(t, faces.Contains(b)) + assert.False(t, faces.Contains(c)) + }) +} diff --git a/internal/face/net_test.go b/internal/face/net_test.go index fd73d1883..bbd581c88 100644 --- a/internal/face/net_test.go +++ b/internal/face/net_test.go @@ -3,13 +3,14 @@ package face import ( "os" "path/filepath" - "strings" "testing" "github.com/photoprism/photoprism/pkg/fastwalk" "github.com/stretchr/testify/assert" ) +var modelPath, _ = filepath.Abs("../../assets/facenet") + func TestNet(t *testing.T) { expected := map[string]int{ "1.jpg": 1, @@ -34,7 +35,7 @@ func TestNet(t *testing.T) { } faceindices := map[string][]int{ - "18.jpg": {0, 1}, + "18.jpg": {1, 0}, "1.jpg": {2}, "4.jpg": {3}, "5.jpg": {4}, @@ -55,7 +56,7 @@ func TestNet(t *testing.T) { faceNet := NewNet(modelPath, "testdata/cache", false) if err := fastwalk.Walk("testdata", func(fileName string, info os.FileMode) error { - if info.IsDir() || strings.HasPrefix(filepath.Base(fileName), ".") || strings.Contains(fileName, "cache") { + if info.IsDir() || filepath.Base(filepath.Dir(fileName)) != "testdata" { return nil } @@ -68,11 +69,11 @@ func TestNet(t *testing.T) { t.Fatal(err) } - t.Logf("found %d faces in '%s'", len(faces), baseName) + // for i, f := range faces { + // t.Logf("FACE %d IN %s: %#v", i, fileName, f.Area) + // } if len(faces) > 0 { - // t.Logf("results: %#v", faces) - for i, f := range faces { if len(f.Embeddings) > 0 { embeddings[faceindices[baseName][i]] = f.Embeddings[0] @@ -83,8 +84,8 @@ func TestNet(t *testing.T) { } if i, ok := expected[baseName]; ok { - assert.Equal(t, i, len(faces)) assert.Equal(t, i, faces.Count()) + if faces.Count() == 0 { assert.Equal(t, 100, faces.Uncertainty()) } else { @@ -127,5 +128,5 @@ func TestNet(t *testing.T) { // there are a few incorrect results // 4 out of 55 with the 1.21 threshold - assert.True(t, correct == 51) + assert.Equal(t, 51, correct) } diff --git a/internal/face/testdata/overlap/1.jpg b/internal/face/testdata/overlap/1.jpg new file mode 100644 index 000000000..fb7831493 Binary files /dev/null and b/internal/face/testdata/overlap/1.jpg differ diff --git a/internal/face/testdata/overlap/2.jpg b/internal/face/testdata/overlap/2.jpg new file mode 100644 index 000000000..1f281e7d4 Binary files /dev/null and b/internal/face/testdata/overlap/2.jpg differ diff --git a/internal/face/testdata/overlap/3.jpg b/internal/face/testdata/overlap/3.jpg new file mode 100644 index 000000000..03298b266 Binary files /dev/null and b/internal/face/testdata/overlap/3.jpg differ diff --git a/internal/face/testdata/overlap/4.jpg b/internal/face/testdata/overlap/4.jpg new file mode 100644 index 000000000..1d82519ac Binary files /dev/null and b/internal/face/testdata/overlap/4.jpg differ