diff --git a/frontend/src/pages/settings/general.vue b/frontend/src/pages/settings/general.vue index 342698c29..017f83a5d 100644 --- a/frontend/src/pages/settings/general.vue +++ b/frontend/src/pages/settings/general.vue @@ -241,7 +241,7 @@ - + qThresh { - faceCoord := NewPoint( - "face", - face.Row-face.Scale/2, - face.Col-face.Scale/2, - face.Scale, - ) + for _, face := range det { + if face.Q < fd.scoreThreshold { + continue + } - if face.Scale > 50 { - // Find left eye. - puploc = &pigo.Puploc{ - Row: face.Row - int(0.075*float32(face.Scale)), - Col: face.Col - int(0.175*float32(face.Scale)), - Scale: float32(face.Scale) * 0.25, - Perturbs: perturb, - } + faceCoord := NewPoint( + "face", + face.Row-face.Scale/2, + face.Col-face.Scale/2, + face.Scale, + ) - leftEye := plc.RunDetector(*puploc, params.ImageParams, fd.angle, false) + if face.Scale > 50 { + // Find left eye. + puploc = &pigo.Puploc{ + Row: face.Row - int(0.075*float32(face.Scale)), + Col: face.Col - int(0.175*float32(face.Scale)), + Scale: float32(face.Scale) * 0.25, + Perturbs: fd.perturb, + } - if leftEye.Row > 0 && leftEye.Col > 0 { - eyesCoords = append(eyesCoords, NewPoint( - "eye_l", - leftEye.Row, - leftEye.Col, - int(leftEye.Scale), - )) - } + leftEye := plc.RunDetector(*puploc, params.ImageParams, fd.angle, false) - // Find right eye. - puploc = &pigo.Puploc{ - Row: face.Row - int(0.075*float32(face.Scale)), - Col: face.Col + int(0.185*float32(face.Scale)), - Scale: float32(face.Scale) * 0.25, - Perturbs: perturb, - } + if leftEye.Row > 0 && leftEye.Col > 0 { + eyesCoords = append(eyesCoords, NewPoint( + "eye_l", + leftEye.Row, + leftEye.Col, + int(leftEye.Scale), + )) + } - rightEye := plc.RunDetector(*puploc, params.ImageParams, fd.angle, false) + // Find right eye. + puploc = &pigo.Puploc{ + Row: face.Row - int(0.075*float32(face.Scale)), + Col: face.Col + int(0.185*float32(face.Scale)), + Scale: float32(face.Scale) * 0.25, + Perturbs: fd.perturb, + } - if rightEye.Row > 0 && rightEye.Col > 0 { - eyesCoords = append(eyesCoords, NewPoint( - "eye_r", - rightEye.Row, - rightEye.Col, - int(rightEye.Scale), - )) - } + rightEye := plc.RunDetector(*puploc, params.ImageParams, fd.angle, false) + if rightEye.Row > 0 && rightEye.Col > 0 { + eyesCoords = append(eyesCoords, NewPoint( + "eye_r", + rightEye.Row, + rightEye.Col, + int(rightEye.Scale), + )) + } + + if leftEye != nil && rightEye != nil { for _, eye := range eyeCascades { for _, flpc := range flpcs[eye] { - flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, perturb, false) + if flpc == nil { + continue + } + + flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, fd.perturb, false) if flp.Row > 0 && flp.Col > 0 { landmarkCoords = append(landmarkCoords, NewPoint( eye, @@ -230,7 +236,7 @@ func (fd *Detector) Results(faces []pigo.Detection, params pigo.CascadeParams) ( )) } - flp = flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, perturb, true) + flp = flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, fd.perturb, true) if flp.Row > 0 && flp.Col > 0 { landmarkCoords = append(landmarkCoords, NewPoint( eye+"_v", @@ -241,22 +247,31 @@ func (fd *Detector) Results(faces []pigo.Detection, params pigo.CascadeParams) ( } } } + } - // Find mouth. - for _, mouth := range mouthCascades { - for _, flpc := range flpcs[mouth] { - flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, perturb, false) - if flp.Row > 0 && flp.Col > 0 { - landmarkCoords = append(landmarkCoords, NewPoint( - "mouth_"+mouth, - flp.Row, - flp.Col, - int(flp.Scale), - )) - } + // Find mouth. + for _, mouth := range mouthCascades { + for _, flpc := range flpcs[mouth] { + if flpc == nil { + continue + } + + flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, fd.perturb, false) + if flp.Row > 0 && flp.Col > 0 { + landmarkCoords = append(landmarkCoords, NewPoint( + "mouth_"+mouth, + flp.Row, + flp.Col, + int(flp.Scale), + )) } } - flp := flpcs["lp84"][0].GetLandmarkPoint(leftEye, rightEye, params.ImageParams, perturb, true) + } + + flpc := flpcs["lp84"][0] + + if flpc != nil { + flp := flpc.GetLandmarkPoint(leftEye, rightEye, params.ImageParams, fd.perturb, true) if flp.Row > 0 && flp.Col > 0 { landmarkCoords = append(landmarkCoords, NewPoint( "lp84", @@ -266,16 +281,18 @@ func (fd *Detector) Results(faces []pigo.Detection, params pigo.CascadeParams) ( )) } } - - detections = append(detections, Face{ - Rows: params.ImageParams.Rows, - Cols: params.ImageParams.Cols, - Face: faceCoord, - Eyes: eyesCoords, - Landmarks: landmarkCoords, - }) } + + results = append(results, Face{ + Rows: params.ImageParams.Rows, + Cols: params.ImageParams.Cols, + Score: int(face.Q), + Face: faceCoord, + Eyes: eyesCoords, + Landmarks: landmarkCoords, + }) + } - return detections, nil + return results, nil } diff --git a/internal/face/face.go b/internal/face/face.go index e71271b92..ef8813ac7 100644 --- a/internal/face/face.go +++ b/internal/face/face.go @@ -47,6 +47,7 @@ type Faces []Face type Face struct { Rows int `json:"rows,omitempty"` Cols int `json:"cols,omitempty"` + Score int `json:"score,omitempty"` Face Point `json:"face,omitempty"` Eyes Points `json:"eyes,omitempty"` Landmarks Points `json:"landmarks,omitempty"` diff --git a/internal/face/face_test.go b/internal/face/face_test.go index e7ccbb4bf..4780542fb 100644 --- a/internal/face/face_test.go +++ b/internal/face/face_test.go @@ -16,7 +16,7 @@ func TestDetect(t *testing.T) { "2.jpg": 1, "3.jpg": 1, "4.jpg": 1, - "5.jpg": 2, + "5.jpg": 1, "6.jpg": 1, "7.jpg": 0, "8.jpg": 0, @@ -30,6 +30,7 @@ func TestDetect(t *testing.T) { "16.jpg": 1, "17.jpg": 1, "18.jpg": 2, + "19.jpg": 0, } if err := fastwalk.Walk("testdata", func(fileName string, info os.FileMode) error { @@ -40,7 +41,7 @@ func TestDetect(t *testing.T) { t.Run(fileName, func(t *testing.T) { baseName := filepath.Base(fileName) - faces, err := Detect(fileName, DefaultDetector()) + faces, err := Detect(fileName) if err != nil { t.Fatal(err) @@ -58,7 +59,7 @@ func TestDetect(t *testing.T) { } if i, ok := expected[baseName]; ok { - assert.Equal(t, len(faces), i) + assert.Equal(t, i, len(faces)) } else { t.Errorf("unknown test result for %s", baseName) } diff --git a/internal/face/landmarks.go b/internal/face/landmarks.go index bbb548692..38e3d924d 100644 --- a/internal/face/landmarks.go +++ b/internal/face/landmarks.go @@ -1,12 +1,16 @@ package face import ( + "embed" "errors" "path/filepath" pigo "github.com/esimov/pigo/core" ) +//go:embed cascade/lps +var efs embed.FS + // FlpCascade holds the binary representation of the facial landmark points cascade files type FlpCascade struct { *pigo.PuplocCascade @@ -19,7 +23,7 @@ func ReadCascadeDir(plc *pigo.PuplocCascade, path string) (result map[string][]* cascades, err := efs.ReadDir(path) if len(cascades) == 0 { - return nil, errors.New("the cascade directory is empty") + return result, errors.New("the cascade directory is empty") } if err != nil { @@ -27,11 +31,20 @@ func ReadCascadeDir(plc *pigo.PuplocCascade, path string) (result map[string][]* } for _, cascade := range cascades { - cf, err := filepath.Abs(path + "/" + cascade.Name()) + cf := filepath.Join(path, cascade.Name()) + + f, err := efs.ReadFile(cf) + if err != nil { - return nil, err + return result, err } - flpc, err := plc.UnpackFlp(cf) + + flpc, err := plc.UnpackCascade(f) + + if err != nil { + return result, err + } + result[cascade.Name()] = append(result[cascade.Name()], &FlpCascade{flpc, err}) } diff --git a/internal/face/testdata/19.jpg b/internal/face/testdata/19.jpg new file mode 100644 index 000000000..4d2d049fb Binary files /dev/null and b/internal/face/testdata/19.jpg differ diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index 6842e3299..209f73f22 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -9,9 +9,11 @@ import ( "time" "github.com/jinzhu/gorm" + "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/internal/meta" "github.com/photoprism/photoprism/internal/nsfw" "github.com/photoprism/photoprism/internal/query" @@ -600,6 +602,13 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( if file.FilePrimary { labels := photo.ClassifyLabels() + if Config().Experimental() && Config().Settings().Features.People { + faces := ind.detectFaces(m) + + photo.AddLabels(classify.FaceLabels(len(faces), entity.SrcImage, 10)) + photo.PhotoPeople = len(faces) + } + if err := photo.UpdateTitle(labels); err != nil { log.Debugf("%s in %s (update title)", err, logName) } @@ -759,7 +768,7 @@ func (ind *Index) NSFW(jpeg *MediaFile) bool { return false } -// classifyImage returns all matching labels for a media file. +// classifyImage classifies a JPEG image and returns matching labels. func (ind *Index) classifyImage(jpeg *MediaFile) (results classify.Labels) { start := time.Now() @@ -812,3 +821,31 @@ func (ind *Index) classifyImage(jpeg *MediaFile) (results classify.Labels) { return results } + +// detectFaces detects faces in a JPEG image and returns them. +func (ind *Index) detectFaces(jpeg *MediaFile) face.Faces { + if jpeg == nil { + return face.Faces{} + } + + thumbName, err := jpeg.Thumbnail(Config().ThumbPath(), "fit_720") + + if err != nil { + log.Debugf("%s in %s", err, txt.Quote(jpeg.BaseName())) + return face.Faces{} + } + + start := time.Now() + + faces, err := face.Detect(thumbName) + + if err != nil { + log.Debugf("%s in %s", err, txt.Quote(jpeg.BaseName())) + } + + elapsed := time.Since(start) + + log.Debugf("index: face detection took %s", elapsed) + + return faces +}