luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

People: Refactor face marker indexing #22

Browse source
This commit is contained in:
Michael Mayer 2021-05-31 15:40:52 +02:00
parent 7ffd9f7b9d
commit 11b4fbd5a0
15 changed files with 208 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
frontend/src/model/file.js
View file

@ -69,6 +69,7 @@ export class File extends RestModel {
Chroma: 0,
Notes: "",
Error: "",
Markers: [],
CreatedAt: "",
CreatedIn: 0,
UpdatedAt: "",

7
internal/classify/label.go
View file

@ -1,6 +1,7 @@
package classify
import (
"github.com/photoprism/photoprism/internal/face"
"strings"
"github.com/photoprism/photoprism/pkg/txt"
@ -49,9 +50,11 @@ func (l Label) Title() string {
}
// FaceLabels returns matching labels if there are people in the image.
func FaceLabels(count int, src string, uncertainty int) Labels {
func FaceLabels(faces face.Faces, src string) Labels {
var r LabelRule
count := faces.Count()
if count < 1 {
return Labels{}
} else if count == 1 {
@ -63,7 +66,7 @@ func FaceLabels(count int, src string, uncertainty int) Labels {
return Labels{Label{
Name: r.Label,
Source: src,
Uncertainty: uncertainty,
Uncertainty: faces.Uncertainty(),
Priority: r.Priority,
Categories: r.Categories,
}}

23
internal/entity/file.go
View file

@ -2,11 +2,12 @@ package entity
import (
"fmt"
"github.com/photoprism/photoprism/internal/face"
"path/filepath"
"strings"
"time"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/pkg/txt"
"github.com/gosimple/slug"
@ -68,7 +69,7 @@ type File struct {
DeletedAt *time.Time `sql:"index" json:"DeletedAt,omitempty" yaml:"-"`
Share []FileShare `json:"-" yaml:"-"`
Sync []FileSync `json:"-" yaml:"-"`
Markers Markers `json:"-" yaml:"-"`
Markers Markers `json:"Markers,omitempty" yaml:"-"`
}
type FileInfos struct {
@ -253,7 +254,7 @@ func (m *File) Create() error {
return err
}
if err := m.Markers.Save(m.FileUID); err != nil {
if err := m.Markers.Save(m.ID); err != nil {
log.Errorf("file: %s (create markers for %s)", err, m.FileUID)
return err
}
@ -281,7 +282,7 @@ func (m *File) Save() error {
return err
}
if err := m.Markers.Save(m.FileUID); err != nil {
if err := m.Markers.Save(m.ID); err != nil {
log.Errorf("file: %s (save markers for %s)", err, m.FileUID)
return err
}
@ -405,5 +406,17 @@ func (m *File) AddFaces(faces face.Faces) {
// AddFace adds a face marker to the file.
func (m *File) AddFace(f face.Face, refUID string) {
m.Markers = append(m.Markers, *NewFaceMarker(f, m.FileUID, refUID))
marker := NewFaceMarker(f, m.ID, refUID)
if !m.Markers.Contains(*marker) {
m.Markers = append(m.Markers, *marker)
}
}
// PreloadMarkers loads existing file markers.
func (m *File) PreloadMarkers() {
if res, err := FindMarkers(m.ID); err != nil {
log.Warnf("file: %s (load markers)", err)
} else {
m.Markers = res
}
}

3
internal/entity/file_test.go
View file

@ -1,10 +1,11 @@
package entity
import (
"github.com/photoprism/photoprism/internal/face"
"testing"
"time"
"github.com/photoprism/photoprism/internal/face"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/stretchr/testify/assert"
)

51
internal/entity/marker.go
View file

@ -2,8 +2,9 @@ package entity
import (
"fmt"
"github.com/photoprism/photoprism/internal/face"
"time"
"github.com/photoprism/photoprism/internal/face"
)
const (
@ -14,21 +15,21 @@ const (
// Marker represents an image marker point.
type Marker struct {
ID uint `gorm:"primary_key" json:"ID" yaml:"-"`
FileUID string `gorm:"type:VARBINARY(42);index;" json:"FileUID" yaml:"FileUID"`
RefUID string `gorm:"type:VARBINARY(42);index;" json:"UID" yaml:"UID,omitempty"`
MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"`
MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"`
MarkerScore int `gorm:"type:SMALLINT"`
MarkerLabel string `gorm:"type:VARCHAR(255);" json:"Label" yaml:"Label,omitempty"`
MarkerMeta string `gorm:"type:TEXT;" json:"Meta" yaml:"Meta,omitempty"`
X float32 `gorm:"type:FLOAT;" json:"X" yaml:"X,omitempty"`
Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"`
W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"`
H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"`
File *File
CreatedAt time.Time
UpdatedAt time.Time
ID uint `gorm:"primary_key" json:"ID" yaml:"-"`
FileID uint `gorm:"index;" json:"-" yaml:"-"`
RefUID string `gorm:"type:VARBINARY(42);index;" json:"RefUID" yaml:"RefUID,omitempty"`
MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"`
MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"`
MarkerScore int `gorm:"type:SMALLINT" json:"Score" yaml:"Score"`
MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"`
MarkerLabel string `gorm:"type:VARCHAR(255);" json:"Label" yaml:"Label,omitempty"`
MarkerMeta string `gorm:"type:TEXT;" json:"Meta" yaml:"Meta,omitempty"`
X float32 `gorm:"type:FLOAT;" json:"X" yaml:"X,omitempty"`
Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"`
W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"`
H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"`
CreatedAt time.Time
UpdatedAt time.Time
}
// TableName returns the entity database table name.
@ -37,9 +38,9 @@ func (Marker) TableName() string {
}
// NewMarker creates a new entity.
func NewMarker(fileUID, refUID, markerSrc, markerType string, x, y, w, h float32) *Marker {
func NewMarker(fileUID uint, refUID, markerSrc, markerType string, x, y, w, h float32) *Marker {
m := &Marker{
FileUID: fileUID,
FileID: fileUID,
RefUID: refUID,
MarkerSrc: markerSrc,
MarkerType: markerType,
@ -53,9 +54,9 @@ func NewMarker(fileUID, refUID, markerSrc, markerType string, x, y, w, h float32
}
// NewFaceMarker creates a new entity.
func NewFaceMarker(f face.Face, fileUID, refUID string) *Marker {
func NewFaceMarker(f face.Face, fileID uint, refUID string) *Marker {
fm := f.Marker()
m := NewMarker(fileUID, refUID, SrcImage, MarkerFace, fm.X, fm.Y, fm.W, fm.H)
m := NewMarker(fileID, refUID, SrcImage, MarkerFace, fm.X, fm.Y, fm.W, fm.H)
m.MarkerScore = f.Score
m.MarkerMeta = string(f.RelativeLandmarksJSON())
@ -97,10 +98,10 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) {
if m.ID > 0 {
err := m.Save()
log.Debugf("faces: saved marker %d for file %s", m.ID, m.FileUID)
log.Debugf("faces: saved marker %d for file %d", m.ID, m.FileID)
return m, err
} else if err := Db().Where(`file_uid = ? AND x > ? AND x < ? AND y > ? AND y < ?`,
m.FileUID, m.X-m.W, m.X+m.W, m.Y-m.H, m.Y+m.H).First(&result).Error; err == nil {
} else if err := Db().Where(`file_id = ? AND x > ? AND x < ? AND y > ? AND y < ?`,
m.FileID, m.X-m.W, m.X+m.W, m.Y-m.H, m.Y+m.H).First(&result).Error; err == nil {
if SrcPriority[m.MarkerSrc] < SrcPriority[result.MarkerSrc] {
// Ignore.
@ -117,11 +118,11 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) {
"RefUID": m.RefUID,
})
log.Debugf("faces: updated existing marker %d for file %s", result.ID, result.FileUID)
log.Debugf("faces: updated existing marker %d for file %d", result.ID, result.FileID)
return &result, err
} else if err := m.Create(); err != nil {
log.Debugf("faces: added marker %d for file %s", m.ID, m.FileUID)
log.Debugf("faces: added marker %d for file %d", m.ID, m.FileID)
return m, err
}

21
internal/entity/marker_test.go
View file

@ -12,9 +12,9 @@ func TestMarker_TableName(t *testing.T) {
}
func TestNewMarker(t *testing.T) {
m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
assert.IsType(t, &Marker{}, m)
assert.Equal(t, "ft8es39w45bnlqdw", m.FileUID)
assert.Equal(t, uint(1000000), m.FileID)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID)
assert.Equal(t, SrcImage, m.MarkerSrc)
assert.Equal(t, MarkerLabel, m.MarkerType)
@ -22,9 +22,9 @@ func TestNewMarker(t *testing.T) {
func TestUpdateOrCreateMarker(t *testing.T) {
t.Run("success", func(t *testing.T) {
m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
assert.IsType(t, &Marker{}, m)
assert.Equal(t, "ft8es39w45bnlqdw", m.FileUID)
assert.Equal(t, uint(1000000), m.FileID)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID)
assert.Equal(t, SrcImage, m.MarkerSrc)
assert.Equal(t, MarkerLabel, m.MarkerType)
@ -47,7 +47,7 @@ func TestUpdateOrCreateMarker(t *testing.T) {
func TestMarker_Updates(t *testing.T) {
t.Run("success", func(t *testing.T) {
m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m, err := UpdateOrCreateMarker(m)
if err != nil {
@ -72,7 +72,7 @@ func TestMarker_Updates(t *testing.T) {
func TestMarker_Update(t *testing.T) {
t.Run("success", func(t *testing.T) {
m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m, err := UpdateOrCreateMarker(m)
if err != nil {
@ -96,7 +96,7 @@ func TestMarker_Update(t *testing.T) {
func TestMarker_Save(t *testing.T) {
t.Run("success", func(t *testing.T) {
m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556)
m, err := UpdateOrCreateMarker(m)
if err != nil {
@ -123,5 +123,12 @@ func TestMarker_Save(t *testing.T) {
if m.ID <= 0 {
t.Errorf("ID should be > 0")
}
p := PhotoFixtures.Get("19800101_000002_D640C559")
assert.Empty(t, p.Files)
p.PreloadFiles(true)
assert.NotEmpty(t, p.Files)
t.Logf("FILES: %#v", p.Files)
})
}

38
internal/entity/markers.go
View file

@ -3,9 +3,12 @@ package entity
type Markers []Marker
// Save stores the markers in the database.
func (m Markers) Save(fileUID string) error {
func (m Markers) Save(fileID uint) error {
for _, marker := range m {
marker.FileUID = fileUID
if fileID > 0 {
marker.FileID = fileID
}
if _, err := UpdateOrCreateMarker(&marker); err != nil {
return err
}
@ -13,3 +16,34 @@ func (m Markers) Save(fileUID string) error {
return nil
}
// Contains returns true if a marker at the same position already exists.
func (m Markers) Contains(m2 Marker) bool {
for _, m1 := range m {
if m2.X > (m1.X-m1.W) && m2.X < (m1.X+m1.W) && m2.Y > (m1.Y-m1.H) && m2.Y < (m1.Y+m1.H) {
return true
}
}
return false
}
// FaceCount returns the number of valid face markers.
func (m Markers) FaceCount() int {
result := 0
for _, marker := range m {
if !marker.MarkerInvalid && marker.MarkerType == MarkerFace {
result++
}
}
return result
}
// FindMarkers returns all markers for a given file id.
func FindMarkers(fileID uint) (Markers, error) {
m := Markers{}
err := Db().Where(`file_id = ?`, fileID).Order("id").Offset(0).Limit(1000).Find(&m).Error
return m, err
}

29
internal/entity/markers_test.go Normal file
View file

@ -0,0 +1,29 @@
package entity
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMarkers_Contains(t *testing.T) {
m1 := *NewMarker(1000000, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 0.308333, 0.206944, 0.355556, 0.355556)
m2 := *NewMarker(1000000, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 0.298133, 0.216944, 0.255556, 0.155556)
m3 := *NewMarker(1000000, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 0.998133, 0.816944, 0.0001, 0.0001)
m := Markers{m1}
assert.True(t, m.Contains(m2))
assert.False(t, m.Contains(m3))
}
func TestMarkers_FaceCount(t *testing.T) {
m1 := *NewMarker(1000000, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 0.308333, 0.206944, 0.355556, 0.355556)
m2 := *NewMarker(1000000, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 0.298133, 0.216944, 0.255556, 0.155556)
m3 := *NewMarker(1000000, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 0.998133, 0.816944, 0.0001, 0.0001)
m3.MarkerInvalid = true
m := Markers{m1, m2, m3}
assert.Equal(t, 2, m.FaceCount())
}

12
internal/entity/photo.go
View file

@ -439,14 +439,20 @@ func (m *Photo) IndexKeywords() error {
}
// PreloadFiles prepares gorm scope to retrieve photo file
func (m *Photo) PreloadFiles() {
func (m *Photo) PreloadFiles(markers bool) {
q := Db().
Table("files").
Select(`files.*`).
Where("files.deleted_at IS NULL AND files.photo_id = ?", m.ID).
Where("files.photo_id = ? AND files.deleted_at IS NULL", m.ID).
Order("files.file_name DESC")
logError(q.Scan(&m.Files))
if markers {
for i := range m.Files {
m.Files[i].PreloadMarkers()
}
}
}
// PreloadKeywords prepares gorm scope to retrieve photo keywords
@ -474,7 +480,7 @@ func (m *Photo) PreloadAlbums() {
// PreloadMany prepares gorm scope to retrieve photo file, albums and keywords
func (m *Photo) PreloadMany() {
m.PreloadFiles()
m.PreloadFiles(true)
m.PreloadKeywords()
m.PreloadAlbums()
}

2
internal/entity/photo_test.go
View file

@ -145,7 +145,7 @@ func TestPhoto_PreloadFiles(t *testing.T) {
t.Run("success", func(t *testing.T) {
m := PhotoFixtures.Get("Photo01")
assert.Empty(t, m.Files)
m.PreloadFiles()
m.PreloadFiles(false)
assert.NotEmpty(t, m.Files)
})
}

6
internal/entity/photo_yaml_test.go
View file

@ -11,7 +11,7 @@ import (
func TestPhoto_Yaml(t *testing.T) {
t.Run("create from fixture", func(t *testing.T) {
m := PhotoFixtures.Get("Photo01")
m.PreloadFiles()
m.PreloadFiles(true)
result, err := m.Yaml()
if err != nil {
@ -25,7 +25,7 @@ func TestPhoto_Yaml(t *testing.T) {
func TestPhoto_SaveAsYaml(t *testing.T) {
t.Run("create from fixture", func(t *testing.T) {
m := PhotoFixtures.Get("Photo01")
m.PreloadFiles()
m.PreloadFiles(true)
fileName := filepath.Join(os.TempDir(), ".photoprism_test.yml")
@ -46,7 +46,7 @@ func TestPhoto_SaveAsYaml(t *testing.T) {
func TestPhoto_YamlFileName(t *testing.T) {
t.Run("create from fixture", func(t *testing.T) {
m := PhotoFixtures.Get("Photo01")
m.PreloadFiles()
m.PreloadFiles(false)
assert.Equal(t, "xxx/2790/02/yyy/Photo01.yml", m.YamlFileName("xxx", "yyy"))
if err := os.RemoveAll("xxx"); err != nil {

7
internal/face/detector.go
View file

@ -3,14 +3,15 @@ package face
import (
_ "embed"
"fmt"
pigo "github.com/esimov/pigo/core"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/txt"
_ "image/jpeg"
"io"
"os"
"path/filepath"
"runtime/debug"
pigo "github.com/esimov/pigo/core"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/txt"
)
//go:embed cascade/facefinder

45
internal/face/face.go
View file

@ -43,6 +43,51 @@ var log = event.Log
// Faces is a list of face detection results.
type Faces []Face
// Count returns the number of faces detected.
func (faces Faces) Count() int {
return len(faces)
}
// Uncertainty return the max face detection uncertainty in percent.
func (faces Faces) Uncertainty() int {
if len(faces) < 1 {
return 100
}
maxScore := 0
for _, f := range faces {
if f.Score > maxScore {
maxScore = f.Score
}
}
switch {
case maxScore > 300:
return 1
case maxScore > 200:
return 5
case maxScore > 100:
return 10
case maxScore > 80:
return 15
case maxScore > 65:
return 20
case maxScore > 50:
return 25
case maxScore > 40:
return 30
case maxScore > 30:
return 35
case maxScore > 20:
return 40
case maxScore > 10:
return 45
}
return 50
}
// Face represents a face detection result.
type Face struct {
Rows int `json:"rows,omitempty"`

9
internal/face/face_test.go
View file

@ -60,8 +60,15 @@ func TestDetect(t *testing.T) {
if i, ok := expected[baseName]; ok {
assert.Equal(t, i, len(faces))
assert.Equal(t, i, faces.Count())
if faces.Count() == 0 {
assert.Equal(t, 100, faces.Uncertainty())
} else {
assert.Truef(t, faces.Uncertainty() >= 0 && faces.Uncertainty() <= 50, "uncertainty should be between 0 and 50")
}
t.Logf("uncertainty: %d", faces.Uncertainty())
} else {
t.Errorf("unknown test result for %s", baseName)
t.Logf("unknown test result for %s", baseName)
}
})

11
internal/photoprism/index_mediafile.go
View file

@ -600,19 +600,22 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
// Main JPEG file.
if file.FilePrimary {
labels := photo.ClassifyLabels()
if Config().Experimental() && Config().Settings().Features.People {
faces := ind.detectFaces(m)
photo.AddLabels(classify.FaceLabels(len(faces), entity.SrcImage, 10))
photo.PhotoFaces = len(faces)
photo.AddLabels(classify.FaceLabels(faces, entity.SrcImage))
file.PreloadMarkers()
if len(faces) > 0 {
file.AddFaces(faces)
}
photo.PhotoFaces = file.Markers.FaceCount()
}
labels := photo.ClassifyLabels()
if err := photo.UpdateTitle(labels); err != nil {
log.Debugf("%s in %s (update title)", err, logName)
}