diff --git a/frontend/src/model/file.js b/frontend/src/model/file.js index 41e47e353..5c4629255 100644 --- a/frontend/src/model/file.js +++ b/frontend/src/model/file.js @@ -69,6 +69,7 @@ export class File extends RestModel { Chroma: 0, Notes: "", Error: "", + Markers: [], CreatedAt: "", CreatedIn: 0, UpdatedAt: "", diff --git a/internal/classify/label.go b/internal/classify/label.go index 0313cc75d..3b445f232 100644 --- a/internal/classify/label.go +++ b/internal/classify/label.go @@ -1,6 +1,7 @@ package classify import ( + "github.com/photoprism/photoprism/internal/face" "strings" "github.com/photoprism/photoprism/pkg/txt" @@ -49,9 +50,11 @@ func (l Label) Title() string { } // FaceLabels returns matching labels if there are people in the image. -func FaceLabels(count int, src string, uncertainty int) Labels { +func FaceLabels(faces face.Faces, src string) Labels { var r LabelRule + count := faces.Count() + if count < 1 { return Labels{} } else if count == 1 { @@ -63,7 +66,7 @@ func FaceLabels(count int, src string, uncertainty int) Labels { return Labels{Label{ Name: r.Label, Source: src, - Uncertainty: uncertainty, + Uncertainty: faces.Uncertainty(), Priority: r.Priority, Categories: r.Categories, }} diff --git a/internal/entity/file.go b/internal/entity/file.go index ddc836ac2..2e65e5890 100644 --- a/internal/entity/file.go +++ b/internal/entity/file.go @@ -2,11 +2,12 @@ package entity import ( "fmt" - "github.com/photoprism/photoprism/internal/face" "path/filepath" "strings" "time" + "github.com/photoprism/photoprism/internal/face" + "github.com/photoprism/photoprism/pkg/txt" "github.com/gosimple/slug" @@ -68,7 +69,7 @@ type File struct { DeletedAt *time.Time `sql:"index" json:"DeletedAt,omitempty" yaml:"-"` Share []FileShare `json:"-" yaml:"-"` Sync []FileSync `json:"-" yaml:"-"` - Markers Markers `json:"-" yaml:"-"` + Markers Markers `json:"Markers,omitempty" yaml:"-"` } type FileInfos struct { @@ -253,7 +254,7 @@ func (m *File) Create() error { return err } - if err := m.Markers.Save(m.FileUID); err != nil { + if err := m.Markers.Save(m.ID); err != nil { log.Errorf("file: %s (create markers for %s)", err, m.FileUID) return err } @@ -281,7 +282,7 @@ func (m *File) Save() error { return err } - if err := m.Markers.Save(m.FileUID); err != nil { + if err := m.Markers.Save(m.ID); err != nil { log.Errorf("file: %s (save markers for %s)", err, m.FileUID) return err } @@ -405,5 +406,17 @@ func (m *File) AddFaces(faces face.Faces) { // AddFace adds a face marker to the file. func (m *File) AddFace(f face.Face, refUID string) { - m.Markers = append(m.Markers, *NewFaceMarker(f, m.FileUID, refUID)) + marker := NewFaceMarker(f, m.ID, refUID) + if !m.Markers.Contains(*marker) { + m.Markers = append(m.Markers, *marker) + } +} + +// PreloadMarkers loads existing file markers. +func (m *File) PreloadMarkers() { + if res, err := FindMarkers(m.ID); err != nil { + log.Warnf("file: %s (load markers)", err) + } else { + m.Markers = res + } } diff --git a/internal/entity/file_test.go b/internal/entity/file_test.go index 648a9726b..42adf7068 100644 --- a/internal/entity/file_test.go +++ b/internal/entity/file_test.go @@ -1,10 +1,11 @@ package entity import ( - "github.com/photoprism/photoprism/internal/face" "testing" "time" + "github.com/photoprism/photoprism/internal/face" + "github.com/photoprism/photoprism/pkg/fs" "github.com/stretchr/testify/assert" ) diff --git a/internal/entity/marker.go b/internal/entity/marker.go index a8e198199..e87036eb0 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -2,8 +2,9 @@ package entity import ( "fmt" - "github.com/photoprism/photoprism/internal/face" "time" + + "github.com/photoprism/photoprism/internal/face" ) const ( @@ -14,21 +15,21 @@ const ( // Marker represents an image marker point. type Marker struct { - ID uint `gorm:"primary_key" json:"ID" yaml:"-"` - FileUID string `gorm:"type:VARBINARY(42);index;" json:"FileUID" yaml:"FileUID"` - RefUID string `gorm:"type:VARBINARY(42);index;" json:"UID" yaml:"UID,omitempty"` - MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"` - MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"` - MarkerScore int `gorm:"type:SMALLINT"` - MarkerLabel string `gorm:"type:VARCHAR(255);" json:"Label" yaml:"Label,omitempty"` - MarkerMeta string `gorm:"type:TEXT;" json:"Meta" yaml:"Meta,omitempty"` - X float32 `gorm:"type:FLOAT;" json:"X" yaml:"X,omitempty"` - Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"` - W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"` - H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"` - File *File - CreatedAt time.Time - UpdatedAt time.Time + ID uint `gorm:"primary_key" json:"ID" yaml:"-"` + FileID uint `gorm:"index;" json:"-" yaml:"-"` + RefUID string `gorm:"type:VARBINARY(42);index;" json:"RefUID" yaml:"RefUID,omitempty"` + MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"` + MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"` + MarkerScore int `gorm:"type:SMALLINT" json:"Score" yaml:"Score"` + MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"` + MarkerLabel string `gorm:"type:VARCHAR(255);" json:"Label" yaml:"Label,omitempty"` + MarkerMeta string `gorm:"type:TEXT;" json:"Meta" yaml:"Meta,omitempty"` + X float32 `gorm:"type:FLOAT;" json:"X" yaml:"X,omitempty"` + Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"` + W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"` + H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"` + CreatedAt time.Time + UpdatedAt time.Time } // TableName returns the entity database table name. @@ -37,9 +38,9 @@ func (Marker) TableName() string { } // NewMarker creates a new entity. -func NewMarker(fileUID, refUID, markerSrc, markerType string, x, y, w, h float32) *Marker { +func NewMarker(fileUID uint, refUID, markerSrc, markerType string, x, y, w, h float32) *Marker { m := &Marker{ - FileUID: fileUID, + FileID: fileUID, RefUID: refUID, MarkerSrc: markerSrc, MarkerType: markerType, @@ -53,9 +54,9 @@ func NewMarker(fileUID, refUID, markerSrc, markerType string, x, y, w, h float32 } // NewFaceMarker creates a new entity. -func NewFaceMarker(f face.Face, fileUID, refUID string) *Marker { +func NewFaceMarker(f face.Face, fileID uint, refUID string) *Marker { fm := f.Marker() - m := NewMarker(fileUID, refUID, SrcImage, MarkerFace, fm.X, fm.Y, fm.W, fm.H) + m := NewMarker(fileID, refUID, SrcImage, MarkerFace, fm.X, fm.Y, fm.W, fm.H) m.MarkerScore = f.Score m.MarkerMeta = string(f.RelativeLandmarksJSON()) @@ -97,10 +98,10 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) { if m.ID > 0 { err := m.Save() - log.Debugf("faces: saved marker %d for file %s", m.ID, m.FileUID) + log.Debugf("faces: saved marker %d for file %d", m.ID, m.FileID) return m, err - } else if err := Db().Where(`file_uid = ? AND x > ? AND x < ? AND y > ? AND y < ?`, - m.FileUID, m.X-m.W, m.X+m.W, m.Y-m.H, m.Y+m.H).First(&result).Error; err == nil { + } else if err := Db().Where(`file_id = ? AND x > ? AND x < ? AND y > ? AND y < ?`, + m.FileID, m.X-m.W, m.X+m.W, m.Y-m.H, m.Y+m.H).First(&result).Error; err == nil { if SrcPriority[m.MarkerSrc] < SrcPriority[result.MarkerSrc] { // Ignore. @@ -117,11 +118,11 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) { "RefUID": m.RefUID, }) - log.Debugf("faces: updated existing marker %d for file %s", result.ID, result.FileUID) + log.Debugf("faces: updated existing marker %d for file %d", result.ID, result.FileID) return &result, err } else if err := m.Create(); err != nil { - log.Debugf("faces: added marker %d for file %s", m.ID, m.FileUID) + log.Debugf("faces: added marker %d for file %d", m.ID, m.FileID) return m, err } diff --git a/internal/entity/marker_test.go b/internal/entity/marker_test.go index e2434eeda..d89d4f763 100644 --- a/internal/entity/marker_test.go +++ b/internal/entity/marker_test.go @@ -12,9 +12,9 @@ func TestMarker_TableName(t *testing.T) { } func TestNewMarker(t *testing.T) { - m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) + m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) assert.IsType(t, &Marker{}, m) - assert.Equal(t, "ft8es39w45bnlqdw", m.FileUID) + assert.Equal(t, uint(1000000), m.FileID) assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID) assert.Equal(t, SrcImage, m.MarkerSrc) assert.Equal(t, MarkerLabel, m.MarkerType) @@ -22,9 +22,9 @@ func TestNewMarker(t *testing.T) { func TestUpdateOrCreateMarker(t *testing.T) { t.Run("success", func(t *testing.T) { - m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) + m := NewMarker(1000000, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) assert.IsType(t, &Marker{}, m) - assert.Equal(t, "ft8es39w45bnlqdw", m.FileUID) + assert.Equal(t, uint(1000000), m.FileID) assert.Equal(t, "lt9k3pw1wowuy3c3", m.RefUID) assert.Equal(t, SrcImage, m.MarkerSrc) assert.Equal(t, MarkerLabel, m.MarkerType) @@ -47,7 +47,7 @@ func TestUpdateOrCreateMarker(t *testing.T) { func TestMarker_Updates(t *testing.T) { t.Run("success", func(t *testing.T) { - m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) + m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) m, err := UpdateOrCreateMarker(m) if err != nil { @@ -72,7 +72,7 @@ func TestMarker_Updates(t *testing.T) { func TestMarker_Update(t *testing.T) { t.Run("success", func(t *testing.T) { - m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) + m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) m, err := UpdateOrCreateMarker(m) if err != nil { @@ -96,7 +96,7 @@ func TestMarker_Update(t *testing.T) { func TestMarker_Save(t *testing.T) { t.Run("success", func(t *testing.T) { - m := NewMarker("ft8es39w45bnlqdw", "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) + m := NewMarker(1000000, "lt9k3pw1wowuy3c4", SrcImage, MarkerLabel, 0.308333, 0.206944, 0.355556, 0.355556) m, err := UpdateOrCreateMarker(m) if err != nil { @@ -123,5 +123,12 @@ func TestMarker_Save(t *testing.T) { if m.ID <= 0 { t.Errorf("ID should be > 0") } + + p := PhotoFixtures.Get("19800101_000002_D640C559") + assert.Empty(t, p.Files) + p.PreloadFiles(true) + assert.NotEmpty(t, p.Files) + + t.Logf("FILES: %#v", p.Files) }) } diff --git a/internal/entity/markers.go b/internal/entity/markers.go index 32fffd662..0405a8038 100644 --- a/internal/entity/markers.go +++ b/internal/entity/markers.go @@ -3,9 +3,12 @@ package entity type Markers []Marker // Save stores the markers in the database. -func (m Markers) Save(fileUID string) error { +func (m Markers) Save(fileID uint) error { for _, marker := range m { - marker.FileUID = fileUID + if fileID > 0 { + marker.FileID = fileID + } + if _, err := UpdateOrCreateMarker(&marker); err != nil { return err } @@ -13,3 +16,34 @@ func (m Markers) Save(fileUID string) error { return nil } + +// Contains returns true if a marker at the same position already exists. +func (m Markers) Contains(m2 Marker) bool { + for _, m1 := range m { + if m2.X > (m1.X-m1.W) && m2.X < (m1.X+m1.W) && m2.Y > (m1.Y-m1.H) && m2.Y < (m1.Y+m1.H) { + return true + } + } + + return false +} + +// FaceCount returns the number of valid face markers. +func (m Markers) FaceCount() int { + result := 0 + for _, marker := range m { + if !marker.MarkerInvalid && marker.MarkerType == MarkerFace { + result++ + } + } + + return result +} + +// FindMarkers returns all markers for a given file id. +func FindMarkers(fileID uint) (Markers, error) { + m := Markers{} + err := Db().Where(`file_id = ?`, fileID).Order("id").Offset(0).Limit(1000).Find(&m).Error + + return m, err +} diff --git a/internal/entity/markers_test.go b/internal/entity/markers_test.go new file mode 100644 index 000000000..764f908c7 --- /dev/null +++ b/internal/entity/markers_test.go @@ -0,0 +1,29 @@ +package entity + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMarkers_Contains(t *testing.T) { + m1 := *NewMarker(1000000, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 0.308333, 0.206944, 0.355556, 0.355556) + m2 := *NewMarker(1000000, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 0.298133, 0.216944, 0.255556, 0.155556) + m3 := *NewMarker(1000000, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 0.998133, 0.816944, 0.0001, 0.0001) + + m := Markers{m1} + + assert.True(t, m.Contains(m2)) + assert.False(t, m.Contains(m3)) +} + +func TestMarkers_FaceCount(t *testing.T) { + m1 := *NewMarker(1000000, "lt9k3pw1wowuy1c1", SrcImage, MarkerFace, 0.308333, 0.206944, 0.355556, 0.355556) + m2 := *NewMarker(1000000, "lt9k3pw1wowuy1c2", SrcImage, MarkerFace, 0.298133, 0.216944, 0.255556, 0.155556) + m3 := *NewMarker(1000000, "lt9k3pw1wowuy1c3", SrcImage, MarkerFace, 0.998133, 0.816944, 0.0001, 0.0001) + m3.MarkerInvalid = true + + m := Markers{m1, m2, m3} + + assert.Equal(t, 2, m.FaceCount()) +} diff --git a/internal/entity/photo.go b/internal/entity/photo.go index 0045be361..9f4c23a92 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -439,14 +439,20 @@ func (m *Photo) IndexKeywords() error { } // PreloadFiles prepares gorm scope to retrieve photo file -func (m *Photo) PreloadFiles() { +func (m *Photo) PreloadFiles(markers bool) { q := Db(). Table("files"). Select(`files.*`). - Where("files.deleted_at IS NULL AND files.photo_id = ?", m.ID). + Where("files.photo_id = ? AND files.deleted_at IS NULL", m.ID). Order("files.file_name DESC") logError(q.Scan(&m.Files)) + + if markers { + for i := range m.Files { + m.Files[i].PreloadMarkers() + } + } } // PreloadKeywords prepares gorm scope to retrieve photo keywords @@ -474,7 +480,7 @@ func (m *Photo) PreloadAlbums() { // PreloadMany prepares gorm scope to retrieve photo file, albums and keywords func (m *Photo) PreloadMany() { - m.PreloadFiles() + m.PreloadFiles(true) m.PreloadKeywords() m.PreloadAlbums() } diff --git a/internal/entity/photo_test.go b/internal/entity/photo_test.go index 457099a59..b31a5a415 100644 --- a/internal/entity/photo_test.go +++ b/internal/entity/photo_test.go @@ -145,7 +145,7 @@ func TestPhoto_PreloadFiles(t *testing.T) { t.Run("success", func(t *testing.T) { m := PhotoFixtures.Get("Photo01") assert.Empty(t, m.Files) - m.PreloadFiles() + m.PreloadFiles(false) assert.NotEmpty(t, m.Files) }) } diff --git a/internal/entity/photo_yaml_test.go b/internal/entity/photo_yaml_test.go index 38ebabf6c..bafdc6e71 100644 --- a/internal/entity/photo_yaml_test.go +++ b/internal/entity/photo_yaml_test.go @@ -11,7 +11,7 @@ import ( func TestPhoto_Yaml(t *testing.T) { t.Run("create from fixture", func(t *testing.T) { m := PhotoFixtures.Get("Photo01") - m.PreloadFiles() + m.PreloadFiles(true) result, err := m.Yaml() if err != nil { @@ -25,7 +25,7 @@ func TestPhoto_Yaml(t *testing.T) { func TestPhoto_SaveAsYaml(t *testing.T) { t.Run("create from fixture", func(t *testing.T) { m := PhotoFixtures.Get("Photo01") - m.PreloadFiles() + m.PreloadFiles(true) fileName := filepath.Join(os.TempDir(), ".photoprism_test.yml") @@ -46,7 +46,7 @@ func TestPhoto_SaveAsYaml(t *testing.T) { func TestPhoto_YamlFileName(t *testing.T) { t.Run("create from fixture", func(t *testing.T) { m := PhotoFixtures.Get("Photo01") - m.PreloadFiles() + m.PreloadFiles(false) assert.Equal(t, "xxx/2790/02/yyy/Photo01.yml", m.YamlFileName("xxx", "yyy")) if err := os.RemoveAll("xxx"); err != nil { diff --git a/internal/face/detector.go b/internal/face/detector.go index eb890ac34..116b8a05f 100644 --- a/internal/face/detector.go +++ b/internal/face/detector.go @@ -3,14 +3,15 @@ package face import ( _ "embed" "fmt" - pigo "github.com/esimov/pigo/core" - "github.com/photoprism/photoprism/pkg/fs" - "github.com/photoprism/photoprism/pkg/txt" _ "image/jpeg" "io" "os" "path/filepath" "runtime/debug" + + pigo "github.com/esimov/pigo/core" + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/txt" ) //go:embed cascade/facefinder diff --git a/internal/face/face.go b/internal/face/face.go index 5c6a5834f..eacb2af64 100644 --- a/internal/face/face.go +++ b/internal/face/face.go @@ -43,6 +43,51 @@ var log = event.Log // Faces is a list of face detection results. type Faces []Face +// Count returns the number of faces detected. +func (faces Faces) Count() int { + return len(faces) +} + +// Uncertainty return the max face detection uncertainty in percent. +func (faces Faces) Uncertainty() int { + if len(faces) < 1 { + return 100 + } + + maxScore := 0 + + for _, f := range faces { + if f.Score > maxScore { + maxScore = f.Score + } + } + + switch { + case maxScore > 300: + return 1 + case maxScore > 200: + return 5 + case maxScore > 100: + return 10 + case maxScore > 80: + return 15 + case maxScore > 65: + return 20 + case maxScore > 50: + return 25 + case maxScore > 40: + return 30 + case maxScore > 30: + return 35 + case maxScore > 20: + return 40 + case maxScore > 10: + return 45 + } + + return 50 +} + // Face represents a face detection result. type Face struct { Rows int `json:"rows,omitempty"` diff --git a/internal/face/face_test.go b/internal/face/face_test.go index 4780542fb..56382f4f4 100644 --- a/internal/face/face_test.go +++ b/internal/face/face_test.go @@ -60,8 +60,15 @@ func TestDetect(t *testing.T) { if i, ok := expected[baseName]; ok { assert.Equal(t, i, len(faces)) + assert.Equal(t, i, faces.Count()) + if faces.Count() == 0 { + assert.Equal(t, 100, faces.Uncertainty()) + } else { + assert.Truef(t, faces.Uncertainty() >= 0 && faces.Uncertainty() <= 50, "uncertainty should be between 0 and 50") + } + t.Logf("uncertainty: %d", faces.Uncertainty()) } else { - t.Errorf("unknown test result for %s", baseName) + t.Logf("unknown test result for %s", baseName) } }) diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index 28d6a61e9..fec492cb1 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -600,19 +600,22 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( // Main JPEG file. if file.FilePrimary { - labels := photo.ClassifyLabels() - if Config().Experimental() && Config().Settings().Features.People { faces := ind.detectFaces(m) - photo.AddLabels(classify.FaceLabels(len(faces), entity.SrcImage, 10)) - photo.PhotoFaces = len(faces) + photo.AddLabels(classify.FaceLabels(faces, entity.SrcImage)) + + file.PreloadMarkers() if len(faces) > 0 { file.AddFaces(faces) } + + photo.PhotoFaces = file.Markers.FaceCount() } + labels := photo.ClassifyLabels() + if err := photo.UpdateTitle(labels); err != nil { log.Debugf("%s in %s (update title)", err, logName) }