package entity import ( "encoding/json" "fmt" "strings" "time" "github.com/photoprism/photoprism/pkg/clusters" "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/pkg/txt" ) const ( MarkerUnknown = "" MarkerFace = "face" MarkerLabel = "label" ) // Marker represents an image marker point. type Marker struct { ID uint `gorm:"primary_key" json:"ID" yaml:"-"` FileID uint `gorm:"index;" json:"-" yaml:"-"` MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"` MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"` MarkerName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name,omitempty"` SubjectUID string `gorm:"type:VARBINARY(42);index;" json:"SubjectUID" yaml:"SubjectUID,omitempty"` SubjectSrc string `gorm:"type:VARBINARY(8);default:'';" json:"SubjectSrc" yaml:"SubjectSrc,omitempty"` Subject *Subject `gorm:"foreignkey:SubjectUID;association_foreignkey:SubjectUID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Subject,omitempty" yaml:"-"` FaceID string `gorm:"type:VARBINARY(42);index;" json:"FaceID" yaml:"FaceID,omitempty"` FaceDist float64 `gorm:"default:-1" json:"FaceDist" yaml:"FaceDist,omitempty"` Face *Face `gorm:"foreignkey:FaceID;association_foreignkey:ID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"-" yaml:"-"` EmbeddingsJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"EmbeddingsJSON,omitempty"` embeddings Embeddings `gorm:"-"` LandmarksJSON json.RawMessage `gorm:"type:MEDIUMBLOB;" json:"-" yaml:"LandmarksJSON,omitempty"` X float32 `gorm:"type:FLOAT;" json:"X" yaml:"X,omitempty"` Y float32 `gorm:"type:FLOAT;" json:"Y" yaml:"Y,omitempty"` W float32 `gorm:"type:FLOAT;" json:"W" yaml:"W,omitempty"` H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"` Size int `gorm:"default:-1" json:"Size" yaml:"Size,omitempty"` Score int `gorm:"type:SMALLINT" json:"Score" yaml:"Score,omitempty"` MarkerInvalid bool `json:"Invalid" yaml:"Invalid,omitempty"` MatchedAt *time.Time `sql:"index" json:"MatchedAt" yaml:"MatchedAt,omitempty"` CreatedAt time.Time UpdatedAt time.Time } // UnknownMarker can be used as a default for unknown markers. var UnknownMarker = NewMarker(0, "", SrcDefault, MarkerUnknown, 0, 0, 0, 0) // TableName returns the entity database table name. func (Marker) TableName() string { return "markers_dev5" } // NewMarker creates a new entity. func NewMarker(fileID uint, subjectUID, markerSrc, markerType string, x, y, w, h float32) *Marker { m := &Marker{ FileID: fileID, SubjectUID: subjectUID, MarkerSrc: markerSrc, MarkerType: markerType, X: x, Y: y, W: w, H: h, } return m } // NewFaceMarker creates a new entity. func NewFaceMarker(f face.Face, fileID uint, refUID string) *Marker { pos := f.Marker() m := NewMarker(fileID, refUID, SrcImage, MarkerFace, pos.X, pos.Y, pos.W, pos.H) m.MatchedAt = nil m.FaceDist = -1 m.EmbeddingsJSON = f.EmbeddingsJSON() m.LandmarksJSON = f.RelativeLandmarksJSON() m.Size = f.Size() m.Score = f.Score return m } // Updates multiple columns in the database. func (m *Marker) Updates(values interface{}) error { return UnscopedDb().Model(m).Updates(values).Error } // Update updates a column in the database. func (m *Marker) Update(attr string, value interface{}) error { return UnscopedDb().Model(m).Update(attr, value).Error } // SaveForm updates the entity using form data and stores it in the database. func (m *Marker) SaveForm(f form.Marker) error { changed := false if m.MarkerInvalid != f.MarkerInvalid { m.MarkerInvalid = f.MarkerInvalid changed = true } if !m.MarkerInvalid && m.Score < 100 { m.Score = 100 changed = true } if f.SubjectSrc == SrcManual && strings.TrimSpace(f.MarkerName) != "" { m.SubjectSrc = SrcManual m.MarkerName = txt.Title(txt.Clip(f.MarkerName, txt.ClipDefault)) if err := m.SyncSubject(true); err != nil { return err } changed = true } if changed { return m.Save() } return nil } // HasFace tests if the marker already has the best matching face. func (m *Marker) HasFace(f *Face, dist float64) bool { if m.FaceID == "" { return false } else if f == nil { return m.FaceID != "" } else if m.FaceID == f.ID { return m.FaceID != "" } else if m.FaceDist < 0 { return false } else if dist < 0 { return true } return m.FaceDist <= dist } // SetFace sets a new face for this marker. func (m *Marker) SetFace(f *Face, dist float64) (updated bool, err error) { if f == nil { return false, fmt.Errorf("face is nil") } if m.MarkerType != MarkerFace { return false, fmt.Errorf("not a face marker") } // Any reason we don't want to set a new face for this marker? if m.SubjectSrc != SrcManual || f.SubjectUID == "" || m.SubjectUID == "" || f.SubjectUID == m.SubjectUID { // Don't skip if subject wasn't set manually, or subjects match. } else if reported, err := f.ReportCollision(m.Embeddings()); err != nil { return false, err } else if reported { log.Infof("faces: marker %d (subject %s) collision with %s (subject %s), source %s", m.ID, m.SubjectUID, f.ID, f.SubjectUID, m.SubjectSrc) return false, nil } else { return false, nil } // Update face with known subject from marker? if f.SubjectUID != "" || m.SubjectUID == "" { // Don't update if face has a known subject, or marker subject is unknown. } else if err := f.Update("SubjectUID", m.SubjectUID); err != nil { return false, err } // Skip update if the same face is already set. if m.SubjectUID == f.SubjectUID && m.FaceID == f.ID { // Update matching timestamp. m.MatchedAt = TimePointer() return false, m.Updates(Values{"MatchedAt": m.MatchedAt}) } // Remember current values for comparison. faceID := m.FaceID subjectUID := m.SubjectUID SubjectSrc := m.SubjectSrc m.FaceID = f.ID m.FaceDist = dist if m.FaceDist < 0 { faceEmbedding := f.Embedding() // Calculate smallest distance to embeddings. for _, e := range m.Embeddings() { if len(e) != len(faceEmbedding) { continue } if d := clusters.EuclideanDistance(e, faceEmbedding); d < m.FaceDist || m.FaceDist < 0 { m.FaceDist = d } } } if f.SubjectUID != "" { m.SubjectUID = f.SubjectUID } if err := m.SyncSubject(false); err != nil { return false, err } // Update face subject? if m.SubjectUID == "" || f.SubjectUID != m.SubjectUID { // Not needed. } else if err := f.Update("SubjectUID", m.SubjectUID); err != nil { return false, err } updated = m.FaceID != faceID || m.SubjectUID != subjectUID || m.SubjectSrc != SubjectSrc // Update matching timestamp. m.MatchedAt = TimePointer() return updated, m.Updates(Values{"FaceID": m.FaceID, "FaceDist": m.FaceDist, "SubjectUID": m.SubjectUID, "SubjectSrc": m.SubjectSrc, "MatchedAt": m.MatchedAt}) } // SyncSubject maintains the marker subject relationship. func (m *Marker) SyncSubject(updateRelated bool) error { // Face marker? If not, return. if m.MarkerType != MarkerFace { return nil } subj := m.GetSubject() if subj == nil { return nil } // Update subject with marker name? if m.MarkerName == "" || subj.SubjectName == m.MarkerName || (subj.SubjectName != "" && m.SubjectSrc != SrcManual) { // Do nothing. } else if err := subj.UpdateName(m.MarkerName); err != nil { return err } // Create known face for subject? if m.FaceID != "" || m.SubjectSrc != SrcManual { // Do nothing. } else if f := m.GetFace(); f != nil { m.FaceID = f.ID } // Update related markers? if m.FaceID == "" || m.SubjectUID == "" { // Do nothing. } else if err := Db().Model(&Face{}).Where("id = ? AND subject_uid = ''", m.FaceID).Update("SubjectUID", m.SubjectUID).Error; err != nil { return fmt.Errorf("%s (update known face)", err) } else if !updateRelated { return nil } else if err := Db().Model(&Marker{}). Where("id <> ?", m.ID). Where("face_id = ?", m.FaceID). Where("subject_src = ?", SrcAuto). Where("subject_uid <> ?", m.SubjectUID). Updates(Values{"SubjectUID": m.SubjectUID, "SubjectSrc": SrcAuto}).Error; err != nil { return fmt.Errorf("%s (update related markers)", err) } else { log.Debugf("marker: matched %s with %s", subj.SubjectName, m.FaceID) } return nil } // Save updates the existing or inserts a new row. func (m *Marker) Save() error { if m.X == 0 || m.Y == 0 || m.X > 1 || m.Y > 1 || m.X < -1 || m.Y < -1 { return fmt.Errorf("marker: invalid position") } return Db().Save(m).Error } // Create inserts a new row to the database. func (m *Marker) Create() error { if m.X == 0 || m.Y == 0 || m.X > 1 || m.Y > 1 || m.X < -1 || m.Y < -1 { return fmt.Errorf("marker: invalid position") } return Db().Create(m).Error } // Embeddings returns parsed marker embeddings. func (m *Marker) Embeddings() Embeddings { if len(m.EmbeddingsJSON) == 0 { return Embeddings{} } else if len(m.embeddings) > 0 { return m.embeddings } else if err := json.Unmarshal(m.EmbeddingsJSON, &m.embeddings); err != nil { log.Errorf("failed parsing marker embeddings json: %s", err) } return m.embeddings } // GetSubject returns a subject entity if possible. func (m *Marker) GetSubject() (subj *Subject) { if m.Subject != nil { return m.Subject } if m.SubjectUID == "" && m.MarkerName != "" { if subj = NewSubject(m.MarkerName, SubjectPerson, SrcMarker); subj == nil { return nil } else if subj = FirstOrCreateSubject(subj); subj == nil { log.Debugf("marker: invalid subject %s", txt.Quote(m.MarkerName)) return nil } m.SubjectUID = subj.SubjectUID m.SubjectSrc = SrcManual return subj } m.Subject = FindSubject(m.SubjectUID) return m.Subject } // ClearSubject removes an existing subject association, and reports a collision. func (m *Marker) ClearSubject(src string) error { if m.Face == nil { m.Face = FindFace(m.FaceID) } if m.Face == nil { // Do nothing } else if reported, err := m.Face.ReportCollision(m.Embeddings()); err != nil { return err } else if err := m.Updates(Values{"MarkerName": "", "FaceID": "", "FaceDist": -1.0, "SubjectUID": "", "SubjectSrc": src}); err != nil { return err } else if reported { log.Debugf("faces: collision with %s", m.Face.ID) } m.Face = nil m.MarkerName = "" m.FaceID = "" m.FaceDist = -1.0 m.SubjectUID = "" m.SubjectSrc = src return nil } // GetFace returns a matching face entity if possible. func (m *Marker) GetFace() (f *Face) { if m.Face != nil { return m.Face } // Add face if size if m.FaceID == "" && m.SubjectSrc == SrcManual { if m.Size < face.ClusterMinSize || m.Score < face.ClusterMinScore { log.Debugf("faces: skipped adding face for low-quality marker %d, size %d, score %d", m.ID, m.Size, m.Score) return nil } else if emb := m.Embeddings(); len(emb) == 0 { log.Warnf("marker: id %d has no embeddings", m.ID) return nil } else if f = NewFace(m.SubjectUID, SrcManual, emb); f == nil { log.Warnf("marker: failed adding face for id %d", m.ID) return nil } else if f = FirstOrCreateFace(f); f == nil { log.Warnf("marker: failed adding face for id %d", m.ID) return nil } else if err := f.MatchMarkers(Faceless); err != nil { log.Errorf("faces: %s (match markers)", err) } m.Face = f m.FaceID = f.ID } else { m.Face = FindFace(m.FaceID) } return m.Face } // ClearFace removes an existing face association. func (m *Marker) ClearFace() (updated bool, err error) { if m.FaceID == "" { return false, m.Matched() } updated = true // Remove face references. m.Face = nil m.FaceID = "" m.MatchedAt = TimePointer() // Remove subject if set automatically. if m.SubjectSrc == SrcAuto { m.SubjectUID = "" err = m.Updates(Values{"FaceID": "", "FaceDist": -1.0, "SubjectUID": "", "MatchedAt": m.MatchedAt}) } else { err = m.Updates(Values{"FaceID": "", "FaceDist": -1.0, "MatchedAt": m.MatchedAt}) } return updated, err } // Matched updates the match timestamp. func (m *Marker) Matched() error { m.MatchedAt = TimePointer() return UnscopedDb().Model(m).UpdateColumns(Values{"MatchedAt": m.MatchedAt}).Error } // FindMarker returns an existing row if exists. func FindMarker(id uint) *Marker { result := Marker{} if err := Db().Where("id = ?", id).First(&result).Error; err == nil { return &result } return nil } // UpdateOrCreateMarker updates a marker in the database or creates a new one if needed. func UpdateOrCreateMarker(m *Marker) (*Marker, error) { const d = 0.07 result := Marker{} if m.ID > 0 { err := m.Save() log.Debugf("faces: saved marker %d for file %d", m.ID, m.FileID) return m, err } else if err := Db().Where(`file_id = ? AND x > ? AND x < ? AND y > ? AND y < ?`, m.FileID, m.X-d, m.X+d, m.Y-d, m.Y+d).First(&result).Error; err == nil { if SrcPriority[m.MarkerSrc] < SrcPriority[result.MarkerSrc] { // Ignore. return &result, nil } err := result.Updates(map[string]interface{}{ "X": m.X, "Y": m.Y, "W": m.W, "H": m.H, "Score": m.Score, "LandmarksJSON": m.LandmarksJSON, "EmbeddingsJSON": m.EmbeddingsJSON, "SubjectUID": m.SubjectUID, }) log.Debugf("faces: updated existing marker %d for file %d", result.ID, result.FileID) return &result, err } else if err := m.Create(); err != nil { log.Debugf("faces: added marker %d for file %d", m.ID, m.FileID) return m, err } return m, nil }