From e78c4df22ecb717a9d2adcff91a7ab7fd45e95c2 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Thu, 12 Aug 2021 17:49:48 +0200 Subject: [PATCH] People: Create person based on existing marker labels #22 Work in progress. --- frontend/src/model/photo.js | 10 +++++-- internal/entity/marker.go | 1 + internal/entity/marker_fixtures.go | 13 +++++++++ internal/entity/markers.go | 7 ++++- internal/entity/person.go | 4 +-- internal/entity/person_face.go | 2 +- internal/entity/src.go | 6 ++-- internal/photoprism/people.go | 46 +++++++++++++++++++++--------- 8 files changed, 67 insertions(+), 22 deletions(-) diff --git a/frontend/src/model/photo.js b/frontend/src/model/photo.js index f0a70efb3..87a55d802 100644 --- a/frontend/src/model/photo.js +++ b/frontend/src/model/photo.js @@ -759,9 +759,15 @@ export class Photo extends RestModel { } file.Markers.forEach((m) => { - if (!valid || !m.Invalid) { - result.push(m); + if (valid && m.Invalid) { + return; } + + if (m.hasOwnProperty("Person") && !!m.Person && !!m.Person.Name) { + m.Label = m.Person.Name; + } + + result.push(m); }); return result; diff --git a/internal/entity/marker.go b/internal/entity/marker.go index da01a6a1f..05d8c15d0 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -37,6 +37,7 @@ type Marker struct { H float32 `gorm:"type:FLOAT;" json:"H" yaml:"H,omitempty"` CreatedAt time.Time UpdatedAt time.Time + Person *Person `gorm:"foreignkey:RefUID;association_foreignkey:PersonUID;association_autoupdate:false;association_autocreate:false;association_save_reference:false" json:"Person" yaml:"-"` } // UnknownMarker can be used as a default for unknown markers. diff --git a/internal/entity/marker_fixtures.go b/internal/entity/marker_fixtures.go index 5a816c5a0..a059e6c0e 100644 --- a/internal/entity/marker_fixtures.go +++ b/internal/entity/marker_fixtures.go @@ -77,6 +77,19 @@ var MarkerFixtures = MarkerMap{ W: 0.1, H: 0.1, }, + "1000003-6": Marker{ + FileID: 1000003, + RefUID: "", + MarkerSrc: SrcImage, + MarkerType: MarkerFace, + MarkerLabel: "", + MarkerMeta: "[{\"name\":\"lp46\",\"x\":-0.10546875,\"y\":-0.045898438,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp46_v\",\"x\":0.11328125,\"y\":0.0126953125,\"h\":0.034179688,\"w\":0.045572918},{\"name\":\"lp44\",\"x\":-0.053385418,\"y\":-0.0546875,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp44_v\",\"x\":0.09375,\"y\":-0.0078125,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp42\",\"x\":-0.015625,\"y\":-0.030273438,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp42_v\",\"x\":0.0546875,\"y\":-0.0087890625,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp38\",\"x\":-0.033854168,\"y\":-0.0087890625,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp38_v\",\"x\":0.037760418,\"y\":0.01171875,\"h\":0.032226562,\"w\":0.04296875},{\"name\":\"lp312\",\"x\":-0.091145836,\"y\":-0.02734375,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp312_v\",\"x\":0.08984375,\"y\":0.021484375,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"mouth_lp93\",\"x\":-0.026041666,\"y\":0.07714844,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"mouth_lp84\",\"x\":-0.102864586,\"y\":0.08496094,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"mouth_lp82\",\"x\":-0.05859375,\"y\":0.12109375,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"mouth_lp81\",\"x\":-0.045572918,\"y\":0.10058594,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"lp84\",\"x\":-0.0065104165,\"y\":0.1171875,\"h\":0.033203125,\"w\":0.044270832},{\"name\":\"eye_l\",\"x\":-0.059895832,\"y\":-0.015625,\"h\":0.022460938,\"w\":0.029947916},{\"name\":\"eye_r\",\"x\":0.059895832,\"y\":0.015625,\"h\":0.022460938,\"w\":0.029947916}]", + Embeddings: "[[-0.11598889,-0.113085627,0.091266885,0.11090832,0.059863407,0.018543985,0.06513644,0.272262324,0.0024608355,0.023480255,-0.034981046,0.028824307,0.031145066,0.020279909,-0.038303424,0.063686624,0.00016155555,-0.040199693,-0.004059554,-0.038183372,0.010284749,0.0393553,-0.013954249,0.014198846,0.047331642,-0.07308497,0.008753774,-0.044199772,0.0351775,-0.009616884,-0.011886778,-0.03133512,-0.008360827,-0.0021931753,-0.0031518617,-0.0007841898,0.0012749507,-0.013682331,-0.0093261255,-0.0646958,0.028137255,-0.051393177,-0.010831488,-0.0019370695,0.026701374,0.08734394,-0.03148508,-0.072140485,-0.008645494,0.03283726,0.025486251,-0.14762828,0.0016828487,-0.049219336,-0.090523295,-0.017858343,-0.0433293,-0.03822806,0.03775215,-0.030626448,-0.005236273,-0.025029438,0.011486794,0.08866543,-0.11626933,-0.012919138,0.011206989,0.029296853,0.029712738,-0.0035943172,-0.0625837,-0.08751456,0.06506425,0.08434424,0.018379156,0.006281598,-0.019832052,0.013404299,0.050819624,-0.0025536602,0.108513094,0.043542076,-0.03385126,-0.013718123,-0.020935653,0.026902547,0.023695294,-0.032848295,0.02122507,-0.06577069,-0.049782418,0.07434279,0.011499832,0.0274455,0.023498816,-0.0024784799,-0.0408338,-0.01835984,-0.07471391,-0.020153865,0.020164149,0.08315784,0.057026222,0.0064989394,-0.030934982,-0.068170995,-0.0447578,-0.08320161,-0.02340753,-0.052578453,-0.014655025,0.011221938,0.020638045,-0.046642996,0.07793852,-0.07613048,-0.0052055195,0.050180323,-0.008933726,-0.006827808,0.046653792,0.046993263,-0.022871166,-0.10476393,-0.019975707,-0.013084017,0.054878037,0.016760936,0.0547656,0.016288247,0.029425414,-0.0039762836,-0.06857062,0.011283167,0.031562343,-0.0301557,0.018833537,0.014527415,0.014445754,0.08944574,-0.0201444,0.07525237,0.00338875,-0.009128363,0.03199888,0.003338322,-0.034410495,0.036284383,0.061515816,-0.04700179,-0.0037704427,-0.014624933,0.00053029,-0.040128306,0.033271074,-0.034320436,-0.087529205,-0.015765699,-0.042281955,0.026603937,-0.03505665,0.0045038764,0.07849869,-0.020796757,-0.027909378,-0.030466244,-0.022126308,0.032068476,0.017038053,0.011572472,-0.023514347,-0.009388009,0.0471232,0.079254,0.038409125,0.09174638,-0.00019651726,0.031568147,0.061691687,0.0035580387,0.01834219,-0.034832783,-0.02472014,-0.0043187817,-0.012822156,-0.006406414,0.04589425,-0.029704314,0.0122809885,-0.003854935,0.034970216,-0.015017675,0.060136527,-0.0018005831,0.122445226,-0.02466541,-0.01479218,0.021981824,0.008459089,-0.024648905,-0.005501727,-0.03819929,0.09229076,-0.00051632634,0.026605012,0.07888228,0.108206324,0.052503213,0.11041478,-0.051830698,0.0057799574,0.06611321,-0.07650234,0.05879657,0.052348997,0.019547585,-0.07638257,-0.044862173,-0.043846298,-0.021011176,-0.01881608,0.107294865,-0.022127088,0.067301944,0.06469245,-0.00538971,-0.014842585,0.008351809,0.004484374,0.040771488,-0.10075861,-0.037090372,0.010828192,-0.014120988,-0.106951684,0.017808948,0.0014778537,-0.052681025,-0.005136359,0.03752494,0.021728097,0.054375395,-0.024591682,-0.028040286,-0.00003123716,-0.009580712,-0.01999454,-0.036310453,0.012808869,0.033562016,-0.048961494,0.049189627,-0.075262696,-0.025279071,-0.017653963,-0.07986348,-0.0058788625,-0.007859457,-0.07787707,-0.0141451955,-0.027267313,0.034545522,-0.066852376,-0.033797234,0.00662154,-0.0042986833,0.06258031,-0.027749743,0.035639953,-0.0860917,-0.021690408,-0.041379962,0.058219258,0.036375474,-0.028576316,0.009679853,0.0540403,-0.00078071933,0.046852123,-0.009272538,0.00014093994,0.08755382,0.029109005,-0.051301096,0.05482741,0.0515714,0.00093064347,-0.0030184423,0.006834386,0.018765705,-0.05297967,0.003963442,0.0058909976,0.0015918964,0.0059644254,-0.010626185,0.016774561,0.019620024,0.012779505,-0.0037311793,0.06090134,0.030533332,-0.0013029866,0.0078083063,-0.07804226,0.055505097,-0.0323863,-0.13843039,0.017691568,0.015463283,-0.0054733357,0.029853273,0.044509612,-0.017058503,-0.009644833,-0.08004177,0.018354492,0.08402592,0.08028074,-0.033064563,0.0071564782,-0.026635528,-0.020762388,-0.09037636,0.07283039,-0.0013951861,0.023556605,-0.025109852,-0.01911825,-0.072672606,0.023367604,0.0066802157,-0.029526467,-0.005831557,-0.056150723,-0.036622144,0.014390345,-0.06641906,0.027680077,-0.0027364918,0.07881769,0.0032321527,0.06692333,-0.023430921,-0.0156005705,-0.012797314,-0.06341876,-0.0075710113,-0.08883436,0.018736875,0.0515824,-0.03050761,-0.038169485,-0.019994874,-0.037897438,0.0030616417,0.016518306,0.017313045,-0.040592685,-0.121986456,0.036341745,-0.055392407,0.06128348,0.09128614,0.085432775,-0.04018598,0.020804984,0.022338325,-0.047893576,-0.04436214,0.010663377,-0.01539266,0.001970492,-0.02548427,-0.0010024207,0.012728738,-0.03458635,0.000458029,-0.07648158,0.017893706,-0.03620278,0.012510285,0.042211026,0.029836254,-0.01023813,-0.014964832,-0.036710255,0.072146155,-0.032623224,-0.054371897,0.059095327,-0.026043909,-0.026475005,-0.03756759,-0.0033844158,0.01665272,0.055144988,0.020163653,0.010291277,0.016448587,-0.08021163,-0.00711534,0.0014388722,0.047059905,0.055735916,0.04716966,0.002357553,0.028611615,0.053914364,-0.022333615,0.01227299,-0.006376796,-0.020971471,-0.031313244,0.004976049,0.027839795,0.032628387,0.014910606,-0.019821445,-0.023739582,-0.061071664,-0.03563204,0.04504174,-0.043931577,-0.09461471,0.035146907,0.018801821,-0.023486922,-0.014275421,-0.04465509,-0.02062559,0.0049236626,-0.018532282,0.0329802,0.08521481,-0.025042368,-0.0031619244,-0.012923802,0.062199675,0.008717194,-0.06812108,-0.059829578,0.012077271,0.04268468,-0.029709337,0.058341388,-0.023695359,0.026195621,0.089156136,-0.05210246,-0.006754805,-0.08180936,-0.016863013,0.01305001,-0.06608525,0.060481545,-0.008385101,-0.022109058,-0.060189016,0.029254012,0.030882897,-0.009892429,0.04769517,0.010766996,-0.032422658,-0.010280704,-0.035042368,-0.046891414,-0.021463934,-0.022753375,0.011999883,-0.027142627,-0.00023438907,0.016639061,-0.022108192,0.06750844,0.031127596,0.062595494,0.035271265,-0.00848079,0.0054323506,-0.038877886,-0.09272652,-0.028250217,-0.008661228,-0.08010225,-0.066937625,-0.022481173,0.023471648,0.0064566983,-0.0010692414,-0.11465305,0.041438285,-0.08444645,0.004585747,-0.028296055,0.04911384,-0.0049776305,0.043981623,-0.010605184,0.07764796,-0.022333274,0.050281037,-0.06543124,-0.00412494,0.08933364,0.043259624,0.0007392553,-0.0055968673,-0.041995358,-0.01275004,0.017657083]]", + X: 0.7, + Y: 0.8, + W: 0.06, + H: 0.08, + }, } // CreateMarkerFixtures inserts known entities into the database for testing. diff --git a/internal/entity/markers.go b/internal/entity/markers.go index 5e45eec8a..1d4cb3028 100644 --- a/internal/entity/markers.go +++ b/internal/entity/markers.go @@ -45,7 +45,12 @@ func (m Markers) FaceCount() int { // FindMarkers returns all markers for a given file id. func FindMarkers(fileID uint) (Markers, error) { m := Markers{} - err := Db().Where(`file_id = ?`, fileID).Order("id").Offset(0).Limit(1000).Find(&m).Error + err := Db(). + Where(`file_id = ?`, fileID). + Preload("Person"). + Order("id"). + Offset(0).Limit(1000). + Find(&m).Error return m, err } diff --git a/internal/entity/person.go b/internal/entity/person.go index 39f400cba..47849a727 100644 --- a/internal/entity/person.go +++ b/internal/entity/person.go @@ -57,9 +57,7 @@ type Person struct { BirthYear int `json:"BirthYear" yaml:"BirthYear,omitempty"` BirthMonth int `json:"BirthMonth" yaml:"BirthMonth,omitempty"` BirthDay int `json:"BirthDay" yaml:"BirthDay,omitempty"` - DeathYear int `json:"DeathYear" yaml:"DeathYear,omitempty"` - DeathMonth int `json:"DeathMonth" yaml:"DeathMonth,omitempty"` - DeathDay int `json:"DeathDay" yaml:"DeathDay,omitempty"` + PassedAway *time.Time `json:"PassedAway" yaml:"PassedAway,omitempty"` CreatedAt time.Time `json:"CreatedAt" yaml:"-"` UpdatedAt time.Time `json:"UpdatedAt" yaml:"-"` DeletedAt *time.Time `sql:"index" json:"DeletedAt,omitempty" yaml:"-"` diff --git a/internal/entity/person_face.go b/internal/entity/person_face.go index fe9bb1a4b..efffe1550 100644 --- a/internal/entity/person_face.go +++ b/internal/entity/person_face.go @@ -11,7 +11,7 @@ type PeopleFaces []PersonFace // PersonFace represents the face of a Person. type PersonFace struct { ID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"ID" yaml:"ID"` - PersonUID string `gorm:"type:VARBINARY(42);index;" json:"PersonUID" yaml:"PersonUID"` + PersonUID string `gorm:"type:VARBINARY(42);index;" json:"PersonUID" yaml:"PersonUID,omitempty"` Embedding string `gorm:"type:LONGTEXT;" json:"Embedding" yaml:"Embedding,omitempty"` CreatedAt time.Time `json:"CreatedAt" yaml:"CreatedAt,omitempty"` UpdatedAt time.Time `json:"UpdatedAt" yaml:"UpdatedAt,omitempty"` diff --git a/internal/entity/src.go b/internal/entity/src.go index ebc4844f6..f17f3828c 100644 --- a/internal/entity/src.go +++ b/internal/entity/src.go @@ -9,11 +9,12 @@ const ( SrcAuto = "" SrcManual = "manual" SrcEstimate = "estimate" - SrcPeople = "people" SrcName = "name" SrcMeta = "meta" SrcXmp = "xmp" SrcYaml = "yaml" + SrcPeople = "people" + SrcMarker = "marker" SrcImage = classify.SrcImage SrcKeyword = classify.SrcKeyword SrcLocation = classify.SrcLocation @@ -23,10 +24,11 @@ const ( var SrcPriority = Priorities{ SrcAuto: 1, SrcEstimate: 2, - SrcPeople: 2, SrcName: 4, SrcYaml: 8, SrcLocation: 8, + SrcPeople: 8, + SrcMarker: 8, SrcImage: 8, SrcKeyword: 16, SrcMeta: 16, diff --git a/internal/photoprism/people.go b/internal/photoprism/people.go index 34f1a6c84..b24cfb50b 100644 --- a/internal/photoprism/people.go +++ b/internal/photoprism/people.go @@ -6,6 +6,8 @@ import ( "runtime/debug" "time" + "github.com/photoprism/photoprism/pkg/txt" + "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/mutex" @@ -124,15 +126,15 @@ func (m *People) Start() (err error) { return err } - uidMap := make(map[string]string, len(peopleFaces)) - faceMap := make(map[string]entity.Embedding, len(peopleFaces)) + type Face = struct { + Embedding entity.Embedding + PersonUID string + } + + faceMap := make(map[string]Face, len(peopleFaces)) for _, f := range peopleFaces { - faceMap[f.ID] = f.UnmarshalEmbedding() - - if f.PersonUID != "" { - uidMap[f.ID] = f.PersonUID - } + faceMap[f.ID] = Face{f.UnmarshalEmbedding(), f.PersonUID} } limit := 500 @@ -157,20 +159,38 @@ func (m *People) Start() (err error) { var faceId string var faceDist float64 - for _, e1 := range marker.UnmarshalEmbeddings() { - for id, e2 := range faceMap { - if d := clusters.EuclideanDistance(e1, e2); faceId == "" || d < faceDist { + for _, e := range marker.UnmarshalEmbeddings() { + for id, f := range faceMap { + if d := clusters.EuclideanDistance(e, f.Embedding); faceId == "" || d < faceDist { faceId = id faceDist = d } } } - if marker.RefUID != "" && marker.RefUID == uidMap[faceId] { + if faceId == "" { continue } - if refUID := uidMap[faceId]; refUID != "" { + if marker.RefUID != "" && marker.RefUID == faceMap[faceId].PersonUID { + continue + } + + // Create person from marker label? + if marker.MarkerLabel == "" { + // Do nothing. + } else if person := entity.NewPerson(marker.MarkerLabel, entity.SrcMarker, 1); person == nil { + log.Errorf("people: person should not be nil - bug?") + } else if person = entity.FirstOrCreatePerson(person); person == nil { + log.Errorf("people: failed adding %s", txt.Quote(marker.MarkerLabel)) + } else if f, ok := faceMap[faceId]; ok { + faceMap[faceId] = Face{Embedding: f.Embedding, PersonUID: person.PersonUID} + entity.Db().Model(&entity.PersonFace{}).Where("id = ?", faceId).Update("PersonUID", person.PersonUID) + log.Infof("people: added %s", txt.Quote(person.PersonName)) + } + + // Existing person? + if refUID := faceMap[faceId].PersonUID; refUID != "" { if err := marker.Updates(entity.Val{"RefUID": refUID, "RefSrc": entity.SrcPeople, "FaceID": ""}); err != nil { log.Errorf("people: %s while updating person uid", err) } else { @@ -188,7 +208,7 @@ func (m *People) Start() (err error) { time.Sleep(50 * time.Millisecond) } - log.Infof("people: %d faces added, %d recognized, %d markers updated, %d errors", addedFaces, recognized, markersUpdated, updateErrors) + log.Infof("people: %d faces added, %d recognized, %d unknown, %d errors", addedFaces, recognized, markersUpdated, updateErrors) return nil }