From 24eff21aa485dcf4076fe8d70038e10bb7872b4a Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Wed, 29 Sep 2021 20:09:34 +0200 Subject: [PATCH] Search: Default to photo names and keywords #1517 #1560 Default to photo name when search term is too short or on the stop list. Search full text index otherwise, which now include names of people (requires reindexing). --- internal/api/marker.go | 10 ++-- internal/api/subject.go | 3 +- internal/entity/face.go | 7 +++ internal/entity/faces.go | 35 +++++++++++++ internal/entity/faces_test.go | 10 ++++ internal/entity/keyword.go | 2 +- internal/entity/keyword_fixtures.go | 5 ++ internal/entity/marker.go | 46 +++++++++++++---- internal/entity/marker_fixtures.go | 20 ++++++++ internal/entity/marker_test.go | 27 +++++++++- internal/entity/photo.go | 1 + internal/entity/photo_keyword_fixtures.go | 4 ++ internal/entity/photo_title.go | 5 ++ internal/entity/subject.go | 10 ++-- internal/entity/subject_test.go | 4 +- internal/entity/subjects.go | 42 ++++++++++++++++ internal/entity/subjects_test.go | 15 ++++++ internal/i18n/messages.go | 2 + internal/photoprism/faces.go | 16 ++++++ internal/photoprism/faces_audit.go | 26 ++++++++++ internal/query/faces_test.go | 2 +- internal/query/subjects.go | 2 +- internal/search/like.go | 22 ++++++++ internal/search/like_test.go | 21 ++++++++ internal/search/photos.go | 61 +++++++++++------------ internal/search/photos_geo.go | 29 ++++------- internal/search/photos_test.go | 18 ++++++- internal/workers/meta.go | 18 +++---- pkg/txt/search.go | 16 ++++++ pkg/txt/search_test.go | 19 +++++++ pkg/txt/strings.go | 19 +++++++ pkg/txt/strings_test.go | 18 +++++++ pkg/txt/words.go | 21 -------- pkg/txt/words_test.go | 12 ----- 34 files changed, 449 insertions(+), 119 deletions(-) create mode 100644 internal/entity/subjects.go create mode 100644 internal/entity/subjects_test.go create mode 100644 pkg/txt/search.go create mode 100644 pkg/txt/search_test.go diff --git a/internal/api/marker.go b/internal/api/marker.go index 494b68184..1b14872e2 100644 --- a/internal/api/marker.go +++ b/internal/api/marker.go @@ -26,7 +26,7 @@ func findFileMarker(c *gin.Context) (file *entity.File, marker *entity.Marker, e // Check feature flags. conf := service.Config() - if !conf.Settings().Features.People || !conf.Settings().Features.Edit { + if !conf.Settings().Features.People { AbortFeatureDisabled(c) return nil, nil, fmt.Errorf("feature disabled") } @@ -70,27 +70,27 @@ func UpdateMarker(router *gin.RouterGroup) { file, marker, err := findFileMarker(c) if err != nil { - log.Debugf("api: %s (update marker)", err) + log.Debugf("marker: %s (find)", err) return } markerForm, err := form.NewMarker(*marker) if err != nil { - log.Errorf("photo: %s (new marker form)", err) + log.Errorf("marker: %s (new form)", err) AbortSaveFailed(c) return } if err := c.BindJSON(&markerForm); err != nil { - log.Errorf("photo: %s (update marker form)", err) + log.Errorf("marker: %s (update form)", err) AbortBadRequest(c) return } // Save marker. if changed, err := marker.SaveForm(markerForm); err != nil { - log.Errorf("photo: %s (save marker form)", err) + log.Errorf("marker: %s", err) AbortSaveFailed(c) return } else if changed { diff --git a/internal/api/subject.go b/internal/api/subject.go index 7e34e194f..18a8f3185 100644 --- a/internal/api/subject.go +++ b/internal/api/subject.go @@ -106,7 +106,8 @@ func UpdateSubject(router *gin.RouterGroup) { } if _, err := m.UpdateName(f.SubjName); err != nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": txt.UcFirst(err.Error())}) + log.Errorf("subject: %s", err) + AbortSaveFailed(c) return } diff --git a/internal/entity/face.go b/internal/entity/face.go index 4ab8da19d..1e7eb44c6 100644 --- a/internal/entity/face.go +++ b/internal/entity/face.go @@ -304,6 +304,13 @@ func (m *Face) Create() error { // Delete removes the face from the database. func (m *Face) Delete() error { + // Remove face id from markers before deleting. + if err := Db().Model(&Marker{}). + Where("face_id = ?", m.ID). + UpdateColumns(Values{"face_id": "", "face_dist": -1}).Error; err != nil { + return err + } + return Db().Delete(m).Error } diff --git a/internal/entity/faces.go b/internal/entity/faces.go index 0fae0527a..d2078b836 100644 --- a/internal/entity/faces.go +++ b/internal/entity/faces.go @@ -1,5 +1,7 @@ package entity +import "fmt" + // Faces represents a Face slice. type Faces []Face @@ -20,3 +22,36 @@ func (f Faces) IDs() (ids []string) { return ids } + +// Delete (soft) deletes all subjects. +func (f Faces) Delete() error { + for _, m := range f { + if err := m.Delete(); err != nil { + return err + } + } + + return nil +} + +// OrphanFaces returns unused faces. +func OrphanFaces() (Faces, error) { + orphans := Faces{} + + err := Db(). + Where(fmt.Sprintf("id NOT IN (SELECT DISTINCT face_id FROM %s)", Marker{}.TableName())). + Find(&orphans).Error + + return orphans, err +} + +// DeleteOrphanFaces finds and (soft) deletes all unused face clusters. +func DeleteOrphanFaces() (count int, err error) { + orphans, err := OrphanFaces() + + if err != nil { + return 0, err + } + + return len(orphans), orphans.Delete() +} diff --git a/internal/entity/faces_test.go b/internal/entity/faces_test.go index 3762bfc69..f0344c221 100644 --- a/internal/entity/faces_test.go +++ b/internal/entity/faces_test.go @@ -21,3 +21,13 @@ func TestFaces_IDs(t *testing.T) { r := Faces{m, m1}.IDs() assert.Equal(t, []string{"VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG6", "VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG7"}, r) } + +func TestDeleteOrphanFaces(t *testing.T) { + t.Run("Success", func(t *testing.T) { + if count, err := DeleteOrphanFaces(); err != nil { + t.Fatal(err) + } else { + t.Logf("deleted %d faces", count) + } + }) +} diff --git a/internal/entity/keyword.go b/internal/entity/keyword.go index 8a3b815d6..aeed93c4f 100644 --- a/internal/entity/keyword.go +++ b/internal/entity/keyword.go @@ -32,7 +32,7 @@ func (m *Keyword) Updates(values interface{}) error { return UnscopedDb().Model(m).UpdateColumns(values).Error } -// Updates a column in the database. +// Update a column in the database. func (m *Keyword) Update(attr string, value interface{}) error { return UnscopedDb().Model(m).UpdateColumn(attr, value).Error } diff --git a/internal/entity/keyword_fixtures.go b/internal/entity/keyword_fixtures.go index 6dd03bfe6..4bf787341 100644 --- a/internal/entity/keyword_fixtures.go +++ b/internal/entity/keyword_fixtures.go @@ -39,6 +39,11 @@ var KeywordFixtures = KeywordMap{ Keyword: "kuh", Skip: false, }, + "actress": { + ID: 1000004, + Keyword: "actress", + Skip: false, + }, } // CreateKeywordFixtures inserts known entities into the database for testing. diff --git a/internal/entity/marker.go b/internal/entity/marker.go index a8793fe0d..1f624f62c 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "math" - "strings" "time" "github.com/jinzhu/gorm" @@ -122,6 +121,29 @@ func (m *Marker) Update(attr string, value interface{}) error { return UnscopedDb().Model(m).Update(attr, value).Error } +// SetName changes the marker name. +func (m *Marker) SetName(name, src string) (changed bool, err error) { + if src == SrcAuto || SrcPriority[src] < SrcPriority[m.SubjSrc] { + return false, nil + } + + name = txt.NormalizeName(name) + + if name == "" { + return false, nil + } + + if m.MarkerName == name { + // Name didn't change. + return false, nil + } + + m.SubjSrc = src + m.MarkerName = name + + return true, m.SyncSubject(true) +} + // SaveForm updates the entity using form data and stores it in the database. func (m *Marker) SaveForm(f form.Marker) (changed bool, err error) { if m.MarkerInvalid != f.MarkerInvalid { @@ -134,14 +156,9 @@ func (m *Marker) SaveForm(f form.Marker) (changed bool, err error) { changed = true } - if f.SubjSrc == SrcManual && strings.TrimSpace(f.MarkerName) != "" && f.MarkerName != m.MarkerName { - m.SubjSrc = SrcManual - m.MarkerName = txt.NormalizeName(f.MarkerName) - - if err := m.SyncSubject(true); err != nil { - return changed, err - } - + if nameChanged, err := m.SetName(f.MarkerName, f.SubjSrc); err != nil { + return changed, err + } else if nameChanged { changed = true } @@ -367,6 +384,7 @@ func (m *Marker) Subject() (subj *Subject) { // Create subject? if m.SubjSrc != SrcAuto && m.MarkerName != "" && m.SubjUID == "" { if subj = NewSubject(m.MarkerName, SubjPerson, m.SubjSrc); subj == nil { + log.Errorf("marker: invalid subject %s", txt.Quote(m.MarkerName)) return nil } else if subj = FirstOrCreateSubject(subj); subj == nil { log.Debugf("marker: invalid subject %s", txt.Quote(m.MarkerName)) @@ -391,6 +409,16 @@ func (m *Marker) ClearSubject(src string) error { m.face = FindFace(m.FaceID) } + defer func() { + // Find and (soft) delete unused subjects. + start := time.Now() + if count, err := DeleteOrphanPeople(); err != nil { + log.Errorf("marker: %s while removing unused subjects [%s]", err, time.Since(start)) + } else if count > 0 { + log.Debugf("marker: removed %d people [%s]", count, time.Since(start)) + } + }() + // Update index & resolve collisions. if err := m.Updates(Values{"MarkerName": "", "FaceID": "", "FaceDist": -1.0, "SubjUID": "", "SubjSrc": src}); err != nil { return err diff --git a/internal/entity/marker_fixtures.go b/internal/entity/marker_fixtures.go index 23e64d710..b43d8e491 100644 --- a/internal/entity/marker_fixtures.go +++ b/internal/entity/marker_fixtures.go @@ -345,6 +345,26 @@ var MarkerFixtures = MarkerMap{ Size: 509, Score: 100, }, + "mqzop6s14ahkyd24": Marker{ //19800101_000002_D640C559 + MarkerUID: "mqzop6s14ahkyd24", + FileUID: "ft3es39w45bnlqdw", + Thumb: "acad9168fa6acc5c5c2965ddf6ec465ca42fd818", + FaceID: "VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG6", + FaceDist: 0.3139983399779298, + SubjSrc: "", + SubjUID: "", + MarkerSrc: SrcImage, + MarkerType: MarkerFace, + MarkerName: "", + LandmarksJSON: []byte{}, + EmbeddingsJSON: []byte("[[0.10730543085474682,-0.007740289179353713,0.04013410115400314,0.01458170011165962,-0.033333988977870946,0.06636234022813034,-0.00010941258007316575,0.0266348918046072,-0.05017391628723953,0.026034562221256254,-0.03388911566430759,-0.03461048494812202,0.040559725024994844,0.02683793627304573,-0.00972269717541027,-0.07836494561032105,-0.022470260049817198,0.011276674801708603,-0.05526434009558201,0.014401617237932205,-0.031258523568474236,-0.05416104192368187,-0.05567222379755878,0.017950877029356768,-0.016397424193561935,0.062346790423413276,-0.019043469394284057,0.04085343435437774,-0.05627231374819698,0.002355368169155769,0.07268979656775187,-0.0015096598716884627,-0.030188596848975374,-0.030941932784964564,-0.02826790015985233,-0.05420075791573048,-0.015742074680253033,0.019360258910790157,-0.008227027287290193,-0.08797317745792674,-0.07358703463505077,0.09688007249803735,0.015168583267354964,-0.034569315837825396,0.054231690688333986,0.018033145214487362,0.01579093209709463,-0.09204238994311237,0.08645247031890774,-0.10499936100221444,0.022421303168151857,0.005288450124515152,-0.017391072021601867,0.011218363053184624,-0.08478270589438915,0.0038618527485391615,-0.023381522484070015,-0.05428399272960853,-0.049397680150033,0.04006855272634697,-0.056704127236808774,-0.00958812557516262,-0.024006645464504622,-0.0073450501057456975,-0.03138197361666756,-0.013765138786817361,0.01162637563227787,0.0023935177775817563,0.08953138773108768,0.05337418268588829,-0.012870218945196915,0.03652425150877475,-0.027783526080406188,-0.019489638927241745,-0.01591402705199299,-0.005031992847164803,-0.014592982936285286,-0.03540697236418762,0.015595597412254449,0.004689344744109726,-0.009276015175478172,0.0058068592886232374,0.10480716412028504,0.0169216338187767,-0.0159497901004467,0.04574707649004688,-0.012214007484710122,-0.04849749380776977,0.054958586523843764,0.055898306713647844,-0.05052226642217827,0.008803732924324036,0.02326267119630642,0.047305830959801676,0.04497242295638694,0.020850376996620942,0.01314765746152954,-0.06768179592533874,0.05844347174572754,-0.03379152370001783,0.009412363744416903,0.04876727547273407,0.03299943491180715,0.01981742466488743,0.0547951049219265,0.020208802772381018,-0.08163521584288311,-0.038910958658009524,-0.004049565234655952,-0.02227413252290535,-0.0176418922441086,0.0568860088455925,-0.03240221023084612,0.0018760896289435579,-0.03234445138420723,0.007601825631139565,0.004916589611196899,-0.07292478312312889,0.021712048014592936,0.008807552270011749,0.0045489283733609,0.018861112444398878,-0.0341377092368577,-0.06305481604926585,0.039113288345403674,-0.01390809621003151,-0.04930861238819008,0.02377057523982868,0.019087416355893325,-0.013899296822125817,0.02251690464443226,0.08074113913260841,-0.018922226267959787,0.07189693789385795,0.060660045672045707,-0.023638294307546808,-0.006141792394255906,-0.06662582397409247,-0.013895529799502565,0.016608829923953898,-0.00390724028582611,0.05038048671591301,-0.015355035841564064,-0.0008532485082750321,-0.004694504582768126,-0.016610601585741958,0.008180847821889228,-0.04035771976174698,0.01847608156922703,0.08409907464663602,-0.029978496458568385,-0.06499117178372192,0.07448235046571827,0.10142187900247382,-0.023405319141915855,0.05237413796294822,0.04315940939233541,-0.02349721355909328,0.012594679585403442,-0.10457832859776592,0.001468614040066719,0.0165479676672657,-0.07708675453700256,-0.05102918249748802,0.045642631412478735,-0.004785828004440499,0.0203317336356945,-0.02006395174473057,0.04201285855375195,0.032883700123707296,0.0477916040669878,0.08070634492548084,-0.09245629058029556,0.05112703265588493,-0.006224603994954872,-0.0005257819460310555,0.005513055457300567,0.02521921247766018,0.012207323409280013,-0.009936333046208725,-0.007426916158089448,0.027260071572856714,0.006004036209329835,-0.039462719505699156,0.04428369084658737,0.005021041270120048,0.00955255667591259,-0.024385389176467896,0.06930311634011002,-0.0389855151682066,0.009325780048200605,0.0067487294106089776,-0.0538568250434906,-0.04132319716445885,0.005287871307727813,0.02836177144018917,0.016369665767238237,-0.02612976718916588,0.0781344821977253,0.0124423230052565,0.007052016124275589,0.07093038059380721,-0.040975969278632354,0.05987170546998787,0.0429845054696949,0.06377765311330413,-0.054260408781722336,0.017124075467253648,0.011034745989844896,-0.01129856537228031,-0.03058279355517101,-0.052326615682374664,0.06340472755555274,-0.007235566082305412,0.08209440086026383,-0.0037407900405261995,-0.02100836190159107,0.051361881913555715,0.035520336595121764,0.019260735587613487,0.04814414379586697,-0.010566343916274241,-0.03353529212573547,0.05283452853813282,-0.027749841873006824,-0.03820509264906912,-0.0015166780129867554,0.02487160170807457,0.03048850776525669,-0.030538799520168875,0.0921192664219265,0.03269134465648327,-0.031787506815418434,-0.01908650508301182,0.05982613160244779,-0.053232109332236294,-0.03650761934345913,0.0026813365359365463,0.032588356136758805,-0.032364926929593085,0.07780626359498405,0.044541174425177764,0.011626562325897788,0.03554684517681643,-0.030510870967539787,-0.04088990689640999,0.07105028789278889,0.03138784011465073,-0.06342823476303319,0.09164142434876824,-0.0112280279000453,-0.04595559070266152,0.08798781996626949,0.026803936697537615,-0.0014241986940294257,-0.020834715982498548,0.023556784775891685,0.008996215819517326,-0.0012677171940084454,-0.07692881668502807,0.024615258007191814,0.02948731386628723,0.06911119150276565,-0.041541930091072085,0.07317672894504547,-0.012252912262506771,-0.03429316172188286,0.03286905748134327,0.025736928383919527,0.003926683415351601,0.006255630871762562,-0.020806247741813468,0.0675457214042778,0.007579460672946357,0.012004441173839569,-0.028187582314963343,-0.0018772867526912688,-0.01844064376148571,-0.05389302147970715,-0.04154738243111,-0.05912346626385308,-0.003186127453911171,-0.015869915592464562,0.036601020266580665,-0.08332522355102062,-0.015594113206121387,0.010554298175920372,0.009863903175943527,-0.04408378851017952,-0.01321298950931368,-0.026788807387387467,-0.00905998101737915,-0.07901183432849217,0.022626760559060342,0.05966787504726859,-0.0373913765745697,-0.00620443077226124,-0.005321248754354935,-0.05629461318153381,-0.04339327553344822,0.03066110013017902,-0.0560899433873785,0.029585001932263853,-0.06142458606396866,0.018855098215825178,0.03336997769082436,-0.07772387048591708,0.02869667860757885,0.04751144987925148,0.07131169258731747,0.01554444873138424,-0.019102520424152183,-0.06713599618322277,0.021553602847260475,0.022784952935549926,-0.07224605423420524,-0.03428428022313595,0.025510370273970604,-0.042455744666400605,0.024999596293880464,0.0007267671517935561,-0.007103063657435513,0.051193967364198685,-0.03918299151588478,-0.05340270113635778,-0.0005553757678619388,-0.04361415384515381,-0.05659870360464592,-0.003001301568729019,-0.10493783691904449,0.007865782491956196,-0.010459198798326888,0.03839990013440418,-0.029396389004837837,0.04123072916591454,-0.003870788638888664,0.011576299454542732,0.021793958225202522,0.0013144587776207917,-0.024084851461598205,-0.007895128372669067,0.02794634672595444,0.013256276108802492,-0.06581846043538475,-0.03512838380870453,0.010219935781849479,0.041956290830379675,-0.02193645334812136,0.036522118692461206,-0.04014683200312634,-0.007509486720670319,0.025035869046040268,0.03341998480559387,-0.03562761249035026,0.04892323307058029,-0.030771232001644132,-0.016917612628533363,0.002604945885121918,-0.044643074882380486,0.01154372547133419,-0.021955625942386627,0.018907365975515553,0.03550167291446045,0.01069377167082758,0.00010183658435096768,-0.04899959038740444,0.04724968668608978,-0.01864932432341235,0.0591259089168789,0.07907125494612216,0.028897156624642945,0.01633692932619137,0.06420496597867965,0.018129071607111358,-0.06522170992608013,-0.03939954941189146,0.04130569647272587,0.04419998725251961,0.04542913027885341,0.018470383181769943,0.008568164058957863,-0.06659949697784996,-0.053012251715078354,-0.020253768667636778,-0.0428765378002491,0.07184141544699574,0.02058260375849676,-0.03779574153167915,0.0021254573788347247,0.00922705617390442,-0.06903300705643031,0.048223514541707424,-0.008124176700399017,0.06623217639861775,0.011399885879904556,0.13320644195552445,-0.015707634324862062,0.004298537653726769,0.007440328888434029,-0.03552852988830433,0.006549433453245544,-0.019685784628289793,0.001693401851796341,0.050209905835451124,0.023254144681681632,-0.04905436637160683,-0.01058279299507389,0.06261640349854469,-0.07554102380998802,-0.010803172210683784,0.04001997347501145,-0.013296409033855438,0.056829244201244355,0.029110596151947783,0.006392164909307861,-0.0035876165295053485,-0.019022594469099045,-0.06487911801050472,0.02178870949222603,0.05293369045270252,0.0014374271403566358,0.02058438161717472,-0.05258523574003887,-0.03312468141761551,0.051533518133239746,0.03929023312081566,-0.07294044148252202,0.01607557897360134,-0.0007034383966050719,0.014925192443655966,0.051449392859268764,-0.06079890988933106,-0.04363216685223599,0.028568039422766974,0.045766175851156804,0.07275596444172669,-0.02276483221781349,0.09294405910429002,0.06625853254336929,-0.04167032707059745,-0.04751508792911625,-0.014774199240300752,0.023224616626467326,-0.01281115503053608,0.03472993899021339,0.008343472536062031,-0.011408440443860645,0.004419146704378701,0.05045044130775452,-0.03518939370823498,-0.04170123182437134,0.022208642446600917,0.07141607704347333,-0.04112406919064011,0.03227901691925602,0.03527487398910847,-0.029543274091153718,0.005872693907862854,-0.008123872357421475,-0.058780187362098696,-0.00027467445847377796,0.024044984289353373,0.057634827237228584,-0.04450547877367153,-0.03946884506688686,-0.02006971111822632,0.006139106799438476,0.014848452844277668,-0.040448605585189826,-0.047422823475079534,0.00047739853115692137,-0.03920787799786568,-0.05102518756346798,0.029106281725284004,0.023013759328845976,-0.0181101632871727,0.003943383191735267,-0.11744085779379082,0.00652325401639185,-0.0016088291387550352,0.004582751362570763,0.06564233218507957,0.014525142593546867,-0.05397913284980278,-0.005146496768864823,0.008265835225847246,-0.09204165418391608,-0.023673615795413973,0.016221329961976054,0.0560354235721693,-0.03387280199538708,0.011243025140723228,0.02789629877560217,0.07942785398379296,0.019745293456116107,-0.03951280953572121,-0.0325216371505229,-0.04877831216997623,0.008021598871560669,0.06607214515587043,0.08340918698473548,-0.06638043362871171,0.0003533690162649157,-0.05787711264029312,0.017585791805968413,-0.004768172475530777,-0.031721018591366806,0.059853391075907716,0.08903246940908241,0.00910143805785122,-0.02198764055408287,0.023417301139897727]]"), + X: 0.1, + Y: 0.229688, + W: 0.246334, + H: 0.29707, + Size: 209, + Score: 55, + }, } // CreateMarkerFixtures inserts known entities into the database for testing. diff --git a/internal/entity/marker_test.go b/internal/entity/marker_test.go index 29decfafe..3d7c470b6 100644 --- a/internal/entity/marker_test.go +++ b/internal/entity/marker_test.go @@ -34,6 +34,31 @@ func TestNewMarker(t *testing.T) { assert.Equal(t, MarkerLabel, m.MarkerType) } +func TestMarker_SetName(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + m := MarkerFixtures.Get("actress-a-1") + assert.IsType(t, Marker{}, m) + assert.Equal(t, "Actress A", m.MarkerName) + changed, err := m.SetName("", SrcManual) + + if err != nil { + t.Fatal(err) + } + + assert.False(t, changed) + assert.Equal(t, "Actress A", m.MarkerName) + + changed, err = m.SetName("Foo Bar", SrcAuto) + + if err != nil { + t.Fatal(err) + } + + assert.False(t, changed) + assert.Equal(t, "Actress A", m.MarkerName) + }) +} + func TestMarker_SaveForm(t *testing.T) { t.Run("fa-ge add new name to marker then rename marker", func(t *testing.T) { m := MarkerFixtures.Get("fa-gr-1") @@ -456,7 +481,7 @@ func TestMarker_GetFace(t *testing.T) { if f := m.Face(); f == nil { t.Fatal("return value must not be nil") } else { - assert.Equal(t, "jqy3y652h8njw0sx", f.SubjUID) + assert.Equal(t, "VF7ANLDET2BKZNT4VQWJMMC6HBEFDOG6", f.ID) } }) t.Run("low quality marker", func(t *testing.T) { diff --git a/internal/entity/photo.go b/internal/entity/photo.go index 967379e72..b3853b3ab 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -418,6 +418,7 @@ func (m *Photo) IndexKeywords() error { // Add title, description and other keywords keywords = append(keywords, txt.Keywords(m.PhotoTitle)...) keywords = append(keywords, txt.Keywords(m.PhotoDescription)...) + keywords = append(keywords, m.SubjectKeywords()...) keywords = append(keywords, txt.Words(details.Keywords)...) keywords = append(keywords, txt.Keywords(details.Subject)...) keywords = append(keywords, txt.Keywords(details.Artist)...) diff --git a/internal/entity/photo_keyword_fixtures.go b/internal/entity/photo_keyword_fixtures.go index 7c3e8e88d..006ebf941 100644 --- a/internal/entity/photo_keyword_fixtures.go +++ b/internal/entity/photo_keyword_fixtures.go @@ -27,6 +27,10 @@ var PhotoKeywordFixtures = PhotoKeywordMap{ PhotoID: 1000023, KeywordID: 1000003, }, + "7": { + PhotoID: 1000027, + KeywordID: 1000004, + }, } // CreatePhotoKeywordFixtures inserts known entities into the database for testing. diff --git a/internal/entity/photo_title.go b/internal/entity/photo_title.go index 81f105417..dbce34591 100644 --- a/internal/entity/photo_title.go +++ b/internal/entity/photo_title.go @@ -244,3 +244,8 @@ func (m *Photo) SubjectNames() []string { return nil } + +// SubjectKeywords returns keywords for all known subject names. +func (m *Photo) SubjectKeywords() []string { + return txt.Words(strings.Join(m.SubjectNames(), " ")) +} diff --git a/internal/entity/subject.go b/internal/entity/subject.go index be1080c7f..2db1fb046 100644 --- a/internal/entity/subject.go +++ b/internal/entity/subject.go @@ -15,9 +15,6 @@ import ( var subjectMutex = sync.Mutex{} -// Subjects represents a list of subjects. -type Subjects []Subject - // Subject represents a named photo subject, typically a person. type Subject struct { SubjUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"UID" yaml:"UID"` @@ -100,6 +97,9 @@ func (m *Subject) Delete() error { return nil } + subjectMutex.Lock() + defer subjectMutex.Unlock() + log.Infof("subject: deleting %s %s", m.SubjType, txt.Quote(m.SubjName)) event.EntitiesDeleted("subjects", []string{m.SubjUID}) @@ -111,6 +111,10 @@ func (m *Subject) Delete() error { }) } + if err := Db().Model(&Face{}).Where("subj_uid = ?", m.SubjUID).Update("subj_uid", "").Error; err != nil { + return err + } + return Db().Delete(m).Error } diff --git a/internal/entity/subject_test.go b/internal/entity/subject_test.go index 3a6162855..8b28d5ed1 100644 --- a/internal/entity/subject_test.go +++ b/internal/entity/subject_test.go @@ -32,7 +32,7 @@ func TestNewSubject(t *testing.T) { } func TestSubject_SetName(t *testing.T) { - t.Run("success", func(t *testing.T) { + t.Run("Success", func(t *testing.T) { m := NewSubject("Jens Mander", SubjPerson, SrcAuto) assert.Equal(t, "Jens Mander", m.SubjName) @@ -45,7 +45,7 @@ func TestSubject_SetName(t *testing.T) { assert.Equal(t, "Foo McBar", m.SubjName) assert.Equal(t, "foo-mcbar", m.SubjSlug) }) - t.Run("new name empty", func(t *testing.T) { + t.Run("Empty", func(t *testing.T) { m := NewSubject("Jens Mander", SubjPerson, SrcAuto) assert.Equal(t, "Jens Mander", m.SubjName) diff --git a/internal/entity/subjects.go b/internal/entity/subjects.go new file mode 100644 index 000000000..946512e8c --- /dev/null +++ b/internal/entity/subjects.go @@ -0,0 +1,42 @@ +package entity + +import ( + "fmt" +) + +// Subjects represents a list of subjects. +type Subjects []Subject + +// Delete (soft) deletes all subjects. +func (m Subjects) Delete() error { + for _, subj := range m { + if err := subj.Delete(); err != nil { + return err + } + } + + return nil +} + +// OrphanPeople returns unused subjects. +func OrphanPeople() (Subjects, error) { + orphans := Subjects{} + + err := Db(). + Where("subj_type = ?", SubjPerson). + Where(fmt.Sprintf("subj_uid NOT IN (SELECT DISTINCT subj_uid FROM %s)", Marker{}.TableName())). + Find(&orphans).Error + + return orphans, err +} + +// DeleteOrphanPeople finds and (soft) deletes all unused people. +func DeleteOrphanPeople() (count int, err error) { + subj, err := OrphanPeople() + + if err != nil { + return 0, err + } + + return len(subj), subj.Delete() +} diff --git a/internal/entity/subjects_test.go b/internal/entity/subjects_test.go new file mode 100644 index 000000000..cd14e27fb --- /dev/null +++ b/internal/entity/subjects_test.go @@ -0,0 +1,15 @@ +package entity + +import ( + "testing" +) + +func TestDeleteOrphanPeople(t *testing.T) { + t.Run("Success", func(t *testing.T) { + if count, err := DeleteOrphanPeople(); err != nil { + t.Fatal(err) + } else { + t.Logf("deleted %d faces", count) + } + }) +} diff --git a/internal/i18n/messages.go b/internal/i18n/messages.go index bc095f011..782cb5964 100644 --- a/internal/i18n/messages.go +++ b/internal/i18n/messages.go @@ -33,6 +33,7 @@ const ( ErrZipFailed ErrInvalidCredentials ErrInvalidLink + ErrInvalidName MsgChangesSaved MsgAlbumCreated @@ -110,6 +111,7 @@ var Messages = MessageMap{ ErrZipFailed: gettext("Failed to create zip file"), ErrInvalidCredentials: gettext("Invalid credentials"), ErrInvalidLink: gettext("Invalid link"), + ErrInvalidName: gettext("Invalid name"), // Info and confirmation messages: MsgChangesSaved: gettext("Changes successfully saved"), diff --git a/internal/photoprism/faces.go b/internal/photoprism/faces.go index 631956b53..58c352df0 100644 --- a/internal/photoprism/faces.go +++ b/internal/photoprism/faces.go @@ -131,6 +131,22 @@ func (w *Faces) Start(opt FacesOptions) (err error) { log.Debugf("faces: %d markers updated, %d faces recognized, %d unknown [%s]", matches.Updated, matches.Recognized, matches.Unknown, time.Since(start)) } + // Remove unused people. + start = time.Now() + if count, err := entity.DeleteOrphanPeople(); err != nil { + log.Errorf("faces: %s (remove people)", err) + } else if count > 0 { + log.Debugf("faces: removed %d people [%s]", count, time.Since(start)) + } + + // Remove unused face clusters. + start = time.Now() + if count, err := entity.DeleteOrphanFaces(); err != nil { + log.Errorf("faces: %s (remove clusters)", err) + } else if count > 0 { + log.Debugf("faces: removed %d clusters [%s]", count, time.Since(start)) + } + return nil } diff --git a/internal/photoprism/faces_audit.go b/internal/photoprism/faces_audit.go index 090b6dfea..2e51c17c7 100644 --- a/internal/photoprism/faces_audit.go +++ b/internal/photoprism/faces_audit.go @@ -117,5 +117,31 @@ func (w *Faces) Audit(fix bool) (err error) { } } + // Find and fix orphan faces clusters. + if orphans, err := entity.OrphanFaces(); err != nil { + log.Errorf("%s while finding orphan faces", err) + } else if l := len(orphans); l == 0 { + log.Infof("found no orphan faces clusters") + } else if !fix { + log.Infof("found %d orphan faces clusters", l) + } else if err := orphans.Delete(); err != nil { + log.Errorf("failed fixing %d orphan faces clusters: %s", l, err) + } else { + log.Infof("removed %d orphan faces clusters", l) + } + + // Find and fix orphan people. + if orphans, err := entity.OrphanPeople(); err != nil { + log.Errorf("%s while finding orphan people", err) + } else if l := len(orphans); l == 0 { + log.Infof("found no orphan people") + } else if !fix { + log.Infof("found %d orphan people", l) + } else if err := orphans.Delete(); err != nil { + log.Errorf("failed fixing %d orphan people: %s", l, err) + } else { + log.Infof("removed %d orphan people", l) + } + return nil } diff --git a/internal/query/faces_test.go b/internal/query/faces_test.go index 905b22924..583b24421 100644 --- a/internal/query/faces_test.go +++ b/internal/query/faces_test.go @@ -79,7 +79,7 @@ func TestMatchFaceMarkers(t *testing.T) { t.Fatal(err) } - assert.Equal(t, int64(1), affected) + assert.Equal(t, int64(2), affected) if m, err := MarkerByUID(faceFixtureId); err != nil { t.Fatal(err) diff --git a/internal/query/subjects.go b/internal/query/subjects.go index b81bbffc4..dd7369a34 100644 --- a/internal/query/subjects.go +++ b/internal/query/subjects.go @@ -93,7 +93,7 @@ func CreateMarkerSubjects() (affected int64, err error) { if name == m.MarkerName && subj != nil { // Do nothing. } else if subj = entity.NewSubject(m.MarkerName, entity.SubjPerson, entity.SrcMarker); subj == nil { - log.Errorf("faces: subject should not be nil - bug?") + log.Errorf("faces: invalid subject %s", txt.Quote(m.MarkerName)) continue } else if subj = entity.FirstOrCreateSubject(subj); subj == nil { log.Errorf("faces: failed adding subject %s", txt.Quote(m.MarkerName)) diff --git a/internal/search/like.go b/internal/search/like.go index c5602794b..f988c2906 100644 --- a/internal/search/like.go +++ b/internal/search/like.go @@ -235,3 +235,25 @@ func AnyInt(col, numbers, sep string, min, max int) (where string) { return strings.Join(wheres, " OR ") } + +// OrLike returns a where condition and values for finding multiple terms combined with OR. +func OrLike(col, s string) (where string, values []interface{}) { + if col == "" || s == "" { + return "", []interface{}{} + } + + s = strings.ReplaceAll(s, "*", "%") + s = strings.ReplaceAll(s, "%%", "%") + + terms := strings.Split(s, txt.Or) + values = make([]interface{}, len(terms)) + + for i := range terms { + values[i] = terms[i] + } + + like := fmt.Sprintf("%s LIKE ?", col) + where = like + strings.Repeat(" OR "+like, len(terms)-1) + + return where, values +} diff --git a/internal/search/like_test.go b/internal/search/like_test.go index e49c7ba65..d9b1be202 100644 --- a/internal/search/like_test.go +++ b/internal/search/like_test.go @@ -296,3 +296,24 @@ func TestAnyInt(t *testing.T) { assert.Equal(t, "", where) }) } + +func TestOrLike(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + where, values := OrLike("k.keyword", "") + + assert.Equal(t, "", where) + assert.Equal(t, []interface{}{}, values) + }) + t.Run("OneTerm", func(t *testing.T) { + where, values := OrLike("k.keyword", "bar") + + assert.Equal(t, "k.keyword LIKE ?", where) + assert.Equal(t, []interface{}{"bar"}, values) + }) + t.Run("TwoTerms", func(t *testing.T) { + where, values := OrLike("k.keyword", "foo*%|bar") + + assert.Equal(t, "k.keyword LIKE ? OR k.keyword LIKE ?", where) + assert.Equal(t, []interface{}{"foo%", "bar"}, values) + }) +} diff --git a/internal/search/photos.go b/internal/search/photos.go index 32744bf7c..21a3c8be0 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -137,19 +137,13 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { // Clip to reasonable size and normalize operators. f.Query = txt.NormalizeQuery(f.Query) - // Modify query if it contains subject names. - if f.Query != "" && f.Subject == "" { - if subj, names, remaining := SubjectUIDs(f.Query); len(subj) > 0 { - f.Subject = strings.Join(subj, txt.And) - log.Debugf("people: searching for %s", txt.Quote(txt.JoinNames(names, false))) - f.Query = remaining - } - } - // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { - f.Name = fs.StripKnownExt(f.Query) + "*" - f.Query = "" + if f.Name == "" { + name := strings.Trim(fs.StripKnownExt(f.Query), "%*") + f.Name = fmt.Sprintf("%s*|%s*", name, strings.ToUpper(name)) + f.Query = "" + } } else if len(terms) > 0 { switch { case terms["faces"]: @@ -230,6 +224,9 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { s = s.Where(fmt.Sprintf("photos.id IN (SELECT photo_id FROM files f JOIN %s m ON f.file_uid = m.file_uid AND m.marker_invalid = 0 WHERE face_id IN (?))", entity.Marker{}.TableName()), strings.Split(f, txt.Or)) } + } else if txt.New(f.Face) { + s = s.Where(fmt.Sprintf("photos.id IN (SELECT photo_id FROM files f JOIN %s m ON f.file_uid = m.file_uid AND m.marker_invalid = 0 AND m.marker_type = ? WHERE subj_uid IS NULL OR subj_uid = '')", + entity.Marker{}.TableName()), entity.MarkerFace) } else if txt.No(f.Face) { s = s.Where(fmt.Sprintf("photos.id IN (SELECT photo_id FROM files f JOIN %s m ON f.file_uid = m.file_uid AND m.marker_invalid = 0 AND m.marker_type = ? WHERE face_id IS NULL OR face_id = '')", entity.Marker{}.TableName()), entity.MarkerFace) @@ -368,40 +365,38 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) { if strings.HasSuffix(p, "/") { s = s.Where("photos.photo_path = ?", p[:len(p)-1]) - } else if strings.Contains(p, txt.Or) { - s = s.Where("photos.photo_path IN (?)", strings.Split(p, txt.Or)) } else { - s = s.Where("photos.photo_path LIKE ?", strings.ReplaceAll(p, "*", "%")) + where, values := OrLike("photos.photo_path", p) + s = s.Where(where, values...) } } - if strings.Contains(f.Name, txt.Or) { - s = s.Where("photos.photo_name IN (?)", strings.Split(f.Name, txt.Or)) - } else if f.Name != "" { - s = s.Where("photos.photo_name LIKE ?", strings.ReplaceAll(fs.StripKnownExt(f.Name), "*", "%")) + // Filter by main file name. + if f.Name != "" { + where, values := OrLike("photos.photo_name", f.Name) + s = s.Where(where, values...) } - if strings.Contains(f.Filename, txt.Or) { - s = s.Where("files.file_name IN (?)", strings.Split(f.Filename, txt.Or)) - } else if f.Filename != "" { - s = s.Where("files.file_name LIKE ?", strings.ReplaceAll(f.Filename, "*", "%")) + // Filter by actual file name. + if f.Filename != "" { + where, values := OrLike("files.file_name", f.Filename) + s = s.Where(where, values...) } - if strings.Contains(f.Original, txt.Or) { - s = s.Where("photos.original_name IN (?)", strings.Split(f.Original, txt.Or)) - } else if f.Original != "" { - s = s.Where("photos.original_name LIKE ?", strings.ReplaceAll(f.Original, "*", "%")) + // Filter by original file name. + if f.Original != "" { + where, values := OrLike("photos.original_name", f.Original) + s = s.Where(where, values...) } - if strings.Contains(f.Title, txt.Or) { - s = s.Where("photos.photo_title IN (?)", strings.Split(strings.ToLower(f.Title), txt.Or)) - } else if f.Title != "" { - s = s.Where("photos.photo_title LIKE ?", strings.ReplaceAll(strings.ToLower(f.Title), "*", "%")) + // Filter by photo title. + if f.Title != "" { + where, values := OrLike("photos.photo_title", f.Title) + s = s.Where(where, values...) } - if strings.Contains(f.Hash, txt.Or) { - s = s.Where("files.file_hash IN (?)", strings.Split(strings.ToLower(f.Hash), txt.Or)) - } else if f.Hash != "" { + // Filter by file hash. + if f.Hash != "" { s = s.Where("files.file_hash IN (?)", strings.Split(strings.ToLower(f.Hash), txt.Or)) } diff --git a/internal/search/photos_geo.go b/internal/search/photos_geo.go index a542fcc9e..5a025d11c 100644 --- a/internal/search/photos_geo.go +++ b/internal/search/photos_geo.go @@ -41,19 +41,13 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) { // Clip to reasonable size and normalize operators. f.Query = txt.NormalizeQuery(f.Query) - // Modify query if it contains subject names. - if f.Query != "" && f.Subject == "" { - if subj, names, remaining := SubjectUIDs(f.Query); len(subj) > 0 { - f.Subject = strings.Join(subj, txt.And) - log.Debugf("search: subject %s", txt.Quote(strings.Join(names, ", "))) - f.Query = remaining - } - } - // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { - f.Name = fs.StripKnownExt(f.Query) + "*" - f.Query = "" + if f.Name == "" { + name := strings.Trim(fs.StripKnownExt(f.Query), "%*") + f.Name = fmt.Sprintf("%s*|%s*", name, strings.ToUpper(name)) + f.Query = "" + } } else if len(terms) > 0 { switch { case terms["faces"]: @@ -226,17 +220,16 @@ func PhotosGeo(f form.PhotoSearchGeo) (results GeoResults, err error) { if strings.HasSuffix(p, "/") { s = s.Where("photos.photo_path = ?", p[:len(p)-1]) - } else if strings.Contains(p, txt.Or) { - s = s.Where("photos.photo_path IN (?)", strings.Split(p, txt.Or)) } else { - s = s.Where("photos.photo_path LIKE ?", strings.ReplaceAll(p, "*", "%")) + where, values := OrLike("photos.photo_path", p) + s = s.Where(where, values...) } } - if strings.Contains(f.Name, txt.Or) { - s = s.Where("photos.photo_name IN (?)", strings.Split(f.Name, txt.Or)) - } else if f.Name != "" { - s = s.Where("photos.photo_name LIKE ?", strings.ReplaceAll(fs.StripKnownExt(f.Name), "*", "%")) + // Filter by main file name. + if f.Name != "" { + where, values := OrLike("photos.photo_name", f.Name) + s = s.Where(where, values...) } // Filter by status. diff --git a/internal/search/photos_test.go b/internal/search/photos_test.go index f825a87fd..8308850e4 100644 --- a/internal/search/photos_test.go +++ b/internal/search/photos_test.go @@ -879,8 +879,7 @@ func TestPhotos(t *testing.T) { t.Run("Subject", func(t *testing.T) { var frm form.PhotoSearch - frm.Query = "John" - frm.Subject = "" + frm.Subject = "jqu0xs11qekk9jx8" frm.Count = 10 frm.Offset = 0 @@ -903,6 +902,21 @@ func TestPhotos(t *testing.T) { } } }) + t.Run("NewFaces", func(t *testing.T) { + var frm form.PhotoSearch + + frm.Face = "new" + frm.Count = 10 + frm.Offset = 0 + + photos, _, err := Photos(frm) + + if err != nil { + t.Fatal(err) + } + + assert.LessOrEqual(t, 1, len(photos)) + }) t.Run("query: videos", func(t *testing.T) { var frm form.PhotoSearch diff --git a/internal/workers/meta.go b/internal/workers/meta.go index 55f681484..eb3126d88 100644 --- a/internal/workers/meta.go +++ b/internal/workers/meta.go @@ -44,6 +44,15 @@ func (m *Meta) Start(delay time.Duration) (err error) { defer mutex.MetaWorker.Stop() + log.Debugf("metadata: running facial recognition") + + // Run faces worker. + if w := photoprism.NewFaces(m.conf); w.Disabled() { + log.Debugf("metadata: skipping facial recognition") + } else if err := w.Start(photoprism.FacesOptions{}); err != nil { + log.Warn(err) + } + log.Debugf("metadata: starting routine check") settings := m.conf.Settings() @@ -118,15 +127,6 @@ func (m *Meta) Start(delay time.Duration) (err error) { log.Warn(err) } - log.Debugf("metadata: running facial recognition") - - // Run faces worker. - if w := photoprism.NewFaces(m.conf); w.Disabled() { - log.Debugf("metadata: skipping facial recognition") - } else if err := w.Start(photoprism.FacesOptions{}); err != nil { - log.Warn(err) - } - log.Debugf("metadata: updating photo counts") // Update precalculated photo and file counts. diff --git a/pkg/txt/search.go b/pkg/txt/search.go new file mode 100644 index 000000000..052fd4501 --- /dev/null +++ b/pkg/txt/search.go @@ -0,0 +1,16 @@ +package txt + +// SearchTerms returns a bool map with all terms as key. +func SearchTerms(s string) map[string]bool { + result := make(map[string]bool) + + if s == "" { + return result + } + + for _, w := range UniqueKeywords(s) { + result[w] = true + } + + return result +} diff --git a/pkg/txt/search_test.go b/pkg/txt/search_test.go new file mode 100644 index 000000000..33cce6701 --- /dev/null +++ b/pkg/txt/search_test.go @@ -0,0 +1,19 @@ +package txt + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSearchTerms(t *testing.T) { + t.Run("Many", func(t *testing.T) { + result := SearchTerms("I'm a lazy-BRoWN fox! Yellow banana, apple; pan-pot b&w") + assert.Len(t, result, 6) + assert.Equal(t, map[string]bool{"apple": true, "banana": true, "fox": true, "lazy-brown": true, "pan-pot": true, "yellow": true}, result) + }) + t.Run("Empty", func(t *testing.T) { + result := SearchTerms("") + assert.Len(t, result, 0) + }) +} diff --git a/pkg/txt/strings.go b/pkg/txt/strings.go index 238b6dc8b..334d46e48 100644 --- a/pkg/txt/strings.go +++ b/pkg/txt/strings.go @@ -17,6 +17,10 @@ func Bool(s string) bool { // Yes returns true if a string represents "yes". func Yes(s string) bool { + if s == "" { + return false + } + s = strings.ToLower(strings.TrimSpace(s)) return strings.IndexAny(s, "ytjposiд") == 0 @@ -24,7 +28,22 @@ func Yes(s string) bool { // No returns true if a string represents "no". func No(s string) bool { + if s == "" { + return false + } + s = strings.ToLower(strings.TrimSpace(s)) return strings.IndexAny(s, "0nhufeн") == 0 } + +// New returns true if a string represents "new". +func New(s string) bool { + if s == "" { + return false + } + + s = strings.ToLower(strings.TrimSpace(s)) + + return s == "new" +} diff --git a/pkg/txt/strings_test.go b/pkg/txt/strings_test.go index 160fc65cc..c38ec501e 100644 --- a/pkg/txt/strings_test.go +++ b/pkg/txt/strings_test.go @@ -125,3 +125,21 @@ func TestNo(t *testing.T) { assert.Equal(t, false, No("")) }) } + +func TestNew(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + assert.Equal(t, false, New("")) + }) + t.Run("Uppercase", func(t *testing.T) { + assert.Equal(t, true, New("NEW")) + }) + t.Run("Lowercase", func(t *testing.T) { + assert.Equal(t, true, New("new")) + }) + t.Run("True", func(t *testing.T) { + assert.Equal(t, true, New("New")) + }) + t.Run("False", func(t *testing.T) { + assert.Equal(t, false, New("non")) + }) +} diff --git a/pkg/txt/words.go b/pkg/txt/words.go index 209f3c819..11988b047 100644 --- a/pkg/txt/words.go +++ b/pkg/txt/words.go @@ -202,24 +202,3 @@ func UniqueKeywords(s string) (results []string) { func SortCaseInsensitive(words []string) { sort.Slice(words, func(i, j int) bool { return strings.ToLower(words[i]) < strings.ToLower(words[j]) }) } - -// SearchTerms returns a bool map with all terms as key. -func SearchTerms(s string) map[string]bool { - result := make(map[string]bool) - - if s == "" { - return result - } - - for _, w := range KeywordsRegexp.FindAllString(s, -1) { - w = strings.Trim(w, "- '") - - if w == "" || len(w) < 3 && IsLatin(w) { - continue - } - - result[w] = true - } - - return result -} diff --git a/pkg/txt/words_test.go b/pkg/txt/words_test.go index 6374032b9..1a3539381 100644 --- a/pkg/txt/words_test.go +++ b/pkg/txt/words_test.go @@ -239,15 +239,3 @@ func TestRemoveFromWords(t *testing.T) { assert.Equal(t, []string{"apple", "brown", "jpg", "lazy"}, result) }) } - -func TestSearchTerms(t *testing.T) { - t.Run("Many", func(t *testing.T) { - result := SearchTerms("I'm a lazy-BRoWN fox! Yellow banana, apple; pan-pot b&w") - assert.Len(t, result, 7) - assert.Equal(t, map[string]bool{"I'm": true, "Yellow": true, "apple": true, "banana": true, "fox": true, "lazy-BRoWN": true, "pan-pot": true}, result) - }) - t.Run("Empty", func(t *testing.T) { - result := SearchTerms("") - assert.Len(t, result, 0) - }) -}