From 2dedbb83dc676d17bedc85d01bc161bc030acb82 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Wed, 15 Dec 2021 12:24:05 +0100 Subject: [PATCH] Sanitize: Add name, query, state, and username filters #1814 --- go.mod | 6 +- go.sum | 11 ++-- internal/entity/label.go | 5 +- internal/entity/marker.go | 4 +- internal/entity/subject.go | 7 ++- internal/entity/user.go | 10 +-- internal/form/serialize.go | 6 +- internal/form/user.go | 4 +- internal/hub/places/location.go | 2 +- internal/maps/location.go | 3 +- internal/query/moments.go | 5 +- internal/search/albums.go | 4 +- internal/search/geojson.go | 4 +- internal/search/like.go | 4 +- internal/search/like_test.go | 8 ++- internal/search/photos.go | 4 +- internal/search/subjects.go | 6 +- pkg/sanitize/const.go | 15 +++++ pkg/sanitize/hex.go | 4 +- pkg/sanitize/id.go | 7 ++- pkg/sanitize/name.go | 32 ++++++++++ pkg/sanitize/name_test.go | 31 +++++++++ pkg/sanitize/query.go | 26 ++++++++ pkg/sanitize/query_test.go | 14 +++++ pkg/sanitize/state.go | 50 +++++++++++++++ pkg/sanitize/state_test.go | 65 +++++++++++++++++++ pkg/sanitize/token.go | 4 +- pkg/sanitize/username.go | 12 ++++ pkg/sanitize/username_test.go | 19 ++++++ pkg/txt/normalize.go | 74 ---------------------- pkg/txt/normalize_test.go | 107 -------------------------------- 31 files changed, 330 insertions(+), 223 deletions(-) create mode 100644 pkg/sanitize/const.go create mode 100644 pkg/sanitize/name.go create mode 100644 pkg/sanitize/name_test.go create mode 100644 pkg/sanitize/query.go create mode 100644 pkg/sanitize/query_test.go create mode 100644 pkg/sanitize/state.go create mode 100644 pkg/sanitize/state_test.go create mode 100644 pkg/sanitize/username.go create mode 100644 pkg/sanitize/username_test.go delete mode 100644 pkg/txt/normalize.go delete mode 100644 pkg/txt/normalize_test.go diff --git a/go.mod b/go.mod index bdc35d415..f95c1a1e7 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/esimov/pigo v1.4.5 github.com/gin-contrib/gzip v0.0.5 github.com/gin-gonic/gin v1.7.7 - github.com/go-errors/errors v1.4.0 // indirect + github.com/go-errors/errors v1.4.1 // indirect github.com/go-playground/validator/v10 v10.9.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 github.com/golang/protobuf v1.5.2 // indirect @@ -56,7 +56,7 @@ require ( go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect - golang.org/x/net v0.0.0-20211209124913-491a49abca63 + golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827 // indirect golang.org/x/text v0.3.7 // indirect gonum.org/v1/gonum v0.9.3 @@ -74,7 +74,7 @@ require ( github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect - github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect + github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mandykoh/go-parallel v0.1.0 // indirect diff --git a/go.sum b/go.sum index 8da9fd619..614239455 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-errors/errors v1.4.0 h1:2OA7MFw38+e9na72T1xgkomPb6GzZzzxvJ5U630FoRM= -github.com/go-errors/errors v1.4.0/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg= +github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -116,8 +116,9 @@ github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPg github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo= github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 h1:PSPmmucxGiFBtbQcttHTUc4LQ3P09AW+ldO2qspyKdY= +github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -376,8 +377,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 h1:kmreh1vGI63l2FxOAYS3Yv6ATsi7lSTuwNSVbGfJV9I= +golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/internal/entity/label.go b/internal/entity/label.go index e558845ec..b845d4782 100644 --- a/internal/entity/label.go +++ b/internal/entity/label.go @@ -5,9 +5,12 @@ import ( "time" "github.com/jinzhu/gorm" + "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/pkg/rnd" + "github.com/photoprism/photoprism/pkg/sanitize" "github.com/photoprism/photoprism/pkg/txt" ) @@ -160,7 +163,7 @@ func (m *Label) AfterCreate(scope *gorm.Scope) error { // SetName changes the label name. func (m *Label) SetName(name string) { - name = txt.NormalizeName(name) + name = sanitize.Name(name) if name == "" { return diff --git a/internal/entity/marker.go b/internal/entity/marker.go index bdec3d2fa..9ba0ee00d 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -14,9 +14,9 @@ import ( "github.com/photoprism/photoprism/internal/crop" "github.com/photoprism/photoprism/internal/face" "github.com/photoprism/photoprism/internal/form" + "github.com/photoprism/photoprism/pkg/rnd" "github.com/photoprism/photoprism/pkg/sanitize" - "github.com/photoprism/photoprism/pkg/txt" ) const ( @@ -158,7 +158,7 @@ func (m *Marker) SetName(name, src string) (changed bool, err error) { return false, nil } - name = txt.NormalizeName(name) + name = sanitize.Name(name) if name == "" { return false, nil diff --git a/internal/entity/subject.go b/internal/entity/subject.go index 84a43bb5c..13370533d 100644 --- a/internal/entity/subject.go +++ b/internal/entity/subject.go @@ -11,6 +11,7 @@ import ( "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/form" + "github.com/photoprism/photoprism/pkg/rnd" "github.com/photoprism/photoprism/pkg/sanitize" "github.com/photoprism/photoprism/pkg/txt" @@ -218,7 +219,7 @@ func FindSubject(s string) *Subject { // FindSubjectByName find an existing subject by name. func FindSubjectByName(name string) *Subject { - name = txt.NormalizeName(name) + name = sanitize.Name(name) if name == "" { return nil @@ -253,7 +254,7 @@ func (m *Subject) Person() *Person { // SetName changes the subject's name. func (m *Subject) SetName(name string) error { - name = txt.NormalizeName(name) + name = sanitize.Name(name) if name == m.SubjName { // Nothing to do. @@ -280,7 +281,7 @@ func (m *Subject) SaveForm(f form.Subject) (changed bool, err error) { } // Change name? - if name := txt.NormalizeName(f.SubjName); name != "" && name != m.SubjName { + if name := sanitize.Name(f.SubjName); name != "" && name != m.SubjName { existing, err := m.UpdateName(name) if existing.SubjUID != m.SubjUID || err != nil { diff --git a/internal/entity/user.go b/internal/entity/user.go index 82c570d4f..847393d7e 100644 --- a/internal/entity/user.go +++ b/internal/entity/user.go @@ -6,13 +6,13 @@ import ( "net/mail" "time" + "github.com/jinzhu/gorm" + + "github.com/photoprism/photoprism/internal/acl" "github.com/photoprism/photoprism/internal/form" - "github.com/jinzhu/gorm" - "github.com/photoprism/photoprism/internal/acl" "github.com/photoprism/photoprism/pkg/rnd" "github.com/photoprism/photoprism/pkg/sanitize" - "github.com/photoprism/photoprism/pkg/txt" ) type Users []User @@ -166,7 +166,7 @@ func FirstOrCreateUser(m *User) *User { // FindUserByName returns an existing user or nil if not found. func FindUserByName(userName string) *User { - userName = txt.NormalizeUsername(userName) + userName = sanitize.Username(userName) if userName == "" { return nil @@ -227,7 +227,7 @@ func (m *User) String() string { // Username returns the normalized username. func (m *User) Username() string { - return txt.NormalizeUsername(m.UserName) + return sanitize.Username(m.UserName) } // Registered tests if the user is registered e.g. has a username. diff --git a/internal/form/serialize.go b/internal/form/serialize.go index 37ca9e03f..102aff8a4 100644 --- a/internal/form/serialize.go +++ b/internal/form/serialize.go @@ -8,6 +8,8 @@ import ( "time" "unicode" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/araddon/dateparse" "github.com/photoprism/photoprism/pkg/txt" ) @@ -68,7 +70,7 @@ func Serialize(f interface{}, all bool) string { q = append(q, fmt.Sprintf("%s:%t", fieldName, fieldValue.Bool())) } default: - log.Warnf("can't serialize value of type %s from form field %s", t, fieldName) + log.Warnf("form: can't serialize value of type %s in %s", t, sanitize.Token(fieldName)) } } } @@ -155,7 +157,7 @@ func Unserialize(f SearchForm, q string) (result error) { } if result != nil { - log.Errorf("error while parsing form values: %s", result) + log.Warnf("form: failed parsing values") } return result diff --git a/internal/form/user.go b/internal/form/user.go index ddcf5825b..91e9610cb 100644 --- a/internal/form/user.go +++ b/internal/form/user.go @@ -1,6 +1,6 @@ package form -import "github.com/photoprism/photoprism/pkg/txt" +import "github.com/photoprism/photoprism/pkg/sanitize" // UserCreate represents a User with a new password. type UserCreate struct { @@ -12,5 +12,5 @@ type UserCreate struct { // Username returns the normalized username in lowercase and without whitespace padding. func (f UserCreate) Username() string { - return txt.NormalizeUsername(f.UserName) + return sanitize.Username(f.UserName) } diff --git a/internal/hub/places/location.go b/internal/hub/places/location.go index 36716780a..9e383c166 100644 --- a/internal/hub/places/location.go +++ b/internal/hub/places/location.go @@ -193,7 +193,7 @@ func (l Location) CountryCode() (result string) { // State returns the location address state name. func (l Location) State() (result string) { - return txt.NormalizeState(l.Place.LocState, l.CountryCode()) + return sanitize.State(l.Place.LocState, l.CountryCode()) } // Latitude returns the location position latitude. diff --git a/internal/maps/location.go b/internal/maps/location.go index 44850d385..87bac3912 100644 --- a/internal/maps/location.go +++ b/internal/maps/location.go @@ -6,6 +6,7 @@ import ( "github.com/photoprism/photoprism/internal/hub/places" "github.com/photoprism/photoprism/pkg/s2" + "github.com/photoprism/photoprism/pkg/sanitize" "github.com/photoprism/photoprism/pkg/txt" ) @@ -122,7 +123,7 @@ func (l Location) CountryCode() string { } func (l Location) State() string { - return txt.Clip(txt.NormalizeState(l.LocState, l.CountryCode()), 100) + return txt.Clip(sanitize.State(l.LocState, l.CountryCode()), 100) } func (l Location) CountryName() string { diff --git a/internal/query/moments.go b/internal/query/moments.go index 849dd683c..f2bb2c983 100644 --- a/internal/query/moments.go +++ b/internal/query/moments.go @@ -9,6 +9,7 @@ import ( "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/maps" + "github.com/photoprism/photoprism/pkg/sanitize" "github.com/photoprism/photoprism/pkg/txt" ) @@ -106,7 +107,7 @@ func (m Moment) CountryName() string { // Slug returns an identifier string for a moment. func (m Moment) Slug() (s string) { - state := txt.NormalizeState(m.State, m.Country) + state := sanitize.State(m.State, m.Country) if state == "" { return m.TitleSlug() @@ -132,7 +133,7 @@ func (m Moment) TitleSlug() string { // Title returns an english title for the moment. func (m Moment) Title() string { - state := txt.NormalizeState(m.State, m.Country) + state := sanitize.State(m.State, m.Country) if m.Year == 0 && m.Month == 0 { if m.Label != "" { diff --git a/internal/search/albums.go b/internal/search/albums.go index 4383aea53..a169021c2 100644 --- a/internal/search/albums.go +++ b/internal/search/albums.go @@ -3,6 +3,8 @@ package search import ( "strings" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/pkg/txt" @@ -15,7 +17,7 @@ func Albums(f form.SearchAlbums) (results AlbumResults, err error) { } // Clip and normalize search query. - f.Query = txt.NormalizeQuery(f.Query) + f.Query = sanitize.Query(f.Query) // Base query. s := UnscopedDb().Table("albums"). diff --git a/internal/search/geojson.go b/internal/search/geojson.go index cf6308088..7e903c022 100644 --- a/internal/search/geojson.go +++ b/internal/search/geojson.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/dustin/go-humanize/english" "github.com/jinzhu/gorm" @@ -57,7 +59,7 @@ func Geo(f form.SearchGeo) (results GeoResults, err error) { Where("photos.photo_lat <> 0") // Clip and normalize search query. - f.Query = txt.NormalizeQuery(f.Query) + f.Query = sanitize.Query(f.Query) // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { diff --git a/internal/search/like.go b/internal/search/like.go index d999dccc2..9996e2d6a 100644 --- a/internal/search/like.go +++ b/internal/search/like.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/gosimple/slug" "github.com/photoprism/photoprism/pkg/txt" @@ -16,7 +18,7 @@ func LikeAny(col, s string, keywords, exact bool) (wheres []string) { return wheres } - s = txt.StripOr(txt.NormalizeQuery(s)) + s = txt.StripOr(sanitize.Query(s)) var wildcardThreshold int diff --git a/internal/search/like_test.go b/internal/search/like_test.go index 8550da397..3c93a983a 100644 --- a/internal/search/like_test.go +++ b/internal/search/like_test.go @@ -3,6 +3,8 @@ package search import ( "testing" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/pkg/txt" @@ -205,7 +207,7 @@ func TestLikeAllNames(t *testing.T) { } }) t.Run("Plus", func(t *testing.T) { - if w := LikeAllNames(Cols{"name"}, txt.NormalizeQuery("Paul + Paula")); len(w) == 2 { + if w := LikeAllNames(Cols{"name"}, sanitize.Query("Paul + Paula")); len(w) == 2 { assert.Equal(t, "name LIKE '%paul%'", w[0]) assert.Equal(t, "name LIKE '%paula%'", w[1]) } else { @@ -213,7 +215,7 @@ func TestLikeAllNames(t *testing.T) { } }) t.Run("And", func(t *testing.T) { - if w := LikeAllNames(Cols{"name"}, txt.NormalizeQuery("P and Paula")); len(w) == 2 { + if w := LikeAllNames(Cols{"name"}, sanitize.Query("P and Paula")); len(w) == 2 { assert.Equal(t, "name LIKE '%p%'", w[0]) assert.Equal(t, "name LIKE '%paula%'", w[1]) } else { @@ -221,7 +223,7 @@ func TestLikeAllNames(t *testing.T) { } }) t.Run("Or", func(t *testing.T) { - if w := LikeAllNames(Cols{"name"}, txt.NormalizeQuery("Paul or Paula")); len(w) == 1 { + if w := LikeAllNames(Cols{"name"}, sanitize.Query("Paul or Paula")); len(w) == 1 { assert.Equal(t, "name LIKE '%paul%' OR name LIKE '%paula%'", w[0]) } else { t.Fatalf("unexpected result: %#v", w) diff --git a/internal/search/photos.go b/internal/search/photos.go index a11563b45..6d1254fbd 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/dustin/go-humanize/english" "github.com/jinzhu/gorm" @@ -137,7 +139,7 @@ func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { } // Clip and normalize search query. - f.Query = txt.NormalizeQuery(f.Query) + f.Query = sanitize.Query(f.Query) // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { diff --git a/internal/search/subjects.go b/internal/search/subjects.go index b3cd30012..be2d9badc 100644 --- a/internal/search/subjects.go +++ b/internal/search/subjects.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" + "github.com/photoprism/photoprism/pkg/sanitize" + "github.com/photoprism/photoprism/pkg/txt" "github.com/jinzhu/gorm" @@ -55,7 +57,7 @@ func Subjects(f form.SearchSubjects) (results SubjectResults, err error) { } // Clip to reasonable size and normalize operators. - f.Query = txt.NormalizeQuery(f.Query) + f.Query = sanitize.Query(f.Query) if f.Query != "" { for _, where := range LikeAllNames(Cols{"subj_name", "subj_alias"}, f.Query) { @@ -163,5 +165,5 @@ func SubjectUIDs(s string) (result []string, names []string, remaining string) { result = append(result, strings.Join(subj, txt.Or)) } - return result, names, txt.NormalizeQuery(remaining) + return result, names, sanitize.Query(remaining) } diff --git a/pkg/sanitize/const.go b/pkg/sanitize/const.go new file mode 100644 index 000000000..090864cf2 --- /dev/null +++ b/pkg/sanitize/const.go @@ -0,0 +1,15 @@ +package sanitize + +const ( + EnOr = "or" + EnAnd = "and" + EnWith = "with" + EnIn = "in" + EnAt = "at" + Empty = "" + Space = " " + Or = "|" + And = "&" + Plus = "+" + SpacedPlus = Space + Plus + Space +) diff --git a/pkg/sanitize/hex.go b/pkg/sanitize/hex.go index 8015888cb..9b63932f6 100644 --- a/pkg/sanitize/hex.go +++ b/pkg/sanitize/hex.go @@ -6,8 +6,8 @@ import ( // Hex removes invalid character from a hex string and makes it lowercase. func Hex(s string) string { - if s == "" { - return s + if s == "" || len(s) > 1024 || strings.Contains(s, "${") { + return "" } s = strings.ToLower(s) diff --git a/pkg/sanitize/id.go b/pkg/sanitize/id.go index 03c90798c..a168f0711 100644 --- a/pkg/sanitize/id.go +++ b/pkg/sanitize/id.go @@ -7,7 +7,7 @@ import ( // IdString removes invalid character from an id string. func IdString(s string) string { - if s == "" || len(s) > 256 { + if s == "" || len(s) > 256 || strings.Contains(s, "${") { return "" } @@ -27,7 +27,10 @@ func IdString(s string) string { // IdUint converts the string converted to an unsigned integer and 0 if the string is invalid. func IdUint(s string) uint { - if s == "" || len(s) > 64 { + // Largest possible values: + // UInt64: 18446744073709551615 (20 digits) + // UInt32: 4294967295 (10 digits) + if s == "" || len(s) > 10 || strings.Contains(s, "${") { return 0 } diff --git a/pkg/sanitize/name.go b/pkg/sanitize/name.go new file mode 100644 index 000000000..466edd9a7 --- /dev/null +++ b/pkg/sanitize/name.go @@ -0,0 +1,32 @@ +package sanitize + +import ( + "strings" + + "github.com/photoprism/photoprism/pkg/txt" +) + +// Name sanitizes and capitalizes names. +func Name(name string) string { + if name == "" { + return "" + } + + // Remove double quotes and other special characters. + name = strings.Map(func(r rune) rune { + switch r { + case '"', '`', '~', '\\', '/', '*', '%', '&', '|', '+', '=', '$', '@', '!', '?', ':', ';', '<', '>', '{', '}': + return -1 + } + return r + }, name) + + name = strings.TrimSpace(name) + + if name == "" { + return "" + } + + // Shorten and capitalize. + return txt.Clip(txt.Title(name), txt.ClipDefault) +} diff --git a/pkg/sanitize/name_test.go b/pkg/sanitize/name_test.go new file mode 100644 index 000000000..ca1ccfee8 --- /dev/null +++ b/pkg/sanitize/name_test.go @@ -0,0 +1,31 @@ +package sanitize + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestName(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + assert.Equal(t, "", Name("")) + }) + t.Run("BillGates", func(t *testing.T) { + assert.Equal(t, "William Henry Gates III", Name("William Henry Gates III")) + }) + t.Run("Quotes", func(t *testing.T) { + assert.Equal(t, "William HenRy Gates'", Name("william \"HenRy\" gates' ")) + }) + t.Run("Slash", func(t *testing.T) { + assert.Equal(t, "William McCorn Gates'", Name("william\\ \"McCorn\" / gates' ")) + }) + t.Run("SpecialCharacters", func(t *testing.T) { + assert.Equal(t, + "'', '', '', '', '', '', '', '', '', '', '', '', Foo '', '', '', '', '', '', '', McBar '', ''", + Name("'\"', '`', '~', '\\\\', '/', '*', '%', '&', '|', '+', '=', '$', Foo '@', '!', '?', ':', ';', '<', '>', McBar '{', '}'"), + ) + }) + t.Run("Chinese", func(t *testing.T) { + assert.Equal(t, "陈 赵", Name(" 陈 赵")) + }) +} diff --git a/pkg/sanitize/query.go b/pkg/sanitize/query.go new file mode 100644 index 000000000..e46b409c1 --- /dev/null +++ b/pkg/sanitize/query.go @@ -0,0 +1,26 @@ +package sanitize + +import "strings" + +// spaced returns the string padded with a space left and right. +func spaced(s string) string { + return Space + s + Space +} + +// Query replaces search operator with default symbols. +func Query(s string) string { + if s == "" || len(s) > 1024 || strings.Contains(s, "${") { + return Empty + } + + s = strings.ToLower(s) + s = strings.ReplaceAll(s, spaced(EnOr), Or) + s = strings.ReplaceAll(s, spaced(EnAnd), And) + s = strings.ReplaceAll(s, spaced(EnWith), And) + s = strings.ReplaceAll(s, spaced(EnIn), And) + s = strings.ReplaceAll(s, spaced(EnAt), And) + s = strings.ReplaceAll(s, SpacedPlus, And) + s = strings.ReplaceAll(s, "%", "*") + + return strings.Trim(s, "+&|_-=!@$%^(){}\\<>,.;: ") +} diff --git a/pkg/sanitize/query_test.go b/pkg/sanitize/query_test.go new file mode 100644 index 000000000..bdb71b00d --- /dev/null +++ b/pkg/sanitize/query_test.go @@ -0,0 +1,14 @@ +package sanitize + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestQuery(t *testing.T) { + t.Run("Replace", func(t *testing.T) { + q := Query("table spoon & usa | img% json OR BILL!") + assert.Equal(t, "table spoon & usa | img* json|bill", q) + }) +} diff --git a/pkg/sanitize/state.go b/pkg/sanitize/state.go new file mode 100644 index 000000000..62f002324 --- /dev/null +++ b/pkg/sanitize/state.go @@ -0,0 +1,50 @@ +package sanitize + +import ( + "strings" + "unicode" + + "github.com/photoprism/photoprism/pkg/txt" +) + +// State returns the full, normalized state name. +func State(stateName, countryCode string) string { + // Remove whitespace from name. + stateName = strings.TrimSpace(stateName) + + // Empty? + if stateName == "" || stateName == txt.UnknownStateCode { + // State doesn't have a name. + return "" + } + + // Remove non-printable and other potentially problematic characters. + stateName = strings.Map(func(r rune) rune { + if !unicode.IsPrint(r) { + return -1 + } + + switch r { + case '~', '\\', ':', '|', '"', '?', '*', '<', '>', '{', '}': + return -1 + default: + return r + } + }, stateName) + + // Normalize country code. + countryCode = strings.ToLower(strings.TrimSpace(countryCode)) + + // Is the name an abbreviation that should be normalized? + if states, found := txt.StatesByCountry[countryCode]; !found { + // Unknown country. + } else if normalized, found := states[stateName]; !found { + // Unknown abbreviation. + } else if normalized != "" { + // Yes, use normalized name. + stateName = normalized + } + + // Return normalized state name. + return stateName +} diff --git a/pkg/sanitize/state_test.go b/pkg/sanitize/state_test.go new file mode 100644 index 000000000..15f63c2ef --- /dev/null +++ b/pkg/sanitize/state_test.go @@ -0,0 +1,65 @@ +package sanitize + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestState(t *testing.T) { + t.Run("Berlin", func(t *testing.T) { + result := State("Berlin", "de") + assert.Equal(t, "Berlin", result) + }) + + t.Run("WA", func(t *testing.T) { + result := State("WA", "us") + assert.Equal(t, "Washington", result) + }) + + t.Run("QCUnknownCountry", func(t *testing.T) { + result := State("QC", "") + assert.Equal(t, "QC", result) + }) + + t.Run("QCCanada", func(t *testing.T) { + result := State("QC", "ca") + assert.Equal(t, "Quebec", result) + }) + + t.Run("QCUnitedStates", func(t *testing.T) { + result := State("QC", "us") + assert.Equal(t, "QC", result) + }) + + t.Run("Wa", func(t *testing.T) { + result := State("Wa", "us") + assert.Equal(t, "Wa", result) + }) + + t.Run("Washington", func(t *testing.T) { + result := State("Washington", "us") + assert.Equal(t, "Washington", result) + }) + + t.Run("Never mind Nirvana", func(t *testing.T) { + result := State("Never mind Nirvana.", "us") + assert.Equal(t, "Never mind Nirvana.", result) + }) + + t.Run("Empty", func(t *testing.T) { + result := State("", "us") + assert.Equal(t, "", result) + }) + + t.Run("Unknown", func(t *testing.T) { + result := State("zz", "us") + assert.Equal(t, "", result) + }) + + t.Run("Space", func(t *testing.T) { + result := State(" ", "us") + assert.Equal(t, "", result) + }) + +} diff --git a/pkg/sanitize/token.go b/pkg/sanitize/token.go index 799dfc59b..206d37527 100644 --- a/pkg/sanitize/token.go +++ b/pkg/sanitize/token.go @@ -6,8 +6,8 @@ import ( // Token removes invalid character from a token string. func Token(s string) string { - if s == "" { - return s + if s == "" || len(s) > 200 || strings.Contains(s, "${") { + return "" } // Remove all invalid characters. diff --git a/pkg/sanitize/username.go b/pkg/sanitize/username.go new file mode 100644 index 000000000..2e12dfaf7 --- /dev/null +++ b/pkg/sanitize/username.go @@ -0,0 +1,12 @@ +package sanitize + +import ( + "strings" + + "github.com/photoprism/photoprism/pkg/txt" +) + +// Username returns the normalized username (lowercase, whitespace trimmed). +func Username(s string) string { + return strings.ToLower(txt.Clip(s, txt.ClipUsername)) +} diff --git a/pkg/sanitize/username_test.go b/pkg/sanitize/username_test.go new file mode 100644 index 000000000..e02b33b17 --- /dev/null +++ b/pkg/sanitize/username_test.go @@ -0,0 +1,19 @@ +package sanitize + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUsername(t *testing.T) { + t.Run("Admin ", func(t *testing.T) { + assert.Equal(t, "admin", Username("Admin ")) + }) + t.Run(" Admin ", func(t *testing.T) { + assert.Equal(t, "admin", Username(" Admin ")) + }) + t.Run(" admin ", func(t *testing.T) { + assert.Equal(t, "admin", Username(" admin ")) + }) +} diff --git a/pkg/txt/normalize.go b/pkg/txt/normalize.go deleted file mode 100644 index 83a90492f..000000000 --- a/pkg/txt/normalize.go +++ /dev/null @@ -1,74 +0,0 @@ -package txt - -import "strings" - -// NormalizeName sanitizes and capitalizes names. -func NormalizeName(name string) string { - if name == "" { - return "" - } - - // Remove double quotes and other special characters. - name = strings.Map(func(r rune) rune { - switch r { - case '"', '`', '~', '\\', '/', '*', '%', '&', '|', '+', '=', '$', '@', '!', '?', ':', ';', '<', '>', '{', '}': - return -1 - } - return r - }, name) - - name = strings.TrimSpace(name) - - if name == "" { - return "" - } - - // Shorten and capitalize. - return Clip(Title(name), ClipDefault) -} - -// NormalizeState returns the full, normalized state name. -func NormalizeState(stateName, countryCode string) string { - // Remove whitespace from name. - stateName = strings.TrimSpace(stateName) - - // Empty? - if stateName == "" || stateName == UnknownStateCode { - // State doesn't have a name. - return "" - } - - // Normalize country code. - countryCode = strings.ToLower(strings.TrimSpace(countryCode)) - - // Is the name an abbreviation that should be normalized? - if states, found := StatesByCountry[countryCode]; !found { - // Unknown country. - } else if normalized, found := states[stateName]; !found { - // Unknown abbreviation. - } else if normalized != "" { - // Yes, use normalized name. - stateName = normalized - } - - // Return normalized state name. - return stateName -} - -// NormalizeQuery replaces search operator with default symbols. -func NormalizeQuery(s string) string { - s = strings.ToLower(Clip(s, ClipQuery)) - s = strings.ReplaceAll(s, Spaced(EnOr), Or) - s = strings.ReplaceAll(s, Spaced(EnAnd), And) - s = strings.ReplaceAll(s, Spaced(EnWith), And) - s = strings.ReplaceAll(s, Spaced(EnIn), And) - s = strings.ReplaceAll(s, Spaced(EnAt), And) - s = strings.ReplaceAll(s, SpacedPlus, And) - s = strings.ReplaceAll(s, "%", "*") - return strings.Trim(s, "+&|_-=!@$%^(){}\\<>,.;: ") -} - -// NormalizeUsername returns the normalized username (lowercase, whitespace trimmed). -func NormalizeUsername(s string) string { - return strings.ToLower(Clip(s, ClipUsername)) -} diff --git a/pkg/txt/normalize_test.go b/pkg/txt/normalize_test.go deleted file mode 100644 index 91e88daa8..000000000 --- a/pkg/txt/normalize_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package txt - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNormalizeName(t *testing.T) { - t.Run("Empty", func(t *testing.T) { - assert.Equal(t, "", NormalizeName("")) - }) - t.Run("BillGates", func(t *testing.T) { - assert.Equal(t, "William Henry Gates III", NormalizeName("William Henry Gates III")) - }) - t.Run("Quotes", func(t *testing.T) { - assert.Equal(t, "William HenRy Gates'", NormalizeName("william \"HenRy\" gates' ")) - }) - t.Run("Slash", func(t *testing.T) { - assert.Equal(t, "William McCorn Gates'", NormalizeName("william\\ \"McCorn\" / gates' ")) - }) - t.Run("SpecialCharacters", func(t *testing.T) { - assert.Equal(t, - "'', '', '', '', '', '', '', '', '', '', '', '', Foo '', '', '', '', '', '', '', McBar '', ''", - NormalizeName("'\"', '`', '~', '\\\\', '/', '*', '%', '&', '|', '+', '=', '$', Foo '@', '!', '?', ':', ';', '<', '>', McBar '{', '}'"), - ) - }) - t.Run("Chinese", func(t *testing.T) { - assert.Equal(t, "陈 赵", NormalizeName(" 陈 赵")) - }) -} - -func TestNormalizeState(t *testing.T) { - t.Run("Berlin", func(t *testing.T) { - result := NormalizeState("Berlin", "de") - assert.Equal(t, "Berlin", result) - }) - - t.Run("WA", func(t *testing.T) { - result := NormalizeState("WA", "us") - assert.Equal(t, "Washington", result) - }) - - t.Run("QCUnknownCountry", func(t *testing.T) { - result := NormalizeState("QC", "") - assert.Equal(t, "QC", result) - }) - - t.Run("QCCanada", func(t *testing.T) { - result := NormalizeState("QC", "ca") - assert.Equal(t, "Quebec", result) - }) - - t.Run("QCUnitedStates", func(t *testing.T) { - result := NormalizeState("QC", "us") - assert.Equal(t, "QC", result) - }) - - t.Run("Wa", func(t *testing.T) { - result := NormalizeState("Wa", "us") - assert.Equal(t, "Wa", result) - }) - - t.Run("Washington", func(t *testing.T) { - result := NormalizeState("Washington", "us") - assert.Equal(t, "Washington", result) - }) - - t.Run("Never mind Nirvana", func(t *testing.T) { - result := NormalizeState("Never mind Nirvana.", "us") - assert.Equal(t, "Never mind Nirvana.", result) - }) - - t.Run("Empty", func(t *testing.T) { - result := NormalizeState("", "us") - assert.Equal(t, "", result) - }) - - t.Run("Unknown", func(t *testing.T) { - result := NormalizeState("zz", "us") - assert.Equal(t, "", result) - }) - - t.Run("Space", func(t *testing.T) { - result := NormalizeState(" ", "us") - assert.Equal(t, "", result) - }) - -} -func TestNormalizeQuery(t *testing.T) { - t.Run("Replace", func(t *testing.T) { - q := NormalizeQuery("table spoon & usa | img% json OR BILL!") - assert.Equal(t, "table spoon & usa | img* json|bill", q) - }) -} - -func TestNormalizeUsername(t *testing.T) { - t.Run("Admin ", func(t *testing.T) { - assert.Equal(t, "admin", NormalizeUsername("Admin ")) - }) - t.Run(" Admin ", func(t *testing.T) { - assert.Equal(t, "admin", NormalizeUsername(" Admin ")) - }) - t.Run(" admin ", func(t *testing.T) { - assert.Equal(t, "admin", NormalizeUsername(" admin ")) - }) -}