diff --git a/go.mod b/go.mod index f95c1a1e7..cc187c83e 100644 --- a/go.mod +++ b/go.mod @@ -54,9 +54,9 @@ require ( github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 github.com/urfave/cli v1.22.5 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect - golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827 // indirect golang.org/x/text v0.3.7 // indirect gonum.org/v1/gonum v0.9.3 diff --git a/go.sum b/go.sum index 614239455..6d2409df4 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4= -golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -377,8 +377,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 h1:kmreh1vGI63l2FxOAYS3Yv6ATsi7lSTuwNSVbGfJV9I= -golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/internal/api/search_photos.go b/internal/api/search_photos.go index 13fdaaeb2..e007b80b6 100644 --- a/internal/api/search_photos.go +++ b/internal/api/search_photos.go @@ -51,6 +51,7 @@ func SearchPhotos(router *gin.RouterGroup) { return } + f.UID = "" f.Public = true f.Private = false f.Hidden = false diff --git a/internal/form/serialize.go b/internal/form/serialize.go index 3c3c4f408..0ed3c26e4 100644 --- a/internal/form/serialize.go +++ b/internal/form/serialize.go @@ -126,7 +126,7 @@ func Unserialize(f SearchForm, q string) (result error) { field.SetUint(uint64(intValue)) } case string: - field.SetString(stringValue) + field.SetString(sanitize.Query(stringValue)) case bool: field.SetBool(txt.Bool(stringValue)) default: @@ -155,11 +155,11 @@ func Unserialize(f SearchForm, q string) (result error) { } if len(queryStrings) > 0 { - f.SetQuery(strings.Join(queryStrings, " ")) + f.SetQuery(sanitize.Query(strings.Join(queryStrings, " "))) } if result != nil { - log.Warnf("form: failed parsing values") + log.Warnf("form: failed parsing search query") } return result diff --git a/internal/search/albums.go b/internal/search/albums.go index b33c9ad10..bec9feb0d 100644 --- a/internal/search/albums.go +++ b/internal/search/albums.go @@ -3,8 +3,6 @@ package search import ( "strings" - "github.com/photoprism/photoprism/pkg/sanitize" - "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/pkg/txt" @@ -16,9 +14,6 @@ func Albums(f form.SearchAlbums) (results AlbumResults, err error) { return results, err } - // Clip and normalize search query. - f.Query = sanitize.Query(f.Query) - // Base query. s := UnscopedDb().Table("albums"). Select("albums.*, cp.photo_count, cl.link_count, CASE WHEN albums.album_year = 0 THEN 0 ELSE 1 END AS has_year"). diff --git a/internal/search/geojson.go b/internal/search/geojson.go index 7e903c022..1420a46b6 100644 --- a/internal/search/geojson.go +++ b/internal/search/geojson.go @@ -6,8 +6,6 @@ import ( "strings" "time" - "github.com/photoprism/photoprism/pkg/sanitize" - "github.com/dustin/go-humanize/english" "github.com/jinzhu/gorm" @@ -58,9 +56,6 @@ func Geo(f form.SearchGeo) (results GeoResults, err error) { Where("photos.deleted_at IS NULL"). Where("photos.photo_lat <> 0") - // Clip and normalize search query. - f.Query = sanitize.Query(f.Query) - // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { if f.Title == "" { @@ -368,7 +363,7 @@ func Geo(f form.SearchGeo) (results GeoResults, err error) { return results, result.Error } - log.Infof("geo: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) + log.Debugf("geo: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) return results, nil } diff --git a/internal/search/like_test.go b/internal/search/like_test.go index 3c93a983a..ba4d4234a 100644 --- a/internal/search/like_test.go +++ b/internal/search/like_test.go @@ -208,23 +208,23 @@ func TestLikeAllNames(t *testing.T) { }) t.Run("Plus", func(t *testing.T) { if w := LikeAllNames(Cols{"name"}, sanitize.Query("Paul + Paula")); len(w) == 2 { - assert.Equal(t, "name LIKE '%paul%'", w[0]) - assert.Equal(t, "name LIKE '%paula%'", w[1]) + assert.Equal(t, "name LIKE '%Paul%'", w[0]) + assert.Equal(t, "name LIKE '%Paula%'", w[1]) } else { t.Fatalf("unexpected result: %#v", w) } }) t.Run("And", func(t *testing.T) { if w := LikeAllNames(Cols{"name"}, sanitize.Query("P and Paula")); len(w) == 2 { - assert.Equal(t, "name LIKE '%p%'", w[0]) - assert.Equal(t, "name LIKE '%paula%'", w[1]) + assert.Equal(t, "name LIKE '%P%'", w[0]) + assert.Equal(t, "name LIKE '%Paula%'", w[1]) } else { t.Fatalf("unexpected result: %#v", w) } }) t.Run("Or", func(t *testing.T) { if w := LikeAllNames(Cols{"name"}, sanitize.Query("Paul or Paula")); len(w) == 1 { - assert.Equal(t, "name LIKE '%paul%' OR name LIKE '%paula%'", w[0]) + assert.Equal(t, "name LIKE '%Paul%' OR name LIKE '%Paula%'", w[0]) } else { t.Fatalf("unexpected result: %#v", w) } diff --git a/internal/search/photos.go b/internal/search/photos.go index 47aab9b1d..d8a0d40c9 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -6,8 +6,6 @@ import ( "strings" "time" - "github.com/photoprism/photoprism/pkg/sanitize" - "github.com/dustin/go-humanize/english" "github.com/jinzhu/gorm" @@ -97,14 +95,14 @@ func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { s = s.Where("photos.photo_uid IN (?)", strings.Split(strings.ToLower(f.UID), txt.Or)) // Take shortcut? - if f.Album == "" && f.Albums == "" && f.Label == "" && f.Query == "" { + if f.Album == "" && f.Query == "" { s = s.Order("files.file_primary DESC") if result := s.Scan(&results); result.Error != nil { return results, 0, result.Error } - log.Infof("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) + log.Debugf("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) if f.Merged { return results.Merged() @@ -141,9 +139,6 @@ func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { } } - // Clip and normalize search query. - f.Query = sanitize.Query(f.Query) - // Set search filters based on search terms. if terms := txt.SearchTerms(f.Query); f.Query != "" && len(terms) == 0 { if f.Title == "" { @@ -517,7 +512,7 @@ func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { return results, 0, err } - log.Infof("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) + log.Debugf("photos: found %s for %s [%s]", english.Plural(len(results), "result", "results"), f.SerializeAll(), time.Since(start)) if f.Merged { return results.Merged() diff --git a/internal/search/subjects.go b/internal/search/subjects.go index 2decb9d07..3aa6f28ec 100644 --- a/internal/search/subjects.go +++ b/internal/search/subjects.go @@ -56,9 +56,6 @@ func Subjects(f form.SearchSubjects) (results SubjectResults, err error) { return results, nil } - // Clip to reasonable size and normalize operators. - f.Query = sanitize.Query(f.Query) - if f.Query != "" { for _, where := range LikeAllNames(Cols{"subj_name", "subj_alias"}, f.Query) { s = s.Where("?", gorm.Expr(where)) diff --git a/pkg/sanitize/filename.go b/pkg/sanitize/filename.go index 80337c12e..e54754c86 100644 --- a/pkg/sanitize/filename.go +++ b/pkg/sanitize/filename.go @@ -7,7 +7,7 @@ import ( // FileName removes invalid character from a filename string. func FileName(s string) string { - if len(s) > 512 || strings.Contains(s, "${") || strings.Contains(s, "/") || strings.Contains(s, "..") { + if s == "" || reject(s, 512) || strings.Contains(s, "/") || strings.Contains(s, "..") { return "" } diff --git a/pkg/sanitize/hex.go b/pkg/sanitize/hex.go index 9b63932f6..f1a21d634 100644 --- a/pkg/sanitize/hex.go +++ b/pkg/sanitize/hex.go @@ -6,11 +6,11 @@ import ( // Hex removes invalid character from a hex string and makes it lowercase. func Hex(s string) string { - if s == "" || len(s) > 1024 || strings.Contains(s, "${") { + if s == "" || reject(s, 1024) { return "" } - s = strings.ToLower(s) + s = strings.ToLower(strings.TrimSpace(s)) // Remove all invalid characters. s = strings.Map(func(r rune) rune { diff --git a/pkg/sanitize/id.go b/pkg/sanitize/id.go index a168f0711..a64634c57 100644 --- a/pkg/sanitize/id.go +++ b/pkg/sanitize/id.go @@ -7,7 +7,7 @@ import ( // IdString removes invalid character from an id string. func IdString(s string) string { - if s == "" || len(s) > 256 || strings.Contains(s, "${") { + if s == "" || reject(s, 512) { return "" } @@ -30,10 +30,12 @@ func IdUint(s string) uint { // Largest possible values: // UInt64: 18446744073709551615 (20 digits) // UInt32: 4294967295 (10 digits) - if s == "" || len(s) > 10 || strings.Contains(s, "${") { + if s == "" || reject(s, 10) { return 0 } + s = strings.TrimSpace(s) + result, err := strconv.ParseUint(s, 10, 32) if err != nil { diff --git a/pkg/sanitize/log.go b/pkg/sanitize/log.go index 9215f61f5..225b0cfb9 100644 --- a/pkg/sanitize/log.go +++ b/pkg/sanitize/log.go @@ -8,12 +8,12 @@ import ( // Log sanitizes strings created from user input in response to the log4j debacle. func Log(s string) string { - if len(s) > 200 || strings.Contains(s, "${") { + if reject(s, 512) { return "?" } // Trim quotes, tabs, and newline characters. - s = strings.Trim(s, "'\"“`\t\n\r") + s = strings.Trim(s, " '\"“`\t\n\r") // Remove non-printable and other potentially problematic characters. s = strings.Map(func(r rune) rune { diff --git a/pkg/sanitize/log_test.go b/pkg/sanitize/log_test.go index ba92b4e13..57425104d 100644 --- a/pkg/sanitize/log_test.go +++ b/pkg/sanitize/log_test.go @@ -20,7 +20,7 @@ func TestLog(t *testing.T) { assert.Equal(t, "?", Log("${https://:/}")) }) t.Run("Ldap", func(t *testing.T) { - assert.Equal(t, "'User-Agent: ?jndi:ldap://?host?:?port?/?path??'", Log("User-Agent: {jndi:ldap://:/}")) + assert.Equal(t, "?", Log("User-Agent: {jndi:ldap://:/}")) }) } diff --git a/pkg/sanitize/name.go b/pkg/sanitize/name.go index 466edd9a7..fca883e26 100644 --- a/pkg/sanitize/name.go +++ b/pkg/sanitize/name.go @@ -8,7 +8,7 @@ import ( // Name sanitizes and capitalizes names. func Name(name string) string { - if name == "" { + if name == "" || reject(name, txt.ClipDefault) { return "" } diff --git a/pkg/sanitize/path.go b/pkg/sanitize/path.go index bc9f4eec0..054b7b86b 100644 --- a/pkg/sanitize/path.go +++ b/pkg/sanitize/path.go @@ -7,7 +7,7 @@ import ( // Path removes invalid character from a path string. func Path(s string) string { - if len(s) > 512 || strings.Contains(s, "${") || strings.Contains(s, "..") || strings.Contains(s, "//") { + if s == "" || reject(s, 512) || strings.Contains(s, "..") || strings.Contains(s, "//") { return "" } diff --git a/pkg/sanitize/query.go b/pkg/sanitize/query.go index e46b409c1..837f04237 100644 --- a/pkg/sanitize/query.go +++ b/pkg/sanitize/query.go @@ -1,26 +1,36 @@ package sanitize -import "strings" +import ( + "regexp" + "strings" +) // spaced returns the string padded with a space left and right. func spaced(s string) string { return Space + s + Space } +// replace performs a case-insensitive string replacement. +func replace(subject string, search string, replace string) string { + return regexp.MustCompile("(?i)"+search).ReplaceAllString(subject, replace) +} + // Query replaces search operator with default symbols. func Query(s string) string { - if s == "" || len(s) > 1024 || strings.Contains(s, "${") { + if s == "" || reject(s, MaxLength) { return Empty } - s = strings.ToLower(s) - s = strings.ReplaceAll(s, spaced(EnOr), Or) - s = strings.ReplaceAll(s, spaced(EnAnd), And) - s = strings.ReplaceAll(s, spaced(EnWith), And) - s = strings.ReplaceAll(s, spaced(EnIn), And) - s = strings.ReplaceAll(s, spaced(EnAt), And) + // Normalize. + s = replace(s, spaced(EnOr), Or) + s = replace(s, spaced(EnOr), Or) + s = replace(s, spaced(EnAnd), And) + s = replace(s, spaced(EnWith), And) + s = replace(s, spaced(EnIn), And) + s = replace(s, spaced(EnAt), And) s = strings.ReplaceAll(s, SpacedPlus, And) s = strings.ReplaceAll(s, "%", "*") - return strings.Trim(s, "+&|_-=!@$%^(){}\\<>,.;: ") + // Trim. + return strings.Trim(s, "+&|-=$^(){}\\<>,;: \n\r\t") } diff --git a/pkg/sanitize/query_test.go b/pkg/sanitize/query_test.go index bdb71b00d..3762db350 100644 --- a/pkg/sanitize/query_test.go +++ b/pkg/sanitize/query_test.go @@ -8,7 +8,11 @@ import ( func TestQuery(t *testing.T) { t.Run("Replace", func(t *testing.T) { - q := Query("table spoon & usa | img% json OR BILL!") - assert.Equal(t, "table spoon & usa | img* json|bill", q) + q := Query("table spoon & usa | img% json OR BILL!\n") + assert.Equal(t, "table spoon & usa | img* json|BILL!", q) + }) + t.Run("AndOr", func(t *testing.T) { + q := Query("Jens AND Mander and me Or Kitty AND ") + assert.Equal(t, "Jens&Mander&me|Kitty", q) }) } diff --git a/pkg/sanitize/sanitize.go b/pkg/sanitize/sanitize.go index 93f2b4463..1e60a40e0 100644 --- a/pkg/sanitize/sanitize.go +++ b/pkg/sanitize/sanitize.go @@ -30,3 +30,19 @@ https://docs.photoprism.app/developer-guide/ */ package sanitize + +import "strings" + +const MaxLength = 4096 + +func reject(s string, maxLength int) bool { + if maxLength > 0 && len(s) > maxLength { + return true + } + + if strings.Contains(s, "${") || strings.Contains(s, "ldap://") { + return true + } + + return false +} diff --git a/pkg/sanitize/state.go b/pkg/sanitize/state.go index 62f002324..ad8314e60 100644 --- a/pkg/sanitize/state.go +++ b/pkg/sanitize/state.go @@ -8,18 +8,22 @@ import ( ) // State returns the full, normalized state name. -func State(stateName, countryCode string) string { +func State(s, countryCode string) string { + if s == "" || reject(s, txt.ClipName) { + return Empty + } + // Remove whitespace from name. - stateName = strings.TrimSpace(stateName) + s = strings.TrimSpace(s) // Empty? - if stateName == "" || stateName == txt.UnknownStateCode { + if s == "" || s == txt.UnknownStateCode { // State doesn't have a name. return "" } // Remove non-printable and other potentially problematic characters. - stateName = strings.Map(func(r rune) rune { + s = strings.Map(func(r rune) rune { if !unicode.IsPrint(r) { return -1 } @@ -30,7 +34,7 @@ func State(stateName, countryCode string) string { default: return r } - }, stateName) + }, s) // Normalize country code. countryCode = strings.ToLower(strings.TrimSpace(countryCode)) @@ -38,13 +42,13 @@ func State(stateName, countryCode string) string { // Is the name an abbreviation that should be normalized? if states, found := txt.StatesByCountry[countryCode]; !found { // Unknown country. - } else if normalized, found := states[stateName]; !found { + } else if normalized, found := states[s]; !found { // Unknown abbreviation. } else if normalized != "" { // Yes, use normalized name. - stateName = normalized + s = normalized } // Return normalized state name. - return stateName + return s } diff --git a/pkg/sanitize/token.go b/pkg/sanitize/token.go index 206d37527..457608a09 100644 --- a/pkg/sanitize/token.go +++ b/pkg/sanitize/token.go @@ -6,7 +6,7 @@ import ( // Token removes invalid character from a token string. func Token(s string) string { - if s == "" || len(s) > 200 || strings.Contains(s, "${") { + if s == "" || reject(s, 200) { return "" } diff --git a/pkg/sanitize/username.go b/pkg/sanitize/username.go index 2e12dfaf7..4e255da10 100644 --- a/pkg/sanitize/username.go +++ b/pkg/sanitize/username.go @@ -8,5 +8,11 @@ import ( // Username returns the normalized username (lowercase, whitespace trimmed). func Username(s string) string { - return strings.ToLower(txt.Clip(s, txt.ClipUsername)) + s = strings.TrimSpace(s) + + if s == "" || reject(s, txt.ClipUsername) { + return "" + } + + return strings.ToLower(s) }