luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Backend: Improve query parser #266

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-05-11 14:49:00 +02:00
parent 3c47a85ea5
commit cdadf664ff
9 changed files with 149 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
internal/form/search.go
View file

@ -31,6 +31,8 @@ func ParseQueryString(f SearchForm) (result error) {
q = strings.TrimSpace(q) + "\n"
var queryStrings []string
for _, char := range q {
if unicode.IsSpace(char) && !escaped {
if isKeyValue {
@ -74,8 +76,8 @@ func ParseQueryString(f SearchForm) (result error) {
} else {
result = fmt.Errorf("unknown filter: %s", fieldName)
}
} else {
f.SetQuery(string(key))
} else if len(strings.TrimSpace(string(key))) > 0 {
queryStrings = append(queryStrings, strings.TrimSpace(string(key)))
}
escaped = false
@ -93,6 +95,10 @@ func ParseQueryString(f SearchForm) (result error) {
}
}
if len(queryStrings) > 0 {
f.SetQuery(strings.Join(queryStrings, " "))
}
if result != nil {
log.Errorf("error while parsing search form: %s", result)
}

4
internal/query/geo.go
View file

@ -12,8 +12,8 @@ import (
"github.com/photoprism/photoprism/pkg/txt"
)
// Geo searches for photos based on a Form and returns a PhotoResult slice.
func Geo(f form.GeoSearch) (results []GeoResult, err error) {
// Geo searches for photos based on a Form and returns GeoResults ([]GeoResult).
func Geo(f form.GeoSearch) (results GeoResults, err error) {
if err := f.ParseQueryString(); err != nil {
return results, err
}

2
internal/query/geo_result.go
View file

@ -25,3 +25,5 @@ func (g GeoResult) Lat() float64 {
func (g GeoResult) Lng() float64 {
return float64(g.PhotoLng)
}
type GeoResults []GeoResult

53
internal/query/photos.go
View file

@ -5,15 +5,15 @@ import (
"strings"
"time"
"github.com/gosimple/slug"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/form"
"github.com/photoprism/photoprism/pkg/capture"
"github.com/photoprism/photoprism/pkg/txt"
)
// Photos searches for photos based on a Form and returns a PhotoResult slice.
func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
// Photos searches for photos based on a Form and returns PhotosResults ([]PhotosResult).
func Photos(f form.PhotoSearch) (results PhotosResults, count int, err error) {
if err := f.ParseQueryString(); err != nil {
return results, 0, err
}
@ -32,13 +32,11 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
files.file_diff,
cameras.camera_make, cameras.camera_model,
lenses.lens_make, lenses.lens_model,
places.loc_label, places.loc_city, places.loc_state, places.loc_country
`).
places.loc_label, places.loc_city, places.loc_state, places.loc_country`).
Joins("JOIN files ON files.photo_id = photos.id AND files.file_type = 'jpg' AND files.file_missing = 0 AND files.deleted_at IS NULL").
Joins("JOIN cameras ON cameras.id = photos.camera_id").
Joins("JOIN lenses ON lenses.id = photos.lens_id").
Joins("JOIN places ON photos.place_id = places.id").
Joins("LEFT JOIN photos_labels ON photos_labels.photo_id = photos.id AND photos_labels.uncertainty < 100").
Group("photos.id, files.id")
if f.ID != "" {
@ -58,6 +56,7 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
var categories []entity.Category
var label entity.Label
var labels []entity.Label
var labelIds []uint
if f.Label != "" {
@ -74,46 +73,46 @@ func Photos(f form.PhotoSearch) (results PhotoResults, count int, err error) {
labelIds = append(labelIds, category.LabelID)
}
s = s.Where("photos_labels.label_id IN (?)", labelIds)
s = s.Joins("JOIN photos_labels ON photos_labels.photo_id = photos.id AND photos_labels.uncertainty < 100 AND photos_labels.label_id IN (?)", labelIds)
}
}
if f.Location == true {
s = s.Where("location_id > 0")
if f.Query != "" {
s = s.Joins("LEFT JOIN photos_keywords ON photos_keywords.photo_id = photos.id").
Joins("LEFT JOIN keywords ON photos_keywords.keyword_id = keywords.id").
Where("keywords.keyword LIKE ?", strings.ToLower(txt.Clip(f.Query, txt.ClipKeyword))+"%")
if likeAny := LikeAny("k.keyword", f.Query); likeAny != "" {
s = s.Where("photos.id IN (SELECT pk.photo_id FROM keywords k JOIN photos_keywords pk ON k.id = pk.keyword_id WHERE (?))", gorm.Expr(likeAny))
}
} else if f.Query != "" {
if len(f.Query) < 2 {
return results, 0, fmt.Errorf("query too short")
}
slugString := slug.Make(f.Query)
lowerString := strings.ToLower(f.Query)
likeString := txt.Clip(lowerString, txt.ClipKeyword) + "%"
s = s.Joins("LEFT JOIN photos_keywords ON photos_keywords.photo_id = photos.id").
Joins("LEFT JOIN keywords ON photos_keywords.keyword_id = keywords.id")
if result := Db().First(&label, "label_slug = ? OR custom_slug = ?", slugString, slugString); result.Error != nil {
if err := Db().Where(AnySlug("custom_slug", f.Query)).Find(&labels).Error; len(labels) == 0 || err != nil {
log.Infof("search: label %s not found, using fuzzy search", txt.Quote(f.Query))
s = s.Where("keywords.keyword LIKE ?", likeString)
if likeAny := LikeAny("k.keyword", f.Query); likeAny != "" {
s = s.Where("photos.id IN (SELECT pk.photo_id FROM keywords k JOIN photos_keywords pk ON k.id = pk.keyword_id WHERE (?))", gorm.Expr(likeAny))
}
} else {
labelIds = append(labelIds, label.ID)
for _, l := range labels {
labelIds = append(labelIds, l.ID)
Db().Where("category_id = ?", label.ID).Find(&categories)
Db().Where("category_id = ?", l.ID).Find(&categories)
for _, category := range categories {
labelIds = append(labelIds, category.LabelID)
log.Infof("search: label %s includes %d categories", txt.Quote(l.LabelName), len(categories))
for _, category := range categories {
labelIds = append(labelIds, category.LabelID)
}
}
log.Infof("search: label %s includes %d categories", txt.Quote(label.LabelName), len(labelIds))
s = s.Where("photos_labels.label_id IN (?) OR keywords.keyword LIKE ?", labelIds, likeString)
if likeAny := LikeAny("k.keyword", f.Query); likeAny != "" {
s = s.Where("photos.id IN (SELECT pk.photo_id FROM keywords k JOIN photos_keywords pk ON k.id = pk.keyword_id WHERE (?)) OR "+
"photos.id IN (SELECT pl.photo_id FROM photos_labels pl WHERE pl.uncertainty < 100 AND pl.label_id IN (?))", gorm.Expr(likeAny), labelIds)
} else {
s = s.Where("photos.id IN (SELECT pl.photo_id FROM photos_labels pl WHERE pl.uncertainty < 100 AND pl.label_id IN (?))", labelIds)
}
}
}

12
internal/query/photo_results.go → internal/query/photos_results.go
View file

@ -11,8 +11,8 @@ import (
"github.com/ulule/deepcopier"
)
// PhotoResult contains found photos and their main file plus other meta data.
type PhotoResult struct {
// PhotosResult contains found photos and their main file plus other meta data.
type PhotosResult struct {
// Photo
ID uint
CreatedAt time.Time
@ -81,11 +81,11 @@ type PhotoResult struct {
Files []entity.File
}
type PhotoResults []PhotoResult
type PhotosResults []PhotosResult
func (m PhotoResults) Merged() (PhotoResults, int, error) {
func (m PhotosResults) Merged() (PhotosResults, int, error) {
count := len(m)
merged := make([]PhotoResult, 0, count)
merged := make([]PhotosResult, 0, count)
var lastId uint
var i int
@ -116,7 +116,7 @@ func (m PhotoResults) Merged() (PhotoResults, int, error) {
return merged, count, nil
}
func (m *PhotoResult) ShareFileName() string {
func (m *PhotosResult) ShareFileName() string {
var name string
if m.PhotoTitle != "" {

50
internal/query/query.go
View file

@ -8,8 +8,13 @@ https://github.com/photoprism/photoprism/wiki
package query
import (
"fmt"
"strings"
"github.com/gosimple/slug"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/pkg/txt"
"github.com/jinzhu/gorm"
)
@ -47,3 +52,48 @@ func Db() *gorm.DB {
func UnscopedDb() *gorm.DB {
return entity.Db().Unscoped()
}
// LikeAny returns a where condition that matches any keyword in search.
func LikeAny(col, search string) (where string) {
var wheres []string
words := txt.UniqueKeywords(search)
if len(words) == 0 {
return ""
}
for _, w := range words {
if len(w) > 3 {
wheres = append(wheres, fmt.Sprintf("%s LIKE '%s%%'", col, w))
} else {
wheres = append(wheres, fmt.Sprintf("%s = '%s'", col, w))
}
}
return strings.Join(wheres, " OR ")
}
// AnySlug returns a where condition that matches any slug in search.
func AnySlug(col, search string) (where string) {
if search == "" {
return ""
}
var wheres []string
var words []string
for _, w := range strings.Split(search, " ") {
words = append(words, slug.Make(strings.TrimSpace(w)))
}
if len(words) == 0 {
return ""
}
for _, w := range words {
wheres = append(wheres, fmt.Sprintf("%s = '%s'", col, w))
}
return strings.Join(wheres, " OR ")
}

55
internal/query/query_test.go
View file

@ -7,6 +7,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
@ -29,3 +30,57 @@ func TestMain(m *testing.M) {
os.Exit(code)
}
func TestLikeAny(t *testing.T) {
t.Run("table spoon usa img json", func(t *testing.T) {
where := LikeAny("k.keyword", "table spoon usa img json")
assert.Equal(t, "k.keyword LIKE 'json%' OR k.keyword LIKE 'spoon%' OR k.keyword LIKE 'table%' OR k.keyword = 'usa'", where)
})
t.Run("cat dog", func(t *testing.T) {
where := LikeAny("k.keyword", "cat dog")
assert.Equal(t, "k.keyword = 'cat' OR k.keyword = 'dog'", where)
})
t.Run("spoon", func(t *testing.T) {
where := LikeAny("k.keyword", "spoon")
assert.Equal(t, "k.keyword LIKE 'spoon%'", where)
})
t.Run("img", func(t *testing.T) {
where := LikeAny("k.keyword", "img")
assert.Equal(t, "", where)
})
t.Run("empty", func(t *testing.T) {
where := LikeAny("k.keyword", "")
assert.Equal(t, "", where)
})
}
func TestAnySlug(t *testing.T) {
t.Run("table spoon usa img json", func(t *testing.T) {
where := AnySlug("custom_slug", "table spoon usa img json")
assert.Equal(t, "custom_slug = 'table' OR custom_slug = 'spoon' OR custom_slug = 'usa' OR custom_slug = 'img' OR custom_slug = 'json'", where)
})
t.Run("cat dog", func(t *testing.T) {
where := AnySlug("custom_slug", "cat dog")
assert.Equal(t, "custom_slug = 'cat' OR custom_slug = 'dog'", where)
})
t.Run("spoon", func(t *testing.T) {
where := AnySlug("custom_slug", "spoon")
assert.Equal(t, "custom_slug = 'spoon'", where)
})
t.Run("img", func(t *testing.T) {
where := AnySlug("custom_slug", "img")
assert.Equal(t, "custom_slug = 'img'", where)
})
t.Run("empty", func(t *testing.T) {
where := AnySlug("custom_slug", "")
assert.Equal(t, "", where)
})
}

1
pkg/txt/resources/stopwords.txt
View file

@ -2849,7 +2849,6 @@ una
unas
uno
unos
usa
usais
usamos
usan

1
pkg/txt/stopwords.go
View file

@ -2854,7 +2854,6 @@ var Stopwords = map[string]bool{
"unas": true,
"uno": true,
"unos": true,
"usa": true,
"usais": true,
"usamos": true,
"usan": true,