Places: Normalize US & CA state names #1664

This commit is contained in:
Michael Mayer 2021-11-09 11:42:10 +01:00
parent c0e2d8bdf9
commit e4fd294689
24 changed files with 443 additions and 131 deletions

10
go.mod
View file

@ -21,9 +21,9 @@ require (
github.com/go-playground/validator/v10 v10.9.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/open-location-code/go v0.0.0-20211105222919-588c0e721570
github.com/google/open-location-code/go v0.0.0-20211109014933-06433367679b
github.com/gorilla/websocket v1.4.2
github.com/gosimple/slug v1.11.0
github.com/gosimple/slug v1.11.2
github.com/h2non/filetype v1.1.1
github.com/jinzhu/gorm v1.9.16
github.com/jinzhu/inflection v1.0.0
@ -57,9 +57,9 @@ require (
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6
github.com/urfave/cli v1.22.5
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d // indirect
golang.org/x/net v0.0.0-20211105192438-b53810dc28af
golang.org/x/net v0.0.0-20211108170745-6635138e15ea
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 // indirect
golang.org/x/text v0.3.7 // indirect
gonum.org/v1/gonum v0.9.3
@ -78,7 +78,7 @@ require (
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
github.com/gosimple/unidecode v1.0.0 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect

34
go.sum
View file

@ -80,8 +80,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/esimov/pigo v1.4.4 h1:Ab9uYXw0F0Y7OyZQQGwJjktl5LlHdL3ovdXe/T0juK8=
github.com/esimov/pigo v1.4.4/go.mod h1:SGkOUpm4wlEmQQJKlaymAkThY8/8iP+XE0gFo7g8G6w=
github.com/esimov/pigo v1.4.5 h1:ySG0QqMh02VNALvHnx04L1ScRu66N6XA5vLLga8GiLg=
github.com/esimov/pigo v1.4.5/go.mod h1:SGkOUpm4wlEmQQJKlaymAkThY8/8iP+XE0gFo7g8G6w=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@ -153,10 +151,8 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/open-location-code/go v0.0.0-20210504205230-1796878d947c h1:kiK/0Vz+XhUoQU+PAVuP30aVHObEz0HMawJQXKiSzV4=
github.com/google/open-location-code/go v0.0.0-20210504205230-1796878d947c/go.mod h1:eJfRN6aj+kR/rnua/rw9jAgYhqoMHldQkdTi+sePRKk=
github.com/google/open-location-code/go v0.0.0-20211105222919-588c0e721570 h1:wj685HxZcnyOIGKGgoZJIYyhtbyHUzsumYMLvUuZKwo=
github.com/google/open-location-code/go v0.0.0-20211105222919-588c0e721570/go.mod h1:eJfRN6aj+kR/rnua/rw9jAgYhqoMHldQkdTi+sePRKk=
github.com/google/open-location-code/go v0.0.0-20211109014933-06433367679b h1:6rfkSqY/nWZGdgpfCLumEAh3Remb/v1eyrGnFt5dCIs=
github.com/google/open-location-code/go v0.0.0-20211109014933-06433367679b/go.mod h1:eJfRN6aj+kR/rnua/rw9jAgYhqoMHldQkdTi+sePRKk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@ -165,12 +161,10 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gosimple/slug v1.10.0 h1:3XbiQua1IpCdrvuntWvGBxVm+K99wCSxJjlxkP49GGQ=
github.com/gosimple/slug v1.10.0/go.mod h1:MICb3w495l9KNdZm+Xn5b6T2Hn831f9DMxiJ1r+bAjw=
github.com/gosimple/slug v1.11.0 h1:QkFeOkXIEDvvtIt++P7cUuO4G9PZVQEgLuYbYZzawMA=
github.com/gosimple/slug v1.11.0/go.mod h1:MICb3w495l9KNdZm+Xn5b6T2Hn831f9DMxiJ1r+bAjw=
github.com/gosimple/unidecode v1.0.0 h1:kPdvM+qy0tnk4/BrnkrbdJ82xe88xn7c9hcaipDz4dQ=
github.com/gosimple/unidecode v1.0.0/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
github.com/gosimple/slug v1.11.2 h1:MxFR0TmQ/qz0KvIrBbf4phu+G0RBgpwxOn6jPKFKFOw=
github.com/gosimple/slug v1.11.2/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4=
github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@ -290,18 +284,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df h1:C+J/LwTqP8gRPt1MdSzBNZP0OYuDm5wsmDKgwpLjYzo=
github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f h1:SLJx0nHhb2ZLlYNMAbrYsjwmVwXx4yRT48lNIxOp7ts=
github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
github.com/tensorflow/tensorflow v1.15.2 h1:7/f/A664Tml/nRJg04+p3StcrsT53mkcvmxYHXI21Qo=
github.com/tensorflow/tensorflow v1.15.2/go.mod h1:itOSERT4trABok4UOoG+X4BoKds9F3rIsySdn+Lvu90=
github.com/tidwall/gjson v1.9.1 h1:wrrRk7TyL7MmKanNRck/Mcr3VU1sdMvJHvJXzqBIUNo=
github.com/tidwall/gjson v1.9.1/go.mod h1:jydLKE7s8J0+1/5jC4eXcuFlzKizGrCKvLmBVX/5oXc=
github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4=
github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
@ -331,8 +319,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -396,10 +384,8 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk=
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211105192438-b53810dc28af h1:SMeNJG/vclJ5wyBBd4xupMsSJIHTd1coW9g7q6KOjmY=
golang.org/x/net v0.0.0-20211105192438-b53810dc28af/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8=
golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

View file

@ -1,8 +1,9 @@
package form
import (
"github.com/stretchr/testify/assert"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNewFaceSearch(t *testing.T) {

View file

@ -12,7 +12,7 @@ import (
"github.com/photoprism/photoprism/pkg/txt"
)
// Location
// Location represents a specific geolocation identified by its S2 ID.
type Location struct {
ID string `json:"id"`
LocLat float64 `json:"lat"`
@ -136,7 +136,7 @@ func (l Location) Label() (result string) {
}
func (l Location) State() (result string) {
return l.Place.LocState
return txt.NormalizeState(l.Place.LocState)
}
func (l Location) City() (result string) {

View file

@ -63,7 +63,7 @@ func TestLocationGetters(t *testing.T) {
assert.Equal(t, "TestLocation", location.Name())
assert.Equal(t, "test", location.Category())
assert.Equal(t, "testLabel", location.Label())
assert.Equal(t, "berlin", location.State())
assert.Equal(t, "Berlin", location.State())
assert.Equal(t, "de", location.CountryCode())
assert.Equal(t, "berlin", location.City())
assert.Equal(t, 52.51961810676184, location.Latitude())
@ -71,5 +71,17 @@ func TestLocationGetters(t *testing.T) {
assert.Equal(t, "places", location.Source())
assert.Equal(t, []string{"foobar"}, location.Keywords())
})
}
func TestLocation_State(t *testing.T) {
t.Run("Washington", func(t *testing.T) {
var p = NewPlace("549ed22c0434", "Seattle, WA", "Seattle", "WA", "us", "")
location := NewLocation("54903ee07f74", 47.6129432, -122.4821475, "", "", p, true)
assert.Equal(t, "54903ee07f74", location.CellID())
assert.Equal(t, "Seattle, WA", location.Label())
assert.Equal(t, "Washington", location.State())
assert.Equal(t, "us", location.CountryCode())
assert.Equal(t, "Seattle", location.City())
assert.Equal(t, "places", location.Source())
})
}

View file

@ -1,6 +1,6 @@
package places
// Place
// Place represents a region identified by city, state, and country.
type Place struct {
PlaceID string `json:"id"`
LocLabel string `json:"label"`

View file

@ -7,9 +7,10 @@ import (
"github.com/photoprism/photoprism/internal/hub/places"
"github.com/photoprism/photoprism/internal/maps/osm"
"github.com/photoprism/photoprism/pkg/s2"
"github.com/photoprism/photoprism/pkg/txt"
)
// Photo location
// Location represents a geolocation.
type Location struct {
ID string
LocName string
@ -40,7 +41,7 @@ func NewLocation(id, name, category, label, city, state, country, source string,
LocCategory: category,
LocLabel: label,
LocCity: city,
LocState: state,
LocState: txt.NormalizeState(state),
LocCountry: country,
LocSource: source,
LocKeywords: keywords,
@ -164,7 +165,7 @@ func (l Location) City() string {
}
func (l Location) State() string {
return l.LocState
return txt.NormalizeState(l.LocState)
}
func (l Location) CountryCode() string {

View file

@ -9,7 +9,9 @@ import (
"time"
"github.com/melihmucuk/geocache"
"github.com/photoprism/photoprism/pkg/s2"
"github.com/photoprism/photoprism/pkg/txt"
)
type Location struct {
@ -25,7 +27,7 @@ type Location struct {
var ReverseLookupURL = "https://nominatim.openstreetmap.org/reverse?lat=%f&lon=%f&format=jsonv2&accept-language=en&zoom=18"
// API docs see https://wiki.openstreetmap.org/wiki/Nominatim#Reverse_Geocoding
// FindLocation retrieves geolocation data, docs see https://wiki.openstreetmap.org/wiki/Nominatim#Reverse_Geocoding
func FindLocation(id string) (result Location, err error) {
if len(id) > 16 || len(id) == 0 {
return result, errors.New("osm: invalid location id")
@ -84,9 +86,7 @@ func (l Location) CellID() (result string) {
}
func (l Location) State() (result string) {
result = l.Address.State
return strings.TrimSpace(result)
return txt.NormalizeState(l.Address.State)
}
func (l Location) City() (result string) {

View file

@ -6,7 +6,9 @@ import (
"time"
"github.com/gosimple/slug"
"github.com/photoprism/photoprism/internal/maps"
"github.com/photoprism/photoprism/pkg/txt"
)
// Moment contains photo counts per month and year
@ -77,6 +79,8 @@ func (m Moment) Slug() string {
// Title returns an english title for the moment.
func (m Moment) Title() string {
state := txt.NormalizeState(m.State)
if m.Year == 0 && m.Month == 0 {
if m.Label != "" {
return MomentLabels[m.Label]
@ -84,20 +88,20 @@ func (m Moment) Title() string {
country := maps.CountryName(m.Country)
if strings.Contains(m.State, country) {
return m.State
if strings.Contains(state, country) {
return state
}
if m.State == "" {
if state == "" {
return m.Country
}
return fmt.Sprintf("%s / %s", m.State, country)
return fmt.Sprintf("%s / %s", state, country)
}
if m.Country != "" && m.Year > 1900 && m.Month == 0 {
if m.State != "" {
return fmt.Sprintf("%s / %s / %d", m.State, maps.CountryName(m.Country), m.Year)
if state != "" {
return fmt.Sprintf("%s / %s / %d", state, maps.CountryName(m.Country), m.Year)
}
return fmt.Sprintf("%s %d", maps.CountryName(m.Country), m.Year)

View file

@ -35,6 +35,7 @@ import (
"fmt"
"os"
"path"
"runtime/debug"
"time"
"github.com/photoprism/photoprism/internal/event"
@ -72,8 +73,14 @@ func (c Client) readDir(path string) ([]os.FileInfo, error) {
return c.client.ReadDir(path)
}
// Files returns all files in path as string slice.
// Files returns all files in a directory as string slice.
func (c Client) Files(dir string) (result fs.FileInfos, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("webdav: %s (panic while listing files)\nstack: %s", r, debug.Stack())
}
}()
files, err := c.readDir(dir)
if err != nil {
@ -142,7 +149,13 @@ func (c Client) fetchDirs(root string, recursive bool, start time.Time, timeout
}
// Download downloads a single file to the given location.
func (c Client) Download(from, to string, force bool) error {
func (c Client) Download(from, to string, force bool) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("webdav: %s (panic while downloading)\nstack: %s", r, debug.Stack())
}
}()
if _, err := os.Stat(to); err == nil && !force {
return fmt.Errorf("webdav: download skipped, %s already exists", to)
}
@ -159,7 +172,9 @@ func (c Client) Download(from, to string, force bool) error {
return fmt.Errorf("webdav: %s is not a folder", dir)
}
bytes, err := c.client.Read(from)
var bytes []byte
bytes, err = c.client.Read(from)
if err != nil {
return err
@ -179,18 +194,18 @@ func (c Client) DownloadDir(from, to string, recursive, force bool) (errs []erro
for _, file := range files {
dest := to + string(os.PathSeparator) + file.Abs
if _, err := os.Stat(dest); err == nil {
// File exists
if _, err = os.Stat(dest); err == nil {
// File already exists.
msg := fmt.Errorf("webdav: %s exists", dest)
errs = append(errs, msg)
log.Error(msg)
log.Warn(msg)
continue
}
if err := c.Download(file.Abs, dest, force); err != nil {
msg := fmt.Errorf("webdav: %s", err)
errs = append(errs, msg)
log.Error(msg)
if err = c.Download(file.Abs, dest, force); err != nil {
// Failed to download file.
errs = append(errs, err)
log.Error(err)
continue
}
}
@ -218,14 +233,22 @@ func (c Client) CreateDir(dir string) error {
}
// Upload uploads a single file to the remote server.
func (c Client) Upload(from, to string) error {
func (c Client) Upload(from, to string) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("webdav: %s (panic while uploading)\nstack: %s", r, debug.Stack())
}
}()
file, err := os.Open(from)
if err != nil || file == nil {
return err
}
defer file.Close()
defer func(file *os.File) {
_ = file.Close()
}(file)
return c.client.WriteStream(to, file, 0644)
}

View file

@ -1,8 +1,9 @@
package thumb
import (
"github.com/stretchr/testify/assert"
"testing"
"github.com/stretchr/testify/assert"
)
func TestName_Jpeg(t *testing.T) {

66
pkg/txt/gen_states.go Normal file
View file

@ -0,0 +1,66 @@
//go:build ignore
// +build ignore
// This generates states.go by running "go generate"
package main
import (
"bufio"
"os"
"strings"
"text/template"
"github.com/photoprism/photoprism/pkg/txt"
)
type State struct {
Code string
Name string
}
var states []State
func main() {
file, err := os.Open("./resources/states.txt")
defer file.Close()
if err != nil {
panic(err)
}
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
parts := strings.Split(scanner.Text(), ":")
if len(parts) < 2 {
continue
}
states = append(states, State{Code: strings.ToUpper(parts[0]), Name: txt.Title(parts[1])})
}
f, err := os.Create("states.go")
if err != nil {
panic(err)
}
defer f.Close()
packageTemplate.Execute(f, struct {
States []State
}{
States: states,
})
}
var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
package txt
var States = map[string]string{
{{- range .States }}
{{ printf "%q" .Code }}: {{ printf "%q" .Name }},
{{- end }}
}`))

View file

@ -92,28 +92,3 @@ func NameKeywords(names, aliases string) (results []string) {
return UniqueNames(append(Words(names), Words(aliases)...))
}
// NormalizeName sanitizes and capitalizes names.
func NormalizeName(name string) string {
if name == "" {
return ""
}
// Remove double quotes and other special characters.
name = strings.Map(func(r rune) rune {
switch r {
case '"', '`', '~', '\\', '/', '*', '%', '&', '|', '+', '=', '$', '@', '!', '?', ':', ';', '<', '>', '{', '}':
return -1
}
return r
}, name)
name = strings.TrimSpace(name)
if name == "" {
return ""
}
// Shorten and capitalize.
return Clip(Title(name), ClipDefault)
}

View file

@ -105,27 +105,3 @@ func TestNameKeywords(t *testing.T) {
assert.Equal(t, []string{"william", "henry", "gates", "iii", "windows", "guru"}, result)
})
}
func TestNormalizeName(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
assert.Equal(t, "", NormalizeName(""))
})
t.Run("BillGates", func(t *testing.T) {
assert.Equal(t, "William Henry Gates III", NormalizeName("William Henry Gates III"))
})
t.Run("Quotes", func(t *testing.T) {
assert.Equal(t, "William HenRy Gates'", NormalizeName("william \"HenRy\" gates' "))
})
t.Run("Slash", func(t *testing.T) {
assert.Equal(t, "William McCorn Gates'", NormalizeName("william\\ \"McCorn\" / gates' "))
})
t.Run("SpecialCharacters", func(t *testing.T) {
assert.Equal(t,
"'', '', '', '', '', '', '', '', '', '', '', '', Foo '', '', '', '', '', '', '', McBar '', ''",
NormalizeName("'\"', '`', '~', '\\\\', '/', '*', '%', '&', '|', '+', '=', '$', Foo '@', '!', '?', ':', ';', '<', '>', McBar '{', '}'"),
)
})
t.Run("Chinese", func(t *testing.T) {
assert.Equal(t, "陈 赵", NormalizeName(" 陈 赵"))
})
}

56
pkg/txt/normalize.go Normal file
View file

@ -0,0 +1,56 @@
package txt
import "strings"
// NormalizeName sanitizes and capitalizes names.
func NormalizeName(name string) string {
if name == "" {
return ""
}
// Remove double quotes and other special characters.
name = strings.Map(func(r rune) rune {
switch r {
case '"', '`', '~', '\\', '/', '*', '%', '&', '|', '+', '=', '$', '@', '!', '?', ':', ';', '<', '>', '{', '}':
return -1
}
return r
}, name)
name = strings.TrimSpace(name)
if name == "" {
return ""
}
// Shorten and capitalize.
return Clip(Title(name), ClipDefault)
}
// NormalizeState returns the full, normalized state name.
func NormalizeState(s string) string {
s = strings.TrimSpace(s)
if s == "" || s == UnknownStateCode {
return ""
}
if expanded, ok := States[s]; ok {
return expanded
}
return Title(s)
}
// NormalizeQuery replaces search operator with default symbols.
func NormalizeQuery(s string) string {
s = strings.ToLower(Clip(s, ClipQuery))
s = strings.ReplaceAll(s, Spaced(EnOr), Or)
s = strings.ReplaceAll(s, Spaced(EnAnd), And)
s = strings.ReplaceAll(s, Spaced(EnWith), And)
s = strings.ReplaceAll(s, Spaced(EnIn), And)
s = strings.ReplaceAll(s, Spaced(EnAt), And)
s = strings.ReplaceAll(s, SpacedPlus, And)
s = strings.ReplaceAll(s, "%", "*")
return strings.Trim(s, "+&|_-=!@$%^(){}\\<>,.;: ")
}

80
pkg/txt/normalize_test.go Normal file
View file

@ -0,0 +1,80 @@
package txt
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeName(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
assert.Equal(t, "", NormalizeName(""))
})
t.Run("BillGates", func(t *testing.T) {
assert.Equal(t, "William Henry Gates III", NormalizeName("William Henry Gates III"))
})
t.Run("Quotes", func(t *testing.T) {
assert.Equal(t, "William HenRy Gates'", NormalizeName("william \"HenRy\" gates' "))
})
t.Run("Slash", func(t *testing.T) {
assert.Equal(t, "William McCorn Gates'", NormalizeName("william\\ \"McCorn\" / gates' "))
})
t.Run("SpecialCharacters", func(t *testing.T) {
assert.Equal(t,
"'', '', '', '', '', '', '', '', '', '', '', '', Foo '', '', '', '', '', '', '', McBar '', ''",
NormalizeName("'\"', '`', '~', '\\\\', '/', '*', '%', '&', '|', '+', '=', '$', Foo '@', '!', '?', ':', ';', '<', '>', McBar '{', '}'"),
)
})
t.Run("Chinese", func(t *testing.T) {
assert.Equal(t, "陈 赵", NormalizeName(" 陈 赵"))
})
}
func TestNormalizeState(t *testing.T) {
t.Run("Berlin", func(t *testing.T) {
result := NormalizeState("Berlin")
assert.Equal(t, "Berlin", result)
})
t.Run("WA", func(t *testing.T) {
result := NormalizeState("WA")
assert.Equal(t, "Washington", result)
})
t.Run("Wa", func(t *testing.T) {
result := NormalizeState("Wa")
assert.Equal(t, "Wa", result)
})
t.Run("Washington", func(t *testing.T) {
result := NormalizeState("Washington")
assert.Equal(t, "Washington", result)
})
t.Run("Never mind Nirvana", func(t *testing.T) {
result := NormalizeState("Never mind Nirvana.")
assert.Equal(t, "Never Mind Nirvana.", result)
})
t.Run("Empty", func(t *testing.T) {
result := NormalizeState("")
assert.Equal(t, "", result)
})
t.Run("Unknown", func(t *testing.T) {
result := NormalizeState("zz")
assert.Equal(t, "", result)
})
t.Run("Space", func(t *testing.T) {
result := NormalizeState(" ")
assert.Equal(t, "", result)
})
}
func TestNormalizeQuery(t *testing.T) {
t.Run("Replace", func(t *testing.T) {
q := NormalizeQuery("table spoon & usa | img% json OR BILL!")
assert.Equal(t, "table spoon & usa | img* json|bill", q)
})
}

View file

@ -5,6 +5,7 @@ import (
"strings"
)
var UnknownStateCode = "zz"
var UnknownCountryCode = "zz"
var CountryWordsRegexp = regexp.MustCompile("[\\p{L}]{2,}")

View file

@ -24,19 +24,6 @@ func StripOr(s string) string {
return s
}
// NormalizeQuery replaces search operator with default symbols.
func NormalizeQuery(s string) string {
s = strings.ToLower(Clip(s, ClipQuery))
s = strings.ReplaceAll(s, Spaced(EnOr), Or)
s = strings.ReplaceAll(s, Spaced(EnAnd), And)
s = strings.ReplaceAll(s, Spaced(EnWith), And)
s = strings.ReplaceAll(s, Spaced(EnIn), And)
s = strings.ReplaceAll(s, Spaced(EnAt), And)
s = strings.ReplaceAll(s, SpacedPlus, And)
s = strings.ReplaceAll(s, "%", "*")
return strings.Trim(s, "+&|_-=!@$%^(){}\\<>,.;: ")
}
// QueryTooShort tests if a search query is too short.
func QueryTooShort(q string) bool {
q = strings.Trim(q, "- '")

View file

@ -36,13 +36,6 @@ func TestStripOr(t *testing.T) {
})
}
func TestNormalizeQuery(t *testing.T) {
t.Run("Replace", func(t *testing.T) {
q := NormalizeQuery("table spoon & usa | img% json OR BILL!")
assert.Equal(t, "table spoon & usa | img* json|bill", q)
})
}
func TestQueryTooShort(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
assert.False(t, QueryTooShort(""))

View file

@ -0,0 +1,72 @@
AB:Alberta
AL:Alabama
AK:Alaska
AS:American Samoa
AZ:Arizona
AR:Arkansas
BC:British Columbia
CA:California
CO:Colorado
CT:Connecticut
DE:Delaware
DC:District Of Columbia
FM:Federated States Of Micronesia
FL:Florida
GA:Georgia
GU:Guam
HI:Hawaii
ID:Idaho
IL:Illinois
IN:Indiana
IA:Iowa
KS:Kansas
KY:Kentucky
LA:Louisiana
MB:Manitoba
ME:Maine
MH:Marshall Islands
MD:Maryland
MA:Massachusetts
MI:Michigan
MN:Minnesota
MS:Mississippi
MO:Missouri
MT:Montana
NE:Nebraska
NL:Newfoundland and Labrador
NU:Nunavut
NV:Nevada
NB:New Brunswick
NH:New Hampshire
NJ:New Jersey
NM:New Mexico
NY:New York
NC:North Carolina
ND:North Dakota
NT:Northwest Territories
NS:Nova Scotia
MP:Northern Mariana Islands
OH:Ohio
OK:Oklahoma
ON:Ontario
OR:Oregon
PE:Prince Edward Island
PW:Palau
PA:Pennsylvania
PR:Puerto Rico
QC:Quebec
RI:Rhode Island
SK:Saskatchewan
SC:South Carolina
SD:South Dakota
TN:Tennessee
TX:Texas
UT:Utah
VT:Vermont
VI:Virgin Islands
VA:Virginia
WA:Washington
WV:West Virginia
WI:Wisconsin
WY:Wyoming
YT:Yukon

77
pkg/txt/states.go Normal file
View file

@ -0,0 +1,77 @@
// Code generated by go generate; DO NOT EDIT.
package txt
var States = map[string]string{
"AB": "Alberta",
"AL": "Alabama",
"AK": "Alaska",
"AS": "American Samoa",
"AZ": "Arizona",
"AR": "Arkansas",
"BC": "British Columbia",
"CA": "California",
"CO": "Colorado",
"CT": "Connecticut",
"DE": "Delaware",
"DC": "District of Columbia",
"FM": "Federated States of Micronesia",
"FL": "Florida",
"GA": "Georgia",
"GU": "Guam",
"HI": "Hawaii",
"ID": "Idaho",
"IL": "Illinois",
"IN": "Indiana",
"IA": "Iowa",
"KS": "Kansas",
"KY": "Kentucky",
"LA": "Louisiana",
"MB": "Manitoba",
"ME": "Maine",
"MH": "Marshall Islands",
"MD": "Maryland",
"MA": "Massachusetts",
"MI": "Michigan",
"MN": "Minnesota",
"MS": "Mississippi",
"MO": "Missouri",
"MT": "Montana",
"NE": "Nebraska",
"NL": "Newfoundland and Labrador",
"NU": "Nunavut",
"NV": "Nevada",
"NB": "New Brunswick",
"NH": "New Hampshire",
"NJ": "New Jersey",
"NM": "New Mexico",
"NY": "New York",
"NC": "North Carolina",
"ND": "North Dakota",
"NT": "Northwest Territories",
"NS": "Nova Scotia",
"MP": "Northern Mariana Islands",
"OH": "Ohio",
"OK": "Oklahoma",
"ON": "Ontario",
"OR": "Oregon",
"PE": "Prince Edward Island",
"PW": "Palau",
"PA": "Pennsylvania",
"PR": "Puerto Rico",
"QC": "Quebec",
"RI": "Rhode Island",
"SK": "Saskatchewan",
"SC": "South Carolina",
"SD": "South Dakota",
"TN": "Tennessee",
"TX": "Texas",
"UT": "Utah",
"VT": "Vermont",
"VI": "Virgin Islands",
"VA": "Virginia",
"WA": "Washington",
"WV": "West Virginia",
"WI": "Wisconsin",
"WY": "Wyoming",
"YT": "Yukon",
}

View file

@ -1,6 +1,6 @@
package txt
// List of titles and ranks in lowercase, see https://en.wikipedia.org/wiki/List_of_titles.
// TitlesAndRanks contains a list of name titles and ranks in lowercase, see https://en.wikipedia.org/wiki/List_of_titles.
var TitlesAndRanks = map[string]bool{
"emperor": true,
"caliph": true,

View file

@ -32,5 +32,6 @@ https://docs.photoprism.org/developer-guide/
package txt
//go:generate go run gen_countries.go
//go:generate go run gen_states.go
//go:generate go run gen_stopwords.go
//go:generate go fmt .