diff --git a/Makefile b/Makefile index 2c4217525..23d74e60f 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ test-firefox: test-go: $(info Running all Go unit tests...) $(GOTEST) -tags=slow -timeout 20m ./internal/... -test-debug: +test-verbose: $(info Running all Go unit tests in verbose mode...) $(GOTEST) -tags=slow -timeout 20m -v ./internal/... test-short: diff --git a/go.mod b/go.mod index b160870b1..efb33ce4e 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a + github.com/satori/go.uuid v1.2.0 github.com/simplereach/timeutils v1.2.0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/soheilhy/cmux v0.1.4 // indirect diff --git a/go.sum b/go.sum index d334531c9..93a66b046 100644 --- a/go.sum +++ b/go.sum @@ -251,6 +251,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYo github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a h1:ZDZdsnbMuRSoVbq1gR47o005lfn2OwODNCr23zh9gSk= github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/simplereach/timeutils v1.2.0 h1:btgOAlu9RW6de2r2qQiONhjgxdAG7BL6je0G6J/yPnA= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/internal/config/config.go b/internal/config/config.go index f85721333..0f1ab5dc6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -346,11 +346,14 @@ func (c *Config) CloseDb() error { func (c *Config) MigrateDb() { db := c.Db() + // db.LogMode(true) + db.AutoMigrate( &models.File{}, &models.Photo{}, - &models.Tag{}, - &models.PhotoTag{}, + &models.Label{}, + &models.Synonym{}, + &models.PhotoLabel{}, &models.Album{}, &models.Location{}, &models.Camera{}, diff --git a/internal/forms/photo_search.go b/internal/forms/photo_search.go index 56e9324fa..b8281c8f1 100644 --- a/internal/forms/photo_search.go +++ b/internal/forms/photo_search.go @@ -33,7 +33,7 @@ type PhotoSearchForm struct { Mono bool `form:"mono"` Portrait bool `form:"portrait"` Location bool `form:"location"` - Tags string `form:"tags"` + Label string `form:"label"` Country string `form:"country"` Color string `form:"color"` Camera int `form:"camera"` diff --git a/internal/forms/photo_search_test.go b/internal/forms/photo_search_test.go index 4df724b15..461641d03 100644 --- a/internal/forms/photo_search_test.go +++ b/internal/forms/photo_search_test.go @@ -16,14 +16,14 @@ func TestPhotoSearchForm(t *testing.T) { } func TestParseQueryString(t *testing.T) { - form := &PhotoSearchForm{Query: "tags:foo,bar query:\"fooBar baz\" before:2019-01-15 camera:23"} + form := &PhotoSearchForm{Query: "label:cat query:\"fooBar baz\" before:2019-01-15 camera:23"} err := form.ParseQueryString() log.Debugf("%+v\n", form) assert.Nil(t, err) - assert.Equal(t, "foo,bar", form.Tags) + assert.Equal(t, "cat", form.Label) assert.Equal(t, "foobar baz", form.Query) assert.Equal(t, 23, form.Camera) assert.Equal(t, time.Date(2019, 01, 15, 0, 0, 0, 0, time.UTC), form.Before) diff --git a/internal/models/album.go b/internal/models/album.go index f951c4d28..091324e2b 100644 --- a/internal/models/album.go +++ b/internal/models/album.go @@ -2,11 +2,13 @@ package models import ( "github.com/jinzhu/gorm" + "github.com/satori/go.uuid" ) // Photo album type Album struct { - gorm.Model + Model + AlbumUUID string `gorm:"unique_index;"` AlbumSlug string AlbumName string AlbumDescription string `gorm:"type:text;"` @@ -15,3 +17,7 @@ type Album struct { AlbumPhotoID uint Photos []Photo `gorm:"many2many:album_photos;"` } + +func (m *Album) BeforeCreate(scope *gorm.Scope) error { + return scope.SetColumn("AlbumUUID", uuid.NewV4().String()) +} diff --git a/internal/models/camera.go b/internal/models/camera.go index 11e529b54..cae33a300 100644 --- a/internal/models/camera.go +++ b/internal/models/camera.go @@ -10,7 +10,7 @@ import ( // Camera model and make (as extracted from EXIF metadata) type Camera struct { - gorm.Model + Model CameraSlug string CameraModel string CameraMake string @@ -46,17 +46,17 @@ func NewCamera(modelName string, makeName string) *Camera { return result } -func (c *Camera) FirstOrCreate(db *gorm.DB) *Camera { - db.FirstOrCreate(c, "camera_model = ? AND camera_make = ?", c.CameraModel, c.CameraMake) +func (m *Camera) FirstOrCreate(db *gorm.DB) *Camera { + db.FirstOrCreate(m, "camera_model = ? AND camera_make = ?", m.CameraModel, m.CameraMake) - return c + return m } -func (c *Camera) String() string { - if c.CameraMake != "" && c.CameraModel != "" { - return fmt.Sprintf("%s %s", c.CameraMake, c.CameraModel) - } else if c.CameraModel != "" { - return c.CameraModel +func (m *Camera) String() string { + if m.CameraMake != "" && m.CameraModel != "" { + return fmt.Sprintf("%s %s", m.CameraMake, m.CameraModel) + } else if m.CameraModel != "" { + return m.CameraModel } return "" diff --git a/internal/models/country.go b/internal/models/country.go index 1a96f5734..c819c276b 100644 --- a/internal/models/country.go +++ b/internal/models/country.go @@ -36,8 +36,8 @@ func NewCountry(countryCode string, countryName string) *Country { return result } -func (c *Country) FirstOrCreate(db *gorm.DB) *Country { - db.FirstOrCreate(c, "id = ?", c.ID) +func (m *Country) FirstOrCreate(db *gorm.DB) *Country { + db.FirstOrCreate(m, "id = ?", m.ID) - return c + return m } diff --git a/internal/models/file.go b/internal/models/file.go index 6477348d6..0dcc45c45 100644 --- a/internal/models/file.go +++ b/internal/models/file.go @@ -6,49 +6,56 @@ import ( "github.com/gosimple/slug" "github.com/jinzhu/gorm" + "github.com/satori/go.uuid" ) // An image or sidecar file that belongs to a photo type File struct { - gorm.Model + Model Photo *Photo PhotoID uint - FilePrimary bool - FileMissing bool - FileDuplicate bool - FileName string `gorm:"type:varchar(512);index"` // max 3072 bytes / 4 bytes for utf8mb4 = 768 chars + FileUUID string `gorm:"unique_index;"` + FileName string `gorm:"type:varchar(512);unique_index"` // max 3072 bytes / 4 bytes for utf8mb4 = 768 chars + FileHash string `gorm:"type:varchar(128);unique_index"` FileOriginalName string FileType string `gorm:"type:varchar(32)"` FileMime string `gorm:"type:varchar(64)"` + FilePrimary bool + FileMissing bool + FileDuplicate bool + FilePortrait bool FileWidth int FileHeight int FileOrientation int FileAspectRatio float64 - FilePortrait bool FileMainColor string FileColors string FileLuminance string FileChroma uint - FileHash string `gorm:"type:varchar(128);unique_index"` FileNotes string `gorm:"type:text"` } -func (f *File) DownloadFileName() string { - if f.Photo == nil { - return fmt.Sprintf("%s.%s", f.FileHash, f.FileType) +func (m *File) BeforeCreate(scope *gorm.Scope) error { + return scope.SetColumn("FileUUID", uuid.NewV4().String()) +} + + +func (m *File) DownloadFileName() string { + if m.Photo == nil { + return fmt.Sprintf("%s.%s", m.FileHash, m.FileType) } var name string - if f.Photo.PhotoTitle != "" { - name = strings.Title(slug.Make(f.Photo.PhotoTitle)) + if m.Photo.PhotoTitle != "" { + name = strings.Title(slug.Make(m.Photo.PhotoTitle)) } else { - name = string(f.PhotoID) + name = string(m.PhotoID) } - taken := f.Photo.TakenAt.Format("20060102-150405") + taken := m.Photo.TakenAt.Format("20060102-150405") - result := fmt.Sprintf("%s-%s.%s", taken, name, f.FileType) + result := fmt.Sprintf("%s-%s.%s", taken, name, m.FileType) return result } diff --git a/internal/models/label.go b/internal/models/label.go new file mode 100644 index 000000000..8d2d7f5d7 --- /dev/null +++ b/internal/models/label.go @@ -0,0 +1,43 @@ +package models + +import ( + "strings" + + "github.com/gosimple/slug" + "github.com/jinzhu/gorm" +) + +// Labels for photo, album and location categorization +type Label struct { + Model + LabelSlug string `gorm:"type:varchar(128);index;"` + LabelName string `gorm:"type:varchar(128);index;"` + LabelPriority int + LabelDescription string `gorm:"type:text;"` + LabelNotes string `gorm:"type:text;"` + LabelSynonyms []*Label `gorm:"many2many:synonyms;association_jointable_foreignkey:synonym_id"` +} + +func NewLabel(labelName string, labelPriority int) *Label { + labelName = strings.TrimSpace(labelName) + + if labelName == "" { + labelName = "Unknown" + } + + labelSlug := slug.Make(labelName) + + result := &Label{ + LabelName: labelName, + LabelSlug: labelSlug, + LabelPriority: labelPriority, + } + + return result +} + +func (m *Label) FirstOrCreate(db *gorm.DB) *Label { + db.FirstOrCreate(m, "label_slug = ?", m.LabelSlug) + + return m +} diff --git a/internal/models/lens.go b/internal/models/lens.go index b70445f43..118583d7b 100644 --- a/internal/models/lens.go +++ b/internal/models/lens.go @@ -7,7 +7,7 @@ import ( // Camera lens (as extracted from EXIF metadata) type Lens struct { - gorm.Model + Model LensSlug string LensModel string LensMake string @@ -37,8 +37,8 @@ func NewLens(modelName string, makeName string) *Lens { return result } -func (c *Lens) FirstOrCreate(db *gorm.DB) *Lens { - db.FirstOrCreate(c, "lens_model = ? AND lens_make = ?", c.LensModel, c.LensMake) +func (m *Lens) FirstOrCreate(db *gorm.DB) *Lens { + db.FirstOrCreate(m, "lens_model = ? AND lens_make = ?", m.LensModel, m.LensMake) - return c + return m } diff --git a/internal/models/location.go b/internal/models/location.go index 6aded2752..4237da5b5 100644 --- a/internal/models/location.go +++ b/internal/models/location.go @@ -1,12 +1,8 @@ package models -import ( - "github.com/jinzhu/gorm" -) - // Photo location type Location struct { - gorm.Model + Model LocDisplayName string LocLat float64 LocLong float64 diff --git a/internal/models/model.go b/internal/models/model.go new file mode 100644 index 000000000..50369461e --- /dev/null +++ b/internal/models/model.go @@ -0,0 +1,10 @@ +package models + +import "time" + +type Model struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `sql:"index"` +} diff --git a/internal/models/doc.go b/internal/models/models.go similarity index 100% rename from internal/models/doc.go rename to internal/models/models.go diff --git a/internal/models/photo.go b/internal/models/photo.go index e58e9a871..23e521072 100644 --- a/internal/models/photo.go +++ b/internal/models/photo.go @@ -4,13 +4,13 @@ import ( "time" "github.com/jinzhu/gorm" + "github.com/satori/go.uuid" ) // A photo can have multiple images and sidecar files type Photo struct { - gorm.Model - TakenAt time.Time - TakenAtChanged bool + Model + PhotoUUID string `gorm:"unique_index;"` PhotoTitle string PhotoTitleChanged bool PhotoDescription string `gorm:"type:text;"` @@ -32,7 +32,13 @@ type Photo struct { Location *Location LocationID uint LocationChanged bool - Tags []*Tag `gorm:"many2many:photo_tags;"` + TakenAt time.Time + TakenAtChanged bool + Labels []*PhotoLabel Files []*File Albums []*Album `gorm:"many2many:album_photos;"` } + +func (m *Photo) BeforeCreate(scope *gorm.Scope) error { + return scope.SetColumn("PhotoUUID", uuid.NewV4().String()) +} diff --git a/internal/models/photo_label.go b/internal/models/photo_label.go new file mode 100644 index 000000000..e9e9cf062 --- /dev/null +++ b/internal/models/photo_label.go @@ -0,0 +1,36 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +// Photo labels are weighted by uncertainty (100 - confidence) +type PhotoLabel struct { + PhotoID uint `gorm:"primary_key;auto_increment:false"` + LabelID uint `gorm:"primary_key;auto_increment:false"` + LabelUncertainty int + LabelSource string + Photo *Photo + Label *Label +} + +func (PhotoLabel) TableName() string { + return "photo_labels" +} + +func NewPhotoLabel(photoId, labelId uint, uncertainty int, source string) *PhotoLabel { + result := &PhotoLabel{ + PhotoID: photoId, + LabelID: labelId, + LabelUncertainty: uncertainty, + LabelSource: source, + } + + return result +} + +func (m *PhotoLabel) FirstOrCreate(db *gorm.DB) *PhotoLabel { + db.FirstOrCreate(m, "photo_id = ? AND label_id = ?", m.PhotoID, m.LabelID) + + return m +} diff --git a/internal/models/photo_tag.go b/internal/models/photo_tag.go deleted file mode 100644 index 674f63446..000000000 --- a/internal/models/photo_tag.go +++ /dev/null @@ -1,23 +0,0 @@ -package models - -import ( - "github.com/jinzhu/gorm" -) - -// Photo tags are weighted by confidence (probability * 100) -type PhotoTag struct { - PhotoID uint `gorm:"primary_key"` - TagID uint `gorm:"primary_key"` - TagConfidence uint - TagSource string -} - -func (PhotoTag) TableName() string { - return "photo_tags" -} - -func (t *PhotoTag) FirstOrCreate(db *gorm.DB) *PhotoTag { - db.FirstOrCreate(t, "photo_id = ? AND tag_id = ?", t.PhotoID, t.TagID) - - return t -} diff --git a/internal/models/synonym.go b/internal/models/synonym.go new file mode 100644 index 000000000..9462a3f8e --- /dev/null +++ b/internal/models/synonym.go @@ -0,0 +1,13 @@ +package models + +// Labels can have zero or more synonyms with the same or a similar meaning +type Synonym struct { + LabelID uint `gorm:"primary_key;auto_increment:false"` + SynonymID uint `gorm:"primary_key;auto_increment:false"` + Label *Label + Synonym *Label +} + +func (Synonym) TableName() string { + return "synonyms" +} diff --git a/internal/models/tag.go b/internal/models/tag.go deleted file mode 100644 index 9578ca5fa..000000000 --- a/internal/models/tag.go +++ /dev/null @@ -1,39 +0,0 @@ -package models - -import ( - "strings" - - "github.com/gosimple/slug" - "github.com/jinzhu/gorm" -) - -// Photo tagß -type Tag struct { - gorm.Model - TagSlug string `gorm:"type:varchar(100);unique_index"` - TagLabel string `gorm:"type:varchar(100);unique_index"` -} - -// Create a new tag -func NewTag(label string) *Tag { - if label == "" { - label = "unknown" - } - - tagLabel := strings.ToLower(label) - - tagSlug := slug.MakeLang(tagLabel, "en") - - result := &Tag{ - TagLabel: tagLabel, - TagSlug: tagSlug, - } - - return result -} - -func (t *Tag) FirstOrCreate(db *gorm.DB) *Tag { - db.FirstOrCreate(t, "tag_label = ?", t.TagLabel) - - return t -} diff --git a/internal/photoprism/colors.go b/internal/photoprism/colors.go index 5b52ae7e2..9706a1c81 100644 --- a/internal/photoprism/colors.go +++ b/internal/photoprism/colors.go @@ -6,8 +6,6 @@ import ( "image/color" "math" - log "github.com/sirupsen/logrus" - "github.com/lucasb-eyer/go-colorful" ) diff --git a/internal/photoprism/converter.go b/internal/photoprism/converter.go index ff7429436..cd4270ba8 100644 --- a/internal/photoprism/converter.go +++ b/internal/photoprism/converter.go @@ -5,8 +5,6 @@ import ( "os" "os/exec" "path/filepath" - - log "github.com/sirupsen/logrus" ) // Converter wraps a darktable cli binary. diff --git a/internal/photoprism/importer.go b/internal/photoprism/importer.go index a8c91c22f..e849bf04c 100644 --- a/internal/photoprism/importer.go +++ b/internal/photoprism/importer.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/photoprism/photoprism/internal/config" - log "github.com/sirupsen/logrus" "github.com/photoprism/photoprism/internal/util" ) diff --git a/internal/photoprism/indexer.go b/internal/photoprism/indexer.go index d1d2e09e9..661a708aa 100644 --- a/internal/photoprism/indexer.go +++ b/internal/photoprism/indexer.go @@ -11,7 +11,6 @@ import ( "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/models" - log "github.com/sirupsen/logrus" ) const ( @@ -46,9 +45,9 @@ func (i *Indexer) thumbnailsPath() string { return i.conf.ThumbnailsPath() } -// getImageTags returns all tags of a given mediafile. This function returns +// classifyImage returns all tags of a given mediafile. This function returns // an empty list in the case of an error. -func (i *Indexer) getImageTags(jpeg *MediaFile) (results []*models.Tag) { +func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels) { start := time.Now() var thumbs []string @@ -59,7 +58,7 @@ func (i *Indexer) getImageTags(jpeg *MediaFile) (results []*models.Tag) { thumbs = []string{"tile_224", "left_224", "right_224"} } - var allLabels TensorFlowLabels + var labels Labels for _, thumb := range thumbs { filename, err := jpeg.Thumbnail(i.thumbnailsPath(), thumb) @@ -69,110 +68,143 @@ func (i *Indexer) getImageTags(jpeg *MediaFile) (results []*models.Tag) { continue } - labels, err := i.tensorFlow.GetImageTagsFromFile(filename) + imageLabels, err := i.tensorFlow.LabelsFromFile(filename) if err != nil { log.Error(err) continue } - allLabels = append(allLabels, labels...) + + labels = append(labels, imageLabels...) } - // Sort by probability - sort.Sort(TensorFlowLabels(allLabels)) + // Sort by uncertainty + sort.Sort(labels) - var max float32 = -1 + var confidence int - for _, l := range allLabels { - if max == -1 { - max = l.Probability + for _, label := range labels { + if confidence == 0 { + confidence = 100 - label.Uncertainty } - if l.Probability > (max / 3) { - results = i.appendTag(results, l.Label) + if (100 - label.Uncertainty) > (confidence / 3) { + results = append(results, label) } } elapsed := time.Since(start) - log.Infof("finding %+v labels for %s took %s", allLabels, jpeg.Filename(), elapsed) + log.Infof("finding %+v labels for %s took %s", results, jpeg.Filename(), elapsed) return results } -func (i *Indexer) appendTag(tags []*models.Tag, label string) []*models.Tag { - if label == "" { - return tags - } - - label = strings.ToLower(label) - - for _, tag := range tags { - if tag.TagLabel == label { - return tags - } - } - - tag := models.NewTag(label).FirstOrCreate(i.db) - - return append(tags, tag) -} - func (i *Indexer) indexMediaFile(mediaFile *MediaFile) string { var photo models.Photo var file, primaryFile models.File var isPrimary = false - var tags []*models.Tag + labels := Labels{} canonicalName := mediaFile.CanonicalNameFromFile() fileHash := mediaFile.Hash() relativeFileName := mediaFile.RelativeFilename(i.originalsPath()) photoQuery := i.db.First(&photo, "photo_canonical_name = ?", canonicalName) - if photoQuery.Error != nil { - if jpeg, err := mediaFile.Jpeg(); err == nil { - // Geo Location - if exifData, err := jpeg.ExifData(); err == nil { - photo.PhotoLat = exifData.Lat - photo.PhotoLong = exifData.Long - photo.PhotoArtist = exifData.Artist - } + if photo.TakenAt.IsZero() && photo.TakenAtChanged == false { + photo.TakenAt = mediaFile.DateCreated() + } - // Tags (TensorFlow) - tags = i.getImageTags(jpeg) + if photo.PhotoCanonicalName == "" { + photo.PhotoCanonicalName = canonicalName + } + + if jpeg, err := mediaFile.Jpeg(); err == nil { + // Image classification labels + labels = i.classifyImage(jpeg) + + // Read Exif data + if exifData, err := jpeg.ExifData(); err == nil { + photo.PhotoLat = exifData.Lat + photo.PhotoLong = exifData.Long + photo.PhotoArtist = exifData.Artist } - if location, err := mediaFile.Location(); err == nil { - i.db.FirstOrCreate(location, "id = ?", location.ID) - photo.Location = location - photo.Country = models.NewCountry(location.LocCountryCode, location.LocCountry).FirstOrCreate(i.db) + // Set Camera, Lens, Focal Length and Aperture + photo.Camera = models.NewCamera(mediaFile.CameraModel(), mediaFile.CameraMake()).FirstOrCreate(i.db) + photo.Lens = models.NewLens(mediaFile.LensModel(), mediaFile.LensMake()).FirstOrCreate(i.db) + photo.PhotoFocalLength = mediaFile.FocalLength() + photo.PhotoAperture = mediaFile.Aperture() + } - tags = i.appendTag(tags, location.LocCity) - tags = i.appendTag(tags, location.LocCounty) - tags = i.appendTag(tags, location.LocCountry) - tags = i.appendTag(tags, location.LocCategory) - tags = i.appendTag(tags, location.LocName) - tags = i.appendTag(tags, location.LocType) + if location, err := mediaFile.Location(); err == nil { + i.db.FirstOrCreate(location, "id = ?", location.ID) + photo.Location = location + photo.Country = models.NewCountry(location.LocCountryCode, location.LocCountry).FirstOrCreate(i.db) - if photo.PhotoTitle == "" && location.LocName != "" && location.LocCity != "" { // TODO: User defined title format - if len(location.LocName) > 40 { - photo.PhotoTitle = fmt.Sprintf("%s / %s", strings.Title(location.LocName), mediaFile.DateCreated().Format("2006")) - } else { - photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", strings.Title(location.LocName), location.LocCity, mediaFile.DateCreated().Format("2006")) - } - } else if photo.PhotoTitle == "" && location.LocCity != "" && location.LocCountry != "" { - photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", location.LocCity, location.LocCountry, mediaFile.DateCreated().Format("2006")) - } else if photo.PhotoTitle == "" && location.LocCounty != "" && location.LocCountry != "" { - photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", location.LocCounty, location.LocCountry, mediaFile.DateCreated().Format("2006")) + // Append labels from OpenStreetMap + labels = append(labels, NewLocationLabel(location.LocCity, 0, 1)) + labels = append(labels, NewLocationLabel(location.LocCounty, 0, 0)) + labels = append(labels, NewLocationLabel(location.LocCountry, 0, 0)) + labels = append(labels, NewLocationLabel(location.LocCategory, 0, 2)) + labels = append(labels, NewLocationLabel(location.LocName, 0, 3)) + labels = append(labels, NewLocationLabel(location.LocType, 0, 0)) + + if photo.PhotoTitle == "" && location.LocName != "" && location.LocCity != "" { // TODO: User defined title format + if len(location.LocName) > 40 { + photo.PhotoTitle = fmt.Sprintf("%s / %s", strings.Title(location.LocName), mediaFile.DateCreated().Format("2006")) + } else { + photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", strings.Title(location.LocName), location.LocCity, mediaFile.DateCreated().Format("2006")) } - } else { - log.Debugf("location cannot be determined precisely: %s", err) + } else if photo.PhotoTitle == "" && location.LocCity != "" && location.LocCountry != "" { + photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", location.LocCity, location.LocCountry, mediaFile.DateCreated().Format("2006")) + } else if photo.PhotoTitle == "" && location.LocCounty != "" && location.LocCountry != "" { + photo.PhotoTitle = fmt.Sprintf("%s / %s / %s", location.LocCounty, location.LocCountry, mediaFile.DateCreated().Format("2006")) + } + } else { + log.Debugf("location cannot be determined precisely: %s", err) + } + if photo.PhotoTitle == "" { + if len(labels) > 0 { // TODO: User defined title format + photo.PhotoTitle = fmt.Sprintf("%s / %s", strings.Title(labels[0].Name), mediaFile.DateCreated().Format("2006")) + } else if photo.Camera.String() != "" && photo.Camera.String() != "Unknown" { + photo.PhotoTitle = fmt.Sprintf("%s / %s", photo.Camera, mediaFile.DateCreated().Format("2006")) + } else { + var daytimeString string + hour := mediaFile.DateCreated().Hour() + + switch { + case hour < 8: + daytimeString = "Early Bird" + case hour < 12: + daytimeString = "Morning Mood" + case hour < 17: + daytimeString = "Daytime" + case hour < 20: + daytimeString = "Sunset" + default: + daytimeString = "Late Night" + } + + photo.PhotoTitle = fmt.Sprintf("%s / %s", daytimeString, mediaFile.DateCreated().Format("2006")) + } + } + + log.Debugf("title: \"%s\"", photo.PhotoTitle) + + if photoQuery.Error != nil { + photo.PhotoFavorite = false + + i.db.Create(&photo) + } else { + // Estimate location + if photo.LocationID == 0 { var recentPhoto models.Photo - if result := i.db.Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", mediaFile.DateCreated())).Preload("Country").First(&recentPhoto); result.Error == nil { + if result := i.db.Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Country").First(&recentPhoto); result.Error == nil { if recentPhoto.Country != nil { photo.Country = recentPhoto.Country log.Debugf("approximate location: %s", recentPhoto.Country.CountryName) @@ -180,74 +212,34 @@ func (i *Indexer) indexMediaFile(mediaFile *MediaFile) string { } } - photo.Tags = tags - - photo.Camera = models.NewCamera(mediaFile.CameraModel(), mediaFile.CameraMake()).FirstOrCreate(i.db) - photo.Lens = models.NewLens(mediaFile.LensModel(), mediaFile.LensMake()).FirstOrCreate(i.db) - photo.PhotoFocalLength = mediaFile.FocalLength() - photo.PhotoAperture = mediaFile.Aperture() - - photo.TakenAt = mediaFile.DateCreated() - photo.PhotoCanonicalName = canonicalName - photo.PhotoFavorite = false - - if photo.PhotoTitle == "" { - if len(photo.Tags) > 0 { // TODO: User defined title format - photo.PhotoTitle = fmt.Sprintf("%s / %s", strings.Title(photo.Tags[0].TagLabel), mediaFile.DateCreated().Format("2006")) - } else if photo.Camera.String() != "" && photo.Camera.String() != "Unknown" { - photo.PhotoTitle = fmt.Sprintf("%s / %s", photo.Camera, mediaFile.DateCreated().Format("January 2006")) - } else { - var daytimeString string - hour := mediaFile.DateCreated().Hour() - - switch { - case hour < 8: - daytimeString = "Early Bird" - case hour < 12: - daytimeString = "Morning Mood" - case hour < 17: - daytimeString = "Carpe Diem" - case hour < 20: - daytimeString = "Sunset" - default: - daytimeString = "Late Night" - } - - photo.PhotoTitle = fmt.Sprintf("%s / %s", daytimeString, mediaFile.DateCreated().Format("January 2006")) - } - } - - log.Debugf("title: \"%s\"", photo.PhotoTitle) - - i.db.Create(&photo) - } else if time.Now().Sub(photo.UpdatedAt).Minutes() > 10 { // If updated more than 10 minutes ago - if jpeg, err := mediaFile.Jpeg(); err == nil { - photo.Camera = models.NewCamera(mediaFile.CameraModel(), mediaFile.CameraMake()).FirstOrCreate(i.db) - photo.Lens = models.NewLens(mediaFile.LensModel(), mediaFile.LensMake()).FirstOrCreate(i.db) - photo.PhotoFocalLength = mediaFile.FocalLength() - photo.PhotoAperture = mediaFile.Aperture() - - // Geo Location - if exifData, err := jpeg.ExifData(); err == nil { - photo.PhotoLat = exifData.Lat - photo.PhotoLong = exifData.Long - photo.PhotoArtist = exifData.Artist - } - } - - if photo.LocationID == 0 { - var recentPhoto models.Photo - - if result := i.db.Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Country").First(&recentPhoto); result.Error == nil { - if recentPhoto.Country != nil { - photo.Country = recentPhoto.Country - } - } - } - i.db.Save(&photo) } + log.Infof("adding labels: %+v", labels) + + for _, label := range labels { + lm := models.NewLabel(label.Name, label.Priority).FirstOrCreate(i.db) + + if lm.LabelPriority != label.Priority { + lm.LabelPriority = label.Priority + i.db.Save(&lm) + } + + plm := models.NewPhotoLabel(photo.ID, lm.ID, label.Uncertainty, label.Source).FirstOrCreate(i.db) + + // Add synonyms + for _, synonym := range label.Synonyms { + sn := models.NewLabel(synonym, -1).FirstOrCreate(i.db) + i.db.Model(&lm).Association("LabelSynonyms").Append(sn) + } + + if plm.LabelUncertainty > label.Uncertainty { + plm.LabelUncertainty = label.Uncertainty + plm.LabelSource = label.Source + i.db.Save(&plm) + } + } + if result := i.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); result.Error != nil { isPrimary = mediaFile.Type() == FileTypeJpeg } else { diff --git a/internal/photoprism/label.go b/internal/photoprism/label.go new file mode 100644 index 000000000..15d95c3a1 --- /dev/null +++ b/internal/photoprism/label.go @@ -0,0 +1,35 @@ +package photoprism + +type Label struct { + Name string `json:"label"` // Label name + Source string `json:"source"` // Where was this label found / detected? + Uncertainty int `json:"uncertainty"` // >= 0 + Priority int `json:"priority"` // >= 0 + Synonyms []string `json:"synonyms"` // List of similar labels +} + +func NewLocationLabel(name string, uncertainty int, priority int) Label { + label := Label{Name: name, Source: "location", Uncertainty: uncertainty, Priority: priority} + + return label +} + +type Labels []Label + +func (l Labels) Len() int { return len(l) } +func (l Labels) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l Labels) Less(i, j int) bool { + if l[i].Priority == l[j].Priority { + return l[i].Uncertainty < l[j].Uncertainty + } else { + return l[i].Priority > l[j].Priority + } +} + +func (l Labels) AppendLabel(label Label) Labels { + if label.Name == "" { + return l + } + + return append(l, label) +} diff --git a/internal/photoprism/mediafile.go b/internal/photoprism/mediafile.go index 9927a4652..181d4b2d6 100644 --- a/internal/photoprism/mediafile.go +++ b/internal/photoprism/mediafile.go @@ -14,8 +14,6 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - "github.com/djherbis/times" "github.com/photoprism/photoprism/internal/models" "github.com/photoprism/photoprism/internal/util" diff --git a/internal/photoprism/doc.go b/internal/photoprism/photoprism.go similarity index 62% rename from internal/photoprism/doc.go rename to internal/photoprism/photoprism.go index 42a290f67..8d48a9599 100644 --- a/internal/photoprism/doc.go +++ b/internal/photoprism/photoprism.go @@ -6,3 +6,11 @@ Additional information can be found in our Developer Guide: https://github.com/photoprism/photoprism/wiki */ package photoprism + +import "github.com/sirupsen/logrus" + +var log *logrus.Logger + +func init () { + log = logrus.StandardLogger() +} diff --git a/internal/photoprism/photoprism_test.go b/internal/photoprism/photoprism_test.go index 5d7d94b06..45e9a2dc9 100644 --- a/internal/photoprism/photoprism_test.go +++ b/internal/photoprism/photoprism_test.go @@ -3,12 +3,12 @@ package photoprism import ( "os" "testing" - - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" ) func TestMain(m *testing.M) { - log.SetLevel(log.DebugLevel) + log = logrus.StandardLogger() + log.SetLevel(logrus.DebugLevel) code := m.Run() os.Exit(code) } diff --git a/internal/photoprism/search.go b/internal/photoprism/search.go index 8baa5b90b..275aebef6 100644 --- a/internal/photoprism/search.go +++ b/internal/photoprism/search.go @@ -121,17 +121,38 @@ func (s *Search) Photos(form forms.PhotoSearchForm) (results []PhotoSearchResult countries.country_name, locations.loc_display_name, locations.loc_name, locations.loc_city, locations.loc_postcode, locations.loc_county, locations.loc_state, locations.loc_country, locations.loc_country_code, locations.loc_category, locations.loc_type, - GROUP_CONCAT(tags.tag_label) AS tags`). + GROUP_CONCAT(labels.label_name) AS labels`). Joins("JOIN files ON files.photo_id = photos.id AND files.file_primary AND files.deleted_at IS NULL"). Joins("JOIN cameras ON cameras.id = photos.camera_id"). Joins("JOIN lenses ON lenses.id = photos.lens_id"). Joins("LEFT JOIN countries ON countries.id = photos.country_id"). Joins("LEFT JOIN locations ON locations.id = photos.location_id"). - Joins("LEFT JOIN photo_tags ON photo_tags.photo_id = photos.id"). - Joins("LEFT JOIN tags ON photo_tags.tag_id = tags.id"). + Joins("LEFT JOIN photo_labels ON photo_labels.photo_id = photos.id"). + Joins("LEFT JOIN labels ON photo_labels.label_id = labels.id"). Where("photos.deleted_at IS NULL AND files.file_missing = 0"). Group("photos.id, files.id") + var synonyms []models.Synonym + var label models.Label + var labelIds []uint + + if form.Label != "" { + if result := s.db.First(&label, "label_slug = ?", form.Label); result.Error != nil { + log.Errorf("label \"%s\" not found", form.Label) + return results, fmt.Errorf("label \"%s\" not found", form.Label) + } else { + labelIds = append(labelIds, label.ID) + + s.db.Where("synonym_id = ?", label.ID).Find(&synonyms) + + for _, synonym := range synonyms { + labelIds = append(labelIds, synonym.LabelID) + } + + q = q.Where("labels.id IN (?)", labelIds) + } + } + if form.Location == true { q = q.Where("location_id > 0") @@ -141,7 +162,25 @@ func (s *Search) Photos(form forms.PhotoSearchForm) (results []PhotoSearchResult } } else if form.Query != "" { likeString := "%" + strings.ToLower(form.Query) + "%" - q = q.Where("tags.tag_label LIKE ? OR LOWER(photo_title) LIKE ? OR LOWER(files.file_main_color) LIKE ?", likeString, likeString, likeString) + + if result := s.db.First(&label, "LOWER(label_name) LIKE LOWER(?)", form.Query); result.Error != nil { + log.Infof("label \"%s\" not found", form.Query) + + q = q.Where("LOWER(labels.label_name) LIKE ? OR LOWER(photo_title) LIKE ? OR LOWER(files.file_main_color) LIKE ?", likeString, likeString, likeString) + } else { + labelIds = append(labelIds, label.ID) + + s.db.Where("synonym_id = ?", label.ID).Find(&synonyms) + + for _, synonym := range synonyms { + labelIds = append(labelIds, synonym.LabelID) + } + + log.Infof("searching for label IDs: %#v", form.Query) + + q = q.Where("labels.id IN (?) OR LOWER(photo_title) LIKE ? OR LOWER(files.file_main_color) LIKE ?", labelIds, likeString, likeString) + } + } if form.Camera > 0 { @@ -160,10 +199,6 @@ func (s *Search) Photos(form forms.PhotoSearchForm) (results []PhotoSearchResult q = q.Where("locations.loc_country_code = ?", form.Country) } - if form.Tags != "" { - q = q.Where("tags.tag_label = ?", form.Tags) - } - if form.Title != "" { q = q.Where("LOWER(photos.photo_title) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(form.Title))) } diff --git a/internal/photoprism/tensorflow.go b/internal/photoprism/tensorflow.go index 7228a8216..6dec56e3e 100644 --- a/internal/photoprism/tensorflow.go +++ b/internal/photoprism/tensorflow.go @@ -14,7 +14,6 @@ import ( "github.com/disintegration/imaging" "github.com/photoprism/photoprism/internal/util" - log "github.com/sirupsen/logrus" tf "github.com/tensorflow/tensorflow/tensorflow/go" "gopkg.in/yaml.v2" ) @@ -27,14 +26,6 @@ type TensorFlow struct { labelRules LabelRules } -// TensorFlowLabel defines a Json struct with label and probability. -type TensorFlowLabel struct { - Label string `json:"label"` - Probability float32 `json:"probability"` - Synonyms []string - Priority int -} - type LabelRule struct { Tag string See string @@ -50,10 +41,6 @@ func NewTensorFlow(tensorFlowModelPath string) *TensorFlow { return &TensorFlow{modelPath: tensorFlowModelPath} } -func (a *TensorFlowLabel) Percent() int { - return int(math.Round(float64(a.Probability * 100))) -} - func (t *TensorFlow) loadLabelRules() (err error) { if len(t.labelRules) > 0 { return nil @@ -82,38 +69,25 @@ func (t *TensorFlow) loadLabelRules() (err error) { return err } -// TensorFlowLabels is a slice of tensorflow labels. -type TensorFlowLabels []TensorFlowLabel - -func (a TensorFlowLabels) Len() int { return len(a) } -func (a TensorFlowLabels) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a TensorFlowLabels) Less(i, j int) bool { - if a[i].Priority == a[j].Priority { - return a[i].Probability > a[j].Probability - } else { - return a[i].Priority > a[j].Priority - } -} - -// GetImageTagsFromFile returns tags for a jpeg image file. -func (t *TensorFlow) GetImageTagsFromFile(filename string) (result []TensorFlowLabel, err error) { +// LabelsFromFile returns tags for a jpeg image file. +func (t *TensorFlow) LabelsFromFile(filename string) (result Labels, err error) { imageBuffer, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - return t.GetImageTags(imageBuffer) + return t.Labels(imageBuffer) } -// GetImageTags returns tags for a jpeg image string. -func (t *TensorFlow) GetImageTags(img []byte) (result []TensorFlowLabel, err error) { +// Labels returns tags for a jpeg image string. +func (t *TensorFlow) Labels(img []byte) (result Labels, err error) { if err := t.loadModel(); err != nil { return nil, err } // Make tensor - tensor, err := t.makeTensorFromImage(img, "jpeg") + tensor, err := t.makeTensor(img, "jpeg") if err != nil { return nil, errors.New("invalid image") @@ -138,7 +112,7 @@ func (t *TensorFlow) GetImageTags(img []byte) (result []TensorFlowLabel, err err } // Return best labels - result = t.findBestLabels(output[0].Value().([][]float32)[0]) + result = t.bestLabels(output[0].Value().([][]float32)[0]) log.Debugf("labels: %v", result) @@ -206,13 +180,13 @@ func (t *TensorFlow) labelRule(label string) LabelRule { return LabelRule{Threshold: 0.08} } -func (t *TensorFlow) findBestLabels(probabilities []float32) []TensorFlowLabel { +func (t *TensorFlow) bestLabels(probabilities []float32) Labels { if err := t.loadLabelRules(); err != nil { log.Error(err) } // Make a list of label/probability pairs - var result []TensorFlowLabel + var result Labels for i, p := range probabilities { if i >= len(t.labels) { @@ -235,11 +209,13 @@ func (t *TensorFlow) findBestLabels(probabilities []float32) []TensorFlowLabel { labelText = rule.Tag } - result = append(result, TensorFlowLabel{Label: labelText, Probability: p, Synonyms: rule.Synonyms, Priority: rule.Priority}) + uncertainty := 100 - int(math.Round(float64(p * 100))) + + result = append(result, Label{Name: labelText, Source: "image", Uncertainty: uncertainty, Priority: rule.Priority, Synonyms: rule.Synonyms}) } // Sort by probability - sort.Sort(TensorFlowLabels(result)) + sort.Sort(Labels(result)) if l := len(result); l < 5 { return result[:l] @@ -248,7 +224,7 @@ func (t *TensorFlow) findBestLabels(probabilities []float32) []TensorFlowLabel { } } -func (t *TensorFlow) makeTensorFromImage(image []byte, imageFormat string) (*tf.Tensor, error) { +func (t *TensorFlow) makeTensor(image []byte, imageFormat string) (*tf.Tensor, error) { img, err := imaging.Decode(bytes.NewReader(image), imaging.AutoOrientation(true)) if err != nil { diff --git a/internal/photoprism/tensorflow_test.go b/internal/photoprism/tensorflow_test.go index c738900e5..48978c2e9 100644 --- a/internal/photoprism/tensorflow_test.go +++ b/internal/photoprism/tensorflow_test.go @@ -8,14 +8,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestTensorFlow_GetImageTagsFromFile(t *testing.T) { +func TestTensorFlow_LabelsFromFile(t *testing.T) { ctx := config.TestConfig() ctx.InitializeTestData(t) tensorFlow := NewTensorFlow(ctx.TensorFlowModelPath()) - result, err := tensorFlow.GetImageTagsFromFile(ctx.ImportPath() + "/iphone/IMG_6788.JPG") + result, err := tensorFlow.LabelsFromFile(ctx.ImportPath() + "/iphone/IMG_6788.JPG") assert.Nil(t, err) @@ -25,19 +25,19 @@ func TestTensorFlow_GetImageTagsFromFile(t *testing.T) { } assert.NotNil(t, result) - assert.IsType(t, []TensorFlowLabel{}, result) + assert.IsType(t, Labels{}, result) assert.Equal(t, 2, len(result)) t.Log(result) - assert.Equal(t, "tabby cat", result[0].Label) - assert.Equal(t, "tiger cat", result[1].Label) + assert.Equal(t, "tabby cat", result[0].Name) + assert.Equal(t, "tiger cat", result[1].Name) - assert.Equal(t, 68, result[0].Percent()) - assert.Equal(t, 14, result[1].Percent()) + assert.Equal(t, 32, result[0].Uncertainty) + assert.Equal(t, 86, result[1].Uncertainty) } -func TestTensorFlow_GetImageTags(t *testing.T) { +func TestTensorFlow_Labels(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -51,25 +51,25 @@ func TestTensorFlow_GetImageTags(t *testing.T) { if imageBuffer, err := ioutil.ReadFile(ctx.ImportPath() + "/iphone/IMG_6788.JPG"); err != nil { t.Error(err) } else { - result, err := tensorFlow.GetImageTags(imageBuffer) + result, err := tensorFlow.Labels(imageBuffer) t.Log(result) assert.NotNil(t, result) assert.Nil(t, err) - assert.IsType(t, []TensorFlowLabel{}, result) + assert.IsType(t, Labels{}, result) assert.Equal(t, 2, len(result)) - assert.Equal(t, "tabby cat", result[0].Label) - assert.Equal(t, "tiger cat", result[1].Label) + assert.Equal(t, "tabby cat", result[0].Name) + assert.Equal(t, "tiger cat", result[1].Name) - assert.Equal(t, 68, result[0].Percent()) - assert.Equal(t, 14, result[1].Percent()) + assert.Equal(t, 100 - 68, result[0].Uncertainty) + assert.Equal(t, 100 - 14, result[1].Uncertainty) } } -func TestTensorFlow_GetImageTags_Dog(t *testing.T) { +func TestTensorFlow_Labels_Dog(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -83,20 +83,20 @@ func TestTensorFlow_GetImageTags_Dog(t *testing.T) { if imageBuffer, err := ioutil.ReadFile(ctx.ImportPath() + "/dog.jpg"); err != nil { t.Error(err) } else { - result, err := tensorFlow.GetImageTags(imageBuffer) + result, err := tensorFlow.Labels(imageBuffer) t.Log(result) assert.NotNil(t, result) assert.Nil(t, err) - assert.IsType(t, []TensorFlowLabel{}, result) + assert.IsType(t, Labels{}, result) assert.Equal(t, 3, len(result)) - assert.Equal(t, "belt", result[0].Label) - assert.Equal(t, "beagle dog", result[1].Label) + assert.Equal(t, "belt", result[0].Name) + assert.Equal(t, "beagle dog", result[1].Name) - assert.Equal(t, 11, result[0].Percent()) - assert.Equal(t, 9, result[1].Percent()) + assert.Equal(t, 100 - 11, result[0].Uncertainty) + assert.Equal(t, 100 - 9, result[1].Uncertainty) } } diff --git a/internal/photoprism/thumbnails.go b/internal/photoprism/thumbnails.go index bc691d37c..1590c5195 100644 --- a/internal/photoprism/thumbnails.go +++ b/internal/photoprism/thumbnails.go @@ -10,7 +10,6 @@ import ( "time" "github.com/photoprism/photoprism/internal/config" - log "github.com/sirupsen/logrus" "github.com/disintegration/imaging" "github.com/photoprism/photoprism/internal/util"