Indexer: Improve titles, labels and performance

Signed-off-by: Michael Mayer <michael@liquidbytes.net>
2019-12-12 16:31:55 +01:00 · 2019-12-12 16:31:55 +01:00 · 645d02d782
commit 645d02d782
parent 4df887fffa
7 changed files with 182 additions and 131 deletions
--- a/assets/config/labels.yml
+++ b/assets/config/labels.yml
@ -145,11 +145,11 @@ velvet:

 hair slide:
  label: jewelry
-  threshold: 0.4
+  threshold: 0.6

 shower curtain:
  label: bathroom
-  threshold: 0.15
+  threshold: 0.6

 windsor tie:
  priority: -1
@ -770,18 +770,14 @@ jellyfish:
 sea anemone:
  categories:
    - animal
-    - coral
+    - water

 brain coral:
-  categories:
-    - animal
-    - coral
+  label: nature

 coral reef:
+  label: nature
  threshold: 0.6
-  categories:
-    - ocean
-    - water

 flatworm:
  categories:
@ -2222,9 +2218,10 @@ computer keyboard:
    - laptop

 confectionery:
+  label: shop
  categorie:
-    - sweets
-    - food
+    - store
+    - commercial

 container ship:
  label: ship
@ -2477,10 +2474,6 @@ jeep:
  categories:
    - car

-jigsaw puzzle:
-  categories:
-    - game
-
 ladle:
  categories:
    - kitchen
@ -2794,6 +2787,8 @@ power drill:
    - tool

 prayer rug:
+  threshold: 0.6
+  priority: -1
  categories:
    - religion
    - carpet
@ -3007,8 +3002,10 @@ submarine:
    - boat

 suspension bridge:
+  label: architecture
  categories:
    - bridge
+    - building

 swimming trunks:
  categories:
@ -3419,7 +3416,8 @@ buckeye:

 coral fungus:
  categories:
-    - coral
+    - plant
+    - mushroom

 agaric:
  categories:
@ -3537,3 +3535,15 @@ web site:
  categories:
    - sign
    - screenshot
+
+crossword puzzle:
+  threshold: 0.6
+  priority: -1
+  categories:
+    - game
+
+jigsaw puzzle:
+  threshold: 0.6
+  priority: -1
+  categories:
+    - game
--- a/internal/entity/file.go
+++ b/internal/entity/file.go
@ -22,6 +22,8 @@ type File struct {
 	FileType         string `gorm:"type:varchar(32)"`
 	FileMime         string `gorm:"type:varchar(64)"`
 	FilePrimary      bool
+	FileSidecar      bool
+	FileVideo        bool
 	FileMissing      bool
 	FileDuplicate    bool
 	FilePortrait     bool
--- a/internal/photoprism/filetypes.go
+++ b/internal/photoprism/filetypes.go
@ -6,25 +6,33 @@ import (
 	_ "image/png"
 )

+type FileType string
+
 const (
-	// FileTypeOther is an unkown file format.
-	FileTypeOther = "unknown"
-	// FileTypeYaml is a yaml file format.
-	FileTypeYaml = "yml"
-	// FileTypeJpeg is a jpeg file format.
-	FileTypeJpeg = "jpg"
-	// FileTypePng is a png file format.
-	FileTypePng = "png"
-	// FileTypeRaw is a raw file format.
-	FileTypeRaw = "raw"
-	// FileTypeXmp is an xmp file format.
-	FileTypeXmp = "xmp"
-	// FileTypeAae is an aae file format.
-	FileTypeAae = "aae"
-	// FileTypeMovie is a movie file format.
-	FileTypeMovie = "mov"
-	// FileTypeHEIF High Efficiency Image File Format
-	FileTypeHEIF = "heif" // High Efficiency Image File Format
+	// JPEG image file.
+	FileTypeJpeg FileType = "jpg"
+	// PNG image file.
+	FileTypePng FileType = "png"
+	// RAW image file.
+	FileTypeRaw FileType = "raw"
+	// High Efficiency Image File Format.
+	FileTypeHEIF FileType = "heif" // High Efficiency Image File Format
+	// Movie file.
+	FileTypeMovie FileType = "mov"
+	// Adobe XMP sidecar file (XML).
+	FileTypeXMP FileType = "xmp"
+	// Apple sidecar file (XML).
+	FileTypeAAE FileType = "aae"
+	// XML metadata / config / sidecar file.
+	FileTypeXML FileType = "xml"
+	// YAML metadata / config / sidecar file.
+	FileTypeYaml FileType = "yml"
+	// Text config / sidecar file.
+	FileTypeText FileType = "txt"
+	// Markdown text sidecar file.
+	FileTypeMarkdown FileType = "md"
+	// Unknown file format.
+	FileTypeOther FileType = "unknown"
 )

 const (
@ -33,7 +41,7 @@ const (
 )

 // FileExtensions lists all the available and supported image file formats.
-var FileExtensions = map[string]string{
+var FileExtensions = map[string]FileType{
 	".crw":  FileTypeRaw,
 	".cr2":  FileTypeRaw,
 	".nef":  FileTypeRaw,
@ -45,8 +53,8 @@ var FileExtensions = map[string]string{
 	".jpg":  FileTypeJpeg,
 	".thm":  FileTypeJpeg,
 	".jpeg": FileTypeJpeg,
-	".xmp":  FileTypeXmp,
-	".aae":  FileTypeAae,
+	".xmp":  FileTypeXMP,
+	".aae":  FileTypeAAE,
 	".heif": FileTypeHEIF,
 	".heic": FileTypeHEIF,
 	".3fr":  FileTypeRaw,
--- a/internal/photoprism/indexer_mediafile.go
+++ b/internal/photoprism/indexer_mediafile.go
@ -21,19 +21,18 @@ const (

 type IndexResult string

-func (i *Indexer) indexMediaFile(mediaFile *MediaFile, o IndexerOptions) IndexResult {
+func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
 	var photo entity.Photo
 	var file, primaryFile entity.File
-	var isPrimary = false
 	var exifData *Exif
 	var photoQuery, fileQuery *gorm.DB
 	var keywords []string

 	labels := Labels{}
-	fileBase := mediaFile.Basename()
-	filePath := mediaFile.RelativePath(i.originalsPath())
-	fileName := mediaFile.RelativeFilename(i.originalsPath())
-	fileHash := mediaFile.Hash()
+	fileBase := m.Basename()
+	filePath := m.RelativePath(i.originalsPath())
+	fileName := m.RelativeFilename(i.originalsPath())
+	fileHash := m.Hash()
 	fileChanged := true
 	fileExists := false
 	photoExists := false
@ -50,70 +49,80 @@ func (i *Indexer) indexMediaFile(mediaFile *MediaFile, o IndexerOptions) IndexRe
 	if !fileExists {
 		photoQuery = i.db.Unscoped().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase)

-		if photoQuery.Error != nil && mediaFile.HasTimeAndPlace() {
-			exifData, _ = mediaFile.Exif()
+		if photoQuery.Error != nil && m.HasTimeAndPlace() {
+			exifData, _ = m.Exif()
 			photoQuery = i.db.Unscoped().First(&photo, "photo_lat = ? AND photo_long = ? AND taken_at = ?", exifData.Lat, exifData.Long, exifData.TakenAt)
 		}
 	} else {
 		photoQuery = i.db.Unscoped().First(&photo, "id = ?", file.PhotoID)
 		fileChanged = file.FileHash != fileHash
-		isPrimary = file.FilePrimary
 	}

 	photoExists = photoQuery.Error == nil

-	if !fileChanged && photoExists && !photo.TakenAt.IsZero() && o.SkipUnchanged() {
+	if !fileChanged && photoExists && o.SkipUnchanged() {
 		return indexResultSkipped
 	}

+	if !file.FilePrimary {
+		if photoExists {
+			if q := i.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
+				file.FilePrimary = m.IsJpeg()
+			}
+		} else {
+			file.FilePrimary = m.IsJpeg()
+		}
+	}
+
+	if file.FilePrimary {
+		primaryFile = file
+	}
+
 	photo.PhotoPath = filePath
 	photo.PhotoName = fileBase

-	if isPrimary || !photoExists || photo.TakenAt.IsZero() {
-		if jpeg, err := mediaFile.Jpeg(); err == nil {
-			if fileChanged || o.UpdateLabels || o.UpdateTitle {
-				// Image classification labels
-				labels = i.classifyImage(jpeg)
-			}
+	if file.FilePrimary {
+		if fileChanged || o.UpdateLabels || o.UpdateTitle {
+			// Image classification labels
+			labels = i.classifyImage(m)
+		}

-			if fileChanged || o.UpdateExif {
-				// Read UpdateExif data
-				if exifData, err := jpeg.Exif(); err == nil {
-					photo.PhotoLat = exifData.Lat
-					photo.PhotoLong = exifData.Long
-					photo.TakenAt = exifData.TakenAt
-					photo.TakenAtLocal = exifData.TakenAtLocal
-					photo.TimeZone = exifData.TimeZone
-					photo.PhotoAltitude = exifData.Altitude
-					photo.PhotoArtist = exifData.Artist
+		if fileChanged || o.UpdateExif {
+			// Read UpdateExif data
+			if exifData, err := m.Exif(); err == nil {
+				photo.PhotoLat = exifData.Lat
+				photo.PhotoLong = exifData.Long
+				photo.TakenAt = exifData.TakenAt
+				photo.TakenAtLocal = exifData.TakenAtLocal
+				photo.TimeZone = exifData.TimeZone
+				photo.PhotoAltitude = exifData.Altitude
+				photo.PhotoArtist = exifData.Artist

-					if exifData.UUID != "" {
-						log.Debugf("index: photo uuid \"%s\"", exifData.UUID)
-						photo.PhotoUUID = exifData.UUID
-					} else {
-						log.Debug("index: no photo uuid in exif data")
-					}
+				if len(exifData.UUID) > 15 {
+					log.Debugf("index: file uuid \"%s\"", exifData.UUID)
+
+					file.FileUUID = exifData.UUID
 				}
 			}
-
-			if fileChanged || o.UpdateCamera {
-				// Set UpdateCamera, Lens, Focal Length and F Number
-				photo.Camera = entity.NewCamera(mediaFile.CameraModel(), mediaFile.CameraMake()).FirstOrCreate(i.db)
-				photo.Lens = entity.NewLens(mediaFile.LensModel(), mediaFile.LensMake()).FirstOrCreate(i.db)
-				photo.PhotoFocalLength = mediaFile.FocalLength()
-				photo.PhotoFNumber = mediaFile.FNumber()
-				photo.PhotoIso = mediaFile.Iso()
-				photo.PhotoExposure = mediaFile.Exposure()
-			}
 		}

-		if fileChanged || o.UpdateLocation || o.UpdateTitle {
-			keywords, labels = i.indexLocation(mediaFile, &photo, keywords, labels, fileChanged, o)
+		if fileChanged || o.UpdateCamera {
+			// Set UpdateCamera, Lens, Focal Length and F Number
+			photo.Camera = entity.NewCamera(m.CameraModel(), m.CameraMake()).FirstOrCreate(i.db)
+			photo.Lens = entity.NewLens(m.LensModel(), m.LensMake()).FirstOrCreate(i.db)
+			photo.PhotoFocalLength = m.FocalLength()
+			photo.PhotoFNumber = m.FNumber()
+			photo.PhotoIso = m.Iso()
+			photo.PhotoExposure = m.Exposure()
+		}
+
+		if fileChanged || o.UpdateKeywords || o.UpdateLocation || o.UpdateTitle {
+			keywords, labels = i.indexLocation(m, &photo, keywords, labels, fileChanged, o)
 		}

 		if (fileChanged || o.UpdateTitle) && photo.PhotoTitle == "" {
 			if len(labels) > 0 && labels[0].Priority >= -1 && labels[0].Uncertainty <= 85 && labels[0].Name != "" {
-				photo.PhotoTitle = fmt.Sprintf("%s / %s", util.Title(labels[0].Name), mediaFile.DateCreated().Format("2006"))
+				photo.PhotoTitle = fmt.Sprintf("%s / %s", util.Title(labels[0].Name), m.DateCreated().Format("2006"))
 			} else if !photo.TakenAtLocal.IsZero() {
 				var daytimeString string
 				hour := photo.TakenAtLocal.Hour()
@ -134,14 +143,11 @@ func (i *Indexer) indexMediaFile(mediaFile *MediaFile, o IndexerOptions) IndexRe

 			log.Infof("index: changed empty photo title to \"%s\"", photo.PhotoTitle)
 		}
-	}

-	// This should never happen
-	if photo.TakenAt.IsZero() || photo.TakenAtLocal.IsZero() {
-		photo.TakenAt = mediaFile.DateCreated()
-		photo.TakenAtLocal = photo.TakenAt
-
-		log.Warnf("index: %s has invalid date, set to \"%s\"", filepath.Base(mediaFile.Filename()), photo.TakenAt.Format("2006-01-02 15:04:05"))
+		if photo.TakenAt.IsZero() || photo.TakenAtLocal.IsZero() {
+			photo.TakenAt = m.DateCreated()
+			photo.TakenAtLocal = photo.TakenAt
+		}
 	}

 	if photoExists {
@ -163,35 +169,23 @@ func (i *Indexer) indexMediaFile(mediaFile *MediaFile, o IndexerOptions) IndexRe

 	if len(labels) > 0 {
 		log.Infof("index: adding labels %+v", labels)
-	}
-
-	if fileChanged || o.UpdateLabels {
 		i.addLabels(photo.ID, labels)
 	}

-	if result := i.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); result.Error != nil {
-		isPrimary = mediaFile.IsJpeg()
-	} else {
-		isPrimary = mediaFile.IsJpeg() && (fileName == primaryFile.FileName || fileHash == primaryFile.FileHash)
-	}
-
-	if (fileChanged || o.UpdateKeywords || o.UpdateTitle) && isPrimary {
-		photo.IndexKeywords(keywords, i.db)
-	}
-
 	file.PhotoID = photo.ID
 	file.PhotoUUID = photo.PhotoUUID
-	file.FilePrimary = isPrimary
+	file.FileSidecar = m.IsSidecar()
+	file.FileVideo = m.IsVideo()
 	file.FileMissing = false
 	file.FileName = fileName
 	file.FileHash = fileHash
-	file.FileType = mediaFile.Type()
-	file.FileMime = mediaFile.MimeType()
-	file.FileOrientation = mediaFile.Orientation()
+	file.FileType = string(m.Type())
+	file.FileMime = m.MimeType()
+	file.FileOrientation = m.Orientation()

-	if fileChanged || o.UpdateColors {
+	if m.IsJpeg() && (fileChanged || o.UpdateColors) {
 		// Color information
-		if p, err := mediaFile.Colors(i.thumbnailsPath()); err == nil {
+		if p, err := m.Colors(i.thumbnailsPath()); err == nil {
 			file.FileMainColor = p.MainColor.Name()
 			file.FileColors = p.Colors.Hex()
 			file.FileLuminance = p.Luminance.Hex()
@ -199,15 +193,20 @@ func (i *Indexer) indexMediaFile(mediaFile *MediaFile, o IndexerOptions) IndexRe
 		}
 	}

-	if fileChanged || o.UpdateSize {
-		if mediaFile.Width() > 0 && mediaFile.Height() > 0 {
-			file.FileWidth = mediaFile.Width()
-			file.FileHeight = mediaFile.Height()
-			file.FileAspectRatio = mediaFile.AspectRatio()
-			file.FilePortrait = mediaFile.Width() < mediaFile.Height()
+	if m.IsJpeg() && (fileChanged || o.UpdateSize) {
+		if m.Width() > 0 && m.Height() > 0 {
+			file.FileWidth = m.Width()
+			file.FileHeight = m.Height()
+			file.FileAspectRatio = m.AspectRatio()
+			file.FilePortrait = m.Width() < m.Height()
 		}
 	}

+	if file.FilePrimary && (fileChanged || o.UpdateKeywords || o.UpdateTitle) {
+		keywords = append(keywords, file.FileMainColor)
+		photo.IndexKeywords(keywords, i.db)
+	}
+
 	if fileQuery.Error == nil {
 		i.db.Unscoped().Save(&file)
 		return indexResultUpdated
@ -327,11 +326,13 @@ func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, keywo
 			labels = append(labels, NewLocationLabel(location.LocCountry, 0, -2))
 		}

-		if location.LocCategory != "" {
+		// TODO: Needs refactoring
+		if location.LocCategory != "" && location.LocCategory != "highway" && location.LocCategory != "tourism" {
 			labels = append(labels, NewLocationLabel(location.LocCategory, 0, -2))
 		}

-		if location.LocType != "" {
+		// TODO: Needs refactoring
+		if location.LocType != "" && location.LocType != "tertiary" && location.LocType != "attraction" {
 			labels = append(labels, NewLocationLabel(location.LocType, 0, -1))
 		}

--- a/internal/photoprism/mediafile.go
+++ b/internal/photoprism/mediafile.go
@ -23,7 +23,7 @@ type MediaFile struct {
 	dateCreated    time.Time
 	timeZone       string
 	hash           string
-	fileType       string
+	fileType       FileType
 	mimeType       string
 	perceptualHash string
 	width          int
@ -484,12 +484,12 @@ func (m *MediaFile) Copy(destinationFilename string) error {
 	return nil
 }

-// Extension returns the extension of a mediafile.
+// Extension returns the filename extension of this media file.
 func (m *MediaFile) Extension() string {
 	return strings.ToLower(filepath.Ext(m.filename))
 }

-// IsJpeg return true if the given mediafile is of mimetype Jpeg.
+// IsJpeg return true if this media file is a JPEG image.
 func (m *MediaFile) IsJpeg() bool {
 	// Don't import/use existing thumbnail files (we create our own)
 	if m.Extension() == ".thm" {
@ -500,30 +500,60 @@ func (m *MediaFile) IsJpeg() bool {
 }

 // Type returns the type of the media file.
-func (m *MediaFile) Type() string {
+func (m *MediaFile) Type() FileType {
 	return FileExtensions[m.Extension()]
 }

-// HasType checks whether a media file is of a given type.
-func (m *MediaFile) HasType(typeString string) bool {
-	if typeString == FileTypeJpeg {
+// HasType returns true if this media file is of a given type.
+func (m *MediaFile) HasType(t FileType) bool {
+	if t == FileTypeJpeg {
 		return m.IsJpeg()
 	}

-	return m.Type() == typeString
+	return m.Type() == t
 }

-// IsRaw check whether the given media file a RAW file.
+// IsRaw returns true if this media file a RAW file.
 func (m *MediaFile) IsRaw() bool {
 	return m.HasType(FileTypeRaw)
 }

-// IsHEIF check if a given media file is a High Efficiency Image File Format file.
+// IsHEIF returns true if this media file is a High Efficiency Image File Format file.
 func (m *MediaFile) IsHEIF() bool {
 	return m.HasType(FileTypeHEIF)
 }

-// IsPhoto checks if a media file is a photo / image.
+// IsSidecar returns true if this media file is a sidecar file (containing metadata).
+func (m *MediaFile) IsSidecar() bool {
+	switch m.Type() {
+	case FileTypeXMP:
+		return true
+	case FileTypeAAE:
+		return true
+	case FileTypeXML:
+		return true
+	case FileTypeYaml:
+		return true
+	case FileTypeText:
+		return true
+	case FileTypeMarkdown:
+		return true
+	default:
+		return false
+	}
+}
+
+// IsVideo returns true if this media file is a video file.
+func (m *MediaFile) IsVideo() bool {
+	switch m.Type() {
+	case FileTypeMovie:
+		return true
+	}
+
+	return false
+}
+
+// IsPhoto checks if this media file is a photo / image.
 func (m *MediaFile) IsPhoto() bool {
 	return m.IsJpeg() || m.IsRaw() || m.IsHEIF()
 }
--- a/internal/photoprism/thumbnails.go
+++ b/internal/photoprism/thumbnails.go
@ -147,7 +147,7 @@ func (m *MediaFile) Resample(path string, typeName string) (img image.Image, err
 	return imaging.Open(filename, imaging.AutoOrientation(true))
 }

-func ResampleOptions(opts ...ResampleOption) (method ResampleOption, filter imaging.ResampleFilter, format string) {
+func ResampleOptions(opts ...ResampleOption) (method ResampleOption, filter imaging.ResampleFilter, format FileType) {
 	method = ResampleFit
 	filter = imaging.Lanczos
 	format = FileTypeJpeg
@ -280,7 +280,7 @@ func CreateThumbnail(img image.Image, fileName string, width, height int, opts .

 	var saveOption imaging.EncodeOption

-	if filepath.Ext(fileName) == "."+FileTypePng {
+	if filepath.Ext(fileName) == "."+string(FileTypePng) {
 		saveOption = imaging.PNGCompressionLevel(png.DefaultCompression)
 	} else if width <= 150 && height <= 150 {
 		saveOption = imaging.JPEGQuality(JpegQualitySmall)
--- a/internal/repo/photos.go
+++ b/internal/repo/photos.go
@ -178,7 +178,7 @@ func (s *Repo) Photos(f form.PhotoSearch) (results []PhotoResult, err error) {
 			log.Infof("search: label \"%s\" not found, using fuzzy search", f.Query)

 			q = q.Joins("LEFT JOIN labels ON photos_labels.label_id = labels.id").
-				Where("labels.label_name LIKE ? OR keywords.keyword LIKE ? OR files.file_main_color = ?", likeString, likeString, lowerString)
+				Where("labels.label_name LIKE ? OR keywords.keyword LIKE ?", likeString, likeString)
 		} else {
 			labelIds = append(labelIds, label.ID)

@ -190,7 +190,7 @@ func (s *Repo) Photos(f form.PhotoSearch) (results []PhotoResult, err error) {

 			log.Infof("search: label \"%s\" includes %d categories", label.LabelName, len(labelIds))

-			q = q.Where("photos_labels.label_id IN (?) OR keywords.keyword LIKE ? OR files.file_main_color = ?", labelIds, likeString, lowerString)
+			q = q.Where("photos_labels.label_id IN (?) OR keywords.keyword LIKE ?", labelIds, likeString)
 		}
 	}