From d5d3fa8131aefaea35d0a7a465dbd960f33ccb02 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Thu, 2 Jan 2020 02:58:26 +0100 Subject: [PATCH] Indexer: Use goroutines and channels Signed-off-by: Michael Mayer --- internal/api/index.go | 22 +++++- internal/commands/config.go | 1 + internal/commands/index.go | 2 +- internal/photoprism/importer.go | 30 ++++---- internal/photoprism/indexer.go | 96 ++++++++++++++++++++---- internal/photoprism/indexer_mediafile.go | 74 +++++++++--------- internal/photoprism/indexer_test.go | 4 +- internal/photoprism/indexer_worker.go | 33 ++++++++ internal/photoprism/tensorflow.go | 8 ++ internal/server/routes.go | 3 +- 10 files changed, 198 insertions(+), 75 deletions(-) create mode 100644 internal/photoprism/indexer_worker.go diff --git a/internal/api/index.go b/internal/api/index.go index 59d7fc3af..f48b149bf 100644 --- a/internal/api/index.go +++ b/internal/api/index.go @@ -39,7 +39,7 @@ func initNsfwDetector(conf *config.Config) { } // POST /api/v1/index -func Index(router *gin.RouterGroup, conf *config.Config) { +func StartIndexing(router *gin.RouterGroup, conf *config.Config) { router.POST("/index", func(c *gin.Context) { if Unauthorized(c, conf) { c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized) @@ -73,9 +73,9 @@ func Index(router *gin.RouterGroup, conf *config.Config) { initIndexer(conf) if f.SkipUnchanged { - indexer.IndexOriginals(photoprism.IndexerOptionsNone()) + indexer.Start(photoprism.IndexerOptionsNone()) } else { - indexer.IndexOriginals(photoprism.IndexerOptionsAll()) + indexer.Start(photoprism.IndexerOptionsAll()) } elapsed := int(time.Since(start).Seconds()) @@ -87,3 +87,19 @@ func Index(router *gin.RouterGroup, conf *config.Config) { c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("indexing completed in %d s", elapsed)}) }) } + +// DELETE /api/v1/index +func CancelIndexing(router *gin.RouterGroup, conf *config.Config) { + router.DELETE("/index", func(c *gin.Context) { + if Unauthorized(c, conf) { + c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized) + return + } + + initIndexer(conf) + + indexer.Cancel() + + c.JSON(http.StatusOK, gin.H{"message": "indexing canceled"}) + }) +} diff --git a/internal/commands/config.go b/internal/commands/config.go index 707174198..dd0484f13 100644 --- a/internal/commands/config.go +++ b/internal/commands/config.go @@ -27,6 +27,7 @@ func configAction(ctx *cli.Context) error { fmt.Printf("twitter %s\n", conf.Twitter()) fmt.Printf("version %s\n", conf.Version()) fmt.Printf("copyright %s\n", conf.Copyright()) + fmt.Printf("workers %d\n", conf.Workers()) fmt.Printf("debug %t\n", conf.Debug()) fmt.Printf("read-only %t\n", conf.ReadOnly()) fmt.Printf("public %t\n", conf.Public()) diff --git a/internal/commands/index.go b/internal/commands/index.go index bf290cfed..c8ff5fb55 100644 --- a/internal/commands/index.go +++ b/internal/commands/index.go @@ -45,7 +45,7 @@ func indexAction(ctx *cli.Context) error { indexer := photoprism.NewIndexer(conf, tensorFlow, nsfwDetector) options := photoprism.IndexerOptionsAll() - files := indexer.IndexOriginals(options) + files := indexer.Start(options) elapsed := time.Since(start) diff --git a/internal/photoprism/importer.go b/internal/photoprism/importer.go index 6fd0c1f38..34b95bc6c 100644 --- a/internal/photoprism/importer.go +++ b/internal/photoprism/importer.go @@ -39,13 +39,13 @@ func NewImporter(conf *config.Config, indexer *Indexer, converter *Converter) *I return instance } -func (i *Importer) originalsPath() string { - return i.conf.OriginalsPath() +func (imp *Importer) originalsPath() string { + return imp.conf.OriginalsPath() } // ImportPhotosFromDirectory imports all the photos from a given directory path. // This function ignores errors. -func (i *Importer) ImportPhotosFromDirectory(importPath string) { +func (imp *Importer) ImportPhotosFromDirectory(importPath string) { var directories []string options := IndexerOptionsAll() @@ -63,7 +63,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { return nil } - if i.removeDotFiles && strings.HasPrefix(filepath.Base(filename), ".") { + if imp.removeDotFiles && strings.HasPrefix(filepath.Base(filename), ".") { if err := os.Remove(filename); err != nil { log.Errorf("could not remove \"%s\": %s", filename, err.Error()) } @@ -93,7 +93,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { for _, relatedMediaFile := range related.files { relativeFilename := relatedMediaFile.RelativeFilename(importPath) - if destinationFilename, err := i.DestinationFilename(related.main, relatedMediaFile); err == nil { + if destinationFilename, err := imp.DestinationFilename(related.main, relatedMediaFile); err == nil { if err := os.MkdirAll(path.Dir(destinationFilename), os.ModePerm); err != nil { log.Errorf("could not create directories: %s", err.Error()) } @@ -108,7 +108,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { if err := relatedMediaFile.Move(destinationFilename); err != nil { log.Errorf("could not move file to \"%s\": %s", destinationMainFilename, err.Error()) } - } else if i.removeExistingFiles { + } else if imp.removeExistingFiles { if err := relatedMediaFile.Remove(); err != nil { log.Errorf("could not delete file \"%s\": %s", relatedMediaFile.Filename(), err.Error()) } else { @@ -127,12 +127,12 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { } if importedMainFile.IsRaw() { - if _, err := i.converter.ConvertToJpeg(importedMainFile); err != nil { + if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil { log.Errorf("could not create jpeg from raw: %s", err) } } if importedMainFile.IsHEIF() { - if _, err := i.converter.ConvertToJpeg(importedMainFile); err != nil { + if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil { log.Errorf("could not create jpeg from heif: %s", err) } } @@ -140,12 +140,12 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { if jpg, err := importedMainFile.Jpeg(); err != nil { log.Error(err) } else { - if err := jpg.CreateDefaultThumbnails(i.conf.ThumbnailsPath(), false); err != nil { + if err := jpg.CreateDefaultThumbnails(imp.conf.ThumbnailsPath(), false); err != nil { log.Errorf("could not create default thumbnails: %s", err) } } - i.indexer.IndexRelated(importedMainFile, options) + imp.indexer.IndexRelated(importedMainFile, options) } return nil @@ -155,7 +155,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { return len(directories[i]) > len(directories[j]) }) - if i.removeEmptyDirectories { + if imp.removeEmptyDirectories { // Remove empty directories from import path for _, directory := range directories { if util.DirectoryIsEmpty(directory) { @@ -174,18 +174,18 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) { } // DestinationFilename get the destination of a media file. -func (i *Importer) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) { +func (imp *Importer) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) { fileName := mainFile.CanonicalName() fileExtension := mediaFile.Extension() dateCreated := mainFile.DateCreated() - if file, err := entity.FindFileByHash(i.conf.Db(), mediaFile.Hash()); err == nil { - existingFilename := i.conf.OriginalsPath() + string(os.PathSeparator) + file.FileName + if file, err := entity.FindFileByHash(imp.conf.Db(), mediaFile.Hash()); err == nil { + existingFilename := imp.conf.OriginalsPath() + string(os.PathSeparator) + file.FileName return existingFilename, fmt.Errorf("\"%s\" is identical to \"%s\" (%s)", mediaFile.Filename(), file.FileName, mediaFile.Hash()) } // Mon Jan 2 15:04:05 -0700 MST 2006 - pathName := i.originalsPath() + string(os.PathSeparator) + dateCreated.UTC().Format("2006/01") + pathName := imp.originalsPath() + string(os.PathSeparator) + dateCreated.UTC().Format("2006/01") iteration := 0 diff --git a/internal/photoprism/indexer.go b/internal/photoprism/indexer.go index 6c6b47c25..e2d98eeab 100644 --- a/internal/photoprism/indexer.go +++ b/internal/photoprism/indexer.go @@ -1,12 +1,15 @@ package photoprism import ( + "errors" "os" "path/filepath" "strings" + "sync" "github.com/jinzhu/gorm" "github.com/photoprism/photoprism/internal/config" + "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/nsfw" ) @@ -16,6 +19,8 @@ type Indexer struct { tensorFlow *TensorFlow nsfwDetector *nsfw.Detector db *gorm.DB + running bool + canceled bool } // NewIndexer returns a new indexer. @@ -31,55 +36,97 @@ func NewIndexer(conf *config.Config, tensorFlow *TensorFlow, nsfwDetector *nsfw. return i } -func (i *Indexer) originalsPath() string { - return i.conf.OriginalsPath() +func (ind *Indexer) originalsPath() string { + return ind.conf.OriginalsPath() } -func (i *Indexer) thumbnailsPath() string { - return i.conf.ThumbnailsPath() +func (ind *Indexer) thumbnailsPath() string { + return ind.conf.ThumbnailsPath() } // IndexRelated will index all mediafiles which has relate to a given mediafile. -func (i *Indexer) IndexRelated(mediaFile *MediaFile, o IndexerOptions) map[string]bool { +func (ind *Indexer) IndexRelated(mediaFile *MediaFile, o IndexerOptions) map[string]bool { indexed := make(map[string]bool) related, err := mediaFile.RelatedFiles() if err != nil { - log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(i.originalsPath()), err.Error()) + log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(ind.originalsPath()), err.Error()) return indexed } - mainIndexResult := i.indexMediaFile(related.main, o) + mainIndexResult := ind.indexMediaFile(related.main, o) indexed[related.main.Filename()] = true - log.Infof("index: %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(i.originalsPath())) + log.Infof("index: %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(ind.originalsPath())) for _, relatedMediaFile := range related.files { if indexed[relatedMediaFile.Filename()] { continue } - indexResult := i.indexMediaFile(relatedMediaFile, o) + indexResult := ind.indexMediaFile(relatedMediaFile, o) indexed[relatedMediaFile.Filename()] = true - log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(i.originalsPath())) + log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(ind.originalsPath())) } return indexed } -// IndexOriginals will index mediafiles in the originals directory. -func (i *Indexer) IndexOriginals(o IndexerOptions) map[string]bool { +// Cancel stops the current indexing operation. +func (ind *Indexer) Cancel() { + ind.canceled = true +} + +// Start will index mediafiles in the originals directory. +func (ind *Indexer) Start(o IndexerOptions) map[string]bool { indexed := make(map[string]bool) - err := filepath.Walk(i.originalsPath(), func(filename string, fileInfo os.FileInfo, err error) error { + if ind.running { + event.Error("indexer already running") + return indexed + } + + ind.running = true + ind.canceled = false + + defer func() { + ind.running = false + ind.canceled = false + }() + + if err := ind.tensorFlow.Init(); err != nil { + log.Errorf("index: %s", err.Error()) + + return indexed + } + + jobs := make(chan IndexJob) + + // Start a fixed number of goroutines to read and digest files. + var wg sync.WaitGroup + var numWorkers = ind.conf.Workers() + wg.Add(numWorkers) + for i := 0; i < numWorkers; i++ { + go func() { + indexerWorker(jobs) // HLc + wg.Done() + }() + } + + err := filepath.Walk(ind.originalsPath(), func(filename string, fileInfo os.FileInfo, err error) error { defer func() { if err := recover(); err != nil { log.Errorf("index: panic %s", err) } }() + + if ind.canceled { + return errors.New("indexing canceled") + } + if err != nil || indexed[filename] { return nil } @@ -94,15 +141,32 @@ func (i *Indexer) IndexOriginals(o IndexerOptions) map[string]bool { return nil } - for relatedFilename := range i.IndexRelated(mediaFile, o) { - indexed[relatedFilename] = true + related, err := mediaFile.RelatedFiles() + + if err != nil { + log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(ind.originalsPath()), err.Error()) + + return nil + } + + for _, f := range related.files { + indexed[f.Filename()] = true + } + + jobs <- IndexJob{ + r: related, + o: o, + i: ind, } return nil }) + close(jobs) + wg.Wait() + if err != nil { - log.Warn(err.Error()) + log.Error(err.Error()) } return indexed diff --git a/internal/photoprism/indexer_mediafile.go b/internal/photoprism/indexer_mediafile.go index 90aef116f..5776ca64d 100644 --- a/internal/photoprism/indexer_mediafile.go +++ b/internal/photoprism/indexer_mediafile.go @@ -23,7 +23,7 @@ const ( type IndexResult string -func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { +func (ind *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { start := time.Now() var photo entity.Photo @@ -35,8 +35,8 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { labels := Labels{} fileBase := m.Basename() - filePath := m.RelativePath(i.originalsPath()) - fileName := m.RelativeFilename(i.originalsPath()) + filePath := m.RelativePath(ind.originalsPath()) + fileName := m.RelativeFilename(ind.originalsPath()) fileHash := m.Hash() fileChanged := true fileExists := false @@ -48,18 +48,18 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { "baseName": filepath.Base(fileName), }) - fileQuery = i.db.Unscoped().First(&file, "file_hash = ? OR file_name = ?", fileHash, fileName) + fileQuery = ind.db.Unscoped().First(&file, "file_hash = ? OR file_name = ?", fileHash, fileName) fileExists = fileQuery.Error == nil if !fileExists { - photoQuery = i.db.Unscoped().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase) + photoQuery = ind.db.Unscoped().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase) if photoQuery.Error != nil && m.HasTimeAndPlace() { exifData, _ = m.Exif() - photoQuery = i.db.Unscoped().First(&photo, "photo_lat = ? AND photo_lng = ? AND taken_at = ?", exifData.Lat, exifData.Lng, exifData.TakenAt) + photoQuery = ind.db.Unscoped().First(&photo, "photo_lat = ? AND photo_lng = ? AND taken_at = ?", exifData.Lat, exifData.Lng, exifData.TakenAt) } } else { - photoQuery = i.db.Unscoped().First(&photo, "id = ?", file.PhotoID) + photoQuery = ind.db.Unscoped().First(&photo, "id = ?", file.PhotoID) fileChanged = file.FileHash != fileHash } @@ -71,7 +71,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if !file.FilePrimary { if photoExists { - if q := i.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil { + if q := ind.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil { file.FilePrimary = m.IsJpeg() } } else { @@ -89,7 +89,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if file.FilePrimary { if fileChanged || o.UpdateKeywords || o.UpdateLabels || o.UpdateTitle { // Image classification labels - labels, isNSFW = i.classifyImage(m) + labels, isNSFW = ind.classifyImage(m) photo.PhotoNSFW = isNSFW } @@ -114,8 +114,8 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if fileChanged || o.UpdateCamera { // Set UpdateCamera, Lens, Focal Length and F Number - photo.Camera = entity.NewCamera(m.CameraModel(), m.CameraMake()).FirstOrCreate(i.db) - photo.Lens = entity.NewLens(m.LensModel(), m.LensMake()).FirstOrCreate(i.db) + photo.Camera = entity.NewCamera(m.CameraModel(), m.CameraMake()).FirstOrCreate(ind.db) + photo.Lens = entity.NewLens(m.LensModel(), m.LensMake()).FirstOrCreate(ind.db) photo.PhotoFocalLength = m.FocalLength() photo.PhotoFNumber = m.FNumber() photo.PhotoIso = m.Iso() @@ -123,7 +123,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { } if fileChanged || o.UpdateKeywords || o.UpdateLocation || o.UpdateTitle { - locKeywords, locLabels := i.indexLocation(m, &photo, labels, fileChanged, o) + locKeywords, locLabels := ind.indexLocation(m, &photo, labels, fileChanged, o) keywords = append(keywords, locKeywords...) labels = append(labels, locLabels...) } @@ -164,10 +164,10 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if photoExists { // Estimate location if o.UpdateLocation && photo.NoLocation() { - i.estimateLocation(&photo) + ind.estimateLocation(&photo) } - if err := i.db.Unscoped().Save(&photo).Error; err != nil { + if err := ind.db.Unscoped().Save(&photo).Error; err != nil { log.Errorf("index: %s", err) return indexResultFailed } @@ -178,7 +178,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { photo.PhotoFavorite = false - if err := i.db.Create(&photo).Error; err != nil { + if err := ind.db.Create(&photo).Error; err != nil { log.Errorf("index: %s", err) return indexResultFailed } @@ -186,7 +186,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if len(labels) > 0 { log.Infof("index: adding labels %+v", labels) - i.addLabels(photo.ID, labels) + ind.addLabels(photo.ID, labels) } file.PhotoID = photo.ID @@ -202,7 +202,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if m.IsJpeg() && (fileChanged || o.UpdateColors) { // Color information - if p, err := m.Colors(i.thumbnailsPath()); err == nil { + if p, err := m.Colors(ind.thumbnailsPath()); err == nil { file.FileMainColor = p.MainColor.Name() file.FileColors = p.Colors.Hex() file.FileLuminance = p.Luminance.Hex() @@ -222,13 +222,13 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { if file.FilePrimary && (fileChanged || o.UpdateKeywords || o.UpdateTitle) { keywords = append(keywords, file.FileMainColor) keywords = append(keywords, labels.Keywords()...) - photo.IndexKeywords(keywords, i.db) + photo.IndexKeywords(keywords, ind.db) } if fileQuery.Error == nil { file.UpdatedIn = int64(time.Since(start)) - if err := i.db.Unscoped().Save(&file).Error; err != nil { + if err := ind.db.Unscoped().Save(&file).Error; err != nil { log.Errorf("index: %s", err) return indexResultFailed } @@ -238,7 +238,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { file.CreatedIn = int64(time.Since(start)) - if err := i.db.Create(&file).Error; err != nil { + if err := ind.db.Create(&file).Error; err != nil { log.Errorf("index: %s", err) return indexResultFailed } @@ -247,7 +247,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult { } // classifyImage returns all matching labels for a media file. -func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) { +func (ind *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) { start := time.Now() var thumbs []string @@ -261,14 +261,14 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) { var labels Labels for _, thumb := range thumbs { - filename, err := jpeg.Thumbnail(i.thumbnailsPath(), thumb) + filename, err := jpeg.Thumbnail(ind.thumbnailsPath(), thumb) if err != nil { log.Error(err) continue } - imageLabels, err := i.tensorFlow.LabelsFromFile(filename) + imageLabels, err := ind.tensorFlow.LabelsFromFile(filename) if err != nil { log.Error(err) @@ -278,10 +278,10 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) { labels = append(labels, imageLabels...) } - if filename, err := jpeg.Thumbnail(i.thumbnailsPath(), "fit_720"); err != nil { + if filename, err := jpeg.Thumbnail(ind.thumbnailsPath(), "fit_720"); err != nil { log.Error(err) } else { - if nsfwLabels, err := i.nsfwDetector.LabelsFromFile(filename); err != nil { + if nsfwLabels, err := ind.nsfwDetector.LabelsFromFile(filename); err != nil { log.Error(err) } else { log.Infof("nsfw: %+v", nsfwLabels) @@ -323,9 +323,9 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) { return results, isNSFW } -func (i *Indexer) addLabels(photoId uint, labels Labels) { +func (ind *Indexer) addLabels(photoId uint, labels Labels) { for _, label := range labels { - lm := entity.NewLabel(label.Name, label.Priority).FirstOrCreate(i.db) + lm := entity.NewLabel(label.Name, label.Priority).FirstOrCreate(ind.db) if lm.New && label.Priority >= 0 { event.Publish("count.labels", event.Data{ @@ -336,17 +336,17 @@ func (i *Indexer) addLabels(photoId uint, labels Labels) { if lm.LabelPriority != label.Priority { lm.LabelPriority = label.Priority - if err := i.db.Save(&lm).Error; err != nil { + if err := ind.db.Save(&lm).Error; err != nil { log.Errorf("index: %s", err) } } - plm := entity.NewPhotoLabel(photoId, lm.ID, label.Uncertainty, label.Source).FirstOrCreate(i.db) + plm := entity.NewPhotoLabel(photoId, lm.ID, label.Uncertainty, label.Source).FirstOrCreate(ind.db) // Add categories for _, category := range label.Categories { - sn := entity.NewLabel(category, -3).FirstOrCreate(i.db) - if err := i.db.Model(&lm).Association("LabelCategories").Append(sn).Error; err != nil { + sn := entity.NewLabel(category, -3).FirstOrCreate(ind.db) + if err := ind.db.Model(&lm).Association("LabelCategories").Append(sn).Error; err != nil { log.Errorf("index: %s", err) } } @@ -354,18 +354,18 @@ func (i *Indexer) addLabels(photoId uint, labels Labels) { if plm.LabelUncertainty > label.Uncertainty { plm.LabelUncertainty = label.Uncertainty plm.LabelSource = label.Source - if err := i.db.Save(&plm).Error; err != nil { + if err := ind.db.Save(&plm).Error; err != nil { log.Errorf("index: %s", err) } } } } -func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexerOptions) ([]string, Labels) { +func (ind *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexerOptions) ([]string, Labels) { var keywords []string if location, err := mediaFile.Location(); err == nil { - err := location.Find(i.db) + err := location.Find(ind.db) if err != nil { log.Error(err) @@ -384,7 +384,7 @@ func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, label photo.PlaceID = location.PlaceID photo.LocationEstimated = false - country := entity.NewCountry(location.CountryCode(), location.CountryName()).FirstOrCreate(i.db) + country := entity.NewCountry(location.CountryCode(), location.CountryName()).FirstOrCreate(ind.db) if country.New { event.Publish("count.countries", event.Data{ @@ -442,10 +442,10 @@ func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, label return keywords, labels } -func (i *Indexer) estimateLocation(photo *entity.Photo) { +func (ind *Indexer) estimateLocation(photo *entity.Photo) { var recentPhoto entity.Photo - if result := i.db.Unscoped().Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Place").First(&recentPhoto); result.Error == nil { + if result := ind.db.Unscoped().Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Place").First(&recentPhoto); result.Error == nil { if recentPhoto.HasPlace() { photo.Place = recentPhoto.Place photo.PhotoCountry = photo.Place.LocCountry diff --git a/internal/photoprism/indexer_test.go b/internal/photoprism/indexer_test.go index 1fa204c00..c9e5928e5 100644 --- a/internal/photoprism/indexer_test.go +++ b/internal/photoprism/indexer_test.go @@ -7,7 +7,7 @@ import ( "github.com/photoprism/photoprism/internal/nsfw" ) -func TestIndexer_IndexAll(t *testing.T) { +func TestIndexer_Start(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -29,5 +29,5 @@ func TestIndexer_IndexAll(t *testing.T) { options := IndexerOptionsAll() - indexer.IndexOriginals(options) + indexer.Start(options) } diff --git a/internal/photoprism/indexer_worker.go b/internal/photoprism/indexer_worker.go new file mode 100644 index 000000000..a0772bd17 --- /dev/null +++ b/internal/photoprism/indexer_worker.go @@ -0,0 +1,33 @@ +package photoprism + +type IndexJob struct { + r RelatedFiles + o IndexerOptions + i *Indexer +} + +func indexerWorker(jobs <-chan IndexJob) { + for job := range jobs { + indexed := make(map[string]bool) + r := job.r + o := job.o + i := job.i + + mainIndexResult := i.indexMediaFile(r.main, o) + indexed[r.main.Filename()] = true + + log.Infof("index: %s main %s file \"%s\"", mainIndexResult, r.main.Type(), r.main.RelativeFilename(i.originalsPath())) + + for _, relatedMediaFile := range r.files { + if indexed[relatedMediaFile.Filename()] { + continue + } + + indexResult := i.indexMediaFile(relatedMediaFile, o) + indexed[relatedMediaFile.Filename()] = true + + log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(i.originalsPath())) + } + } + +} diff --git a/internal/photoprism/tensorflow.go b/internal/photoprism/tensorflow.go index 1dc7f4a64..c224a464f 100644 --- a/internal/photoprism/tensorflow.go +++ b/internal/photoprism/tensorflow.go @@ -46,6 +46,14 @@ func NewTensorFlow(conf *config.Config) *TensorFlow { return &TensorFlow{conf: conf, modelName: "nasnet", modelTags: []string{"photoprism"}} } +func (t *TensorFlow) Init() (err error) { + if err := t.loadModel(); err != nil { + return err + } + + return t.loadLabelRules() +} + func (t *TensorFlow) loadLabelRules() (err error) { if len(t.labelRules) > 0 { return nil diff --git a/internal/server/routes.go b/internal/server/routes.go index ffa364ef8..cd509eeb1 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -41,7 +41,8 @@ func registerRoutes(router *gin.Engine, conf *config.Config) { api.Upload(v1, conf) api.Import(v1, conf) - api.Index(v1, conf) + api.StartIndexing(v1, conf) + api.CancelIndexing(v1, conf) api.BatchPhotosDelete(v1, conf) api.BatchPhotosPrivate(v1, conf)