luisgulo/photoprism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Indexer: Use goroutines and channels

Browse Source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-01-02 02:58:26 +01:00
parent a2db77af86
commit d5d3fa8131
10 changed files with 198 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
internal/api/index.go
View File

@ -39,7 +39,7 @@ func initNsfwDetector(conf *config.Config) {
}
// POST /api/v1/index
func Index(router *gin.RouterGroup, conf *config.Config) {
func StartIndexing(router *gin.RouterGroup, conf *config.Config) {
router.POST("/index", func(c *gin.Context) {
if Unauthorized(c, conf) {
c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized)
@ -73,9 +73,9 @@ func Index(router *gin.RouterGroup, conf *config.Config) {
initIndexer(conf)
if f.SkipUnchanged {
indexer.IndexOriginals(photoprism.IndexerOptionsNone())
indexer.Start(photoprism.IndexerOptionsNone())
} else {
indexer.IndexOriginals(photoprism.IndexerOptionsAll())
indexer.Start(photoprism.IndexerOptionsAll())
}
elapsed := int(time.Since(start).Seconds())
@ -87,3 +87,19 @@ func Index(router *gin.RouterGroup, conf *config.Config) {
c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("indexing completed in %d s", elapsed)})
})
}
// DELETE /api/v1/index
func CancelIndexing(router *gin.RouterGroup, conf *config.Config) {
router.DELETE("/index", func(c *gin.Context) {
if Unauthorized(c, conf) {
c.AbortWithStatusJSON(http.StatusUnauthorized, ErrUnauthorized)
return
}
initIndexer(conf)
indexer.Cancel()
c.JSON(http.StatusOK, gin.H{"message": "indexing canceled"})
})
}

1
internal/commands/config.go
View File

@ -27,6 +27,7 @@ func configAction(ctx *cli.Context) error {
fmt.Printf("twitter %s\n", conf.Twitter())
fmt.Printf("version %s\n", conf.Version())
fmt.Printf("copyright %s\n", conf.Copyright())
fmt.Printf("workers %d\n", conf.Workers())
fmt.Printf("debug %t\n", conf.Debug())
fmt.Printf("read-only %t\n", conf.ReadOnly())
fmt.Printf("public %t\n", conf.Public())

2
internal/commands/index.go
View File

@ -45,7 +45,7 @@ func indexAction(ctx *cli.Context) error {
indexer := photoprism.NewIndexer(conf, tensorFlow, nsfwDetector)
options := photoprism.IndexerOptionsAll()
files := indexer.IndexOriginals(options)
files := indexer.Start(options)
elapsed := time.Since(start)

30
internal/photoprism/importer.go
View File

@ -39,13 +39,13 @@ func NewImporter(conf *config.Config, indexer *Indexer, converter *Converter) *I
return instance
}
func (i *Importer) originalsPath() string {
return i.conf.OriginalsPath()
func (imp *Importer) originalsPath() string {
return imp.conf.OriginalsPath()
}
// ImportPhotosFromDirectory imports all the photos from a given directory path.
// This function ignores errors.
func (i *Importer) ImportPhotosFromDirectory(importPath string) {
func (imp *Importer) ImportPhotosFromDirectory(importPath string) {
var directories []string
options := IndexerOptionsAll()
@ -63,7 +63,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
return nil
}
if i.removeDotFiles && strings.HasPrefix(filepath.Base(filename), ".") {
if imp.removeDotFiles && strings.HasPrefix(filepath.Base(filename), ".") {
if err := os.Remove(filename); err != nil {
log.Errorf("could not remove \"%s\": %s", filename, err.Error())
}
@ -93,7 +93,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
for _, relatedMediaFile := range related.files {
relativeFilename := relatedMediaFile.RelativeFilename(importPath)
if destinationFilename, err := i.DestinationFilename(related.main, relatedMediaFile); err == nil {
if destinationFilename, err := imp.DestinationFilename(related.main, relatedMediaFile); err == nil {
if err := os.MkdirAll(path.Dir(destinationFilename), os.ModePerm); err != nil {
log.Errorf("could not create directories: %s", err.Error())
}
@ -108,7 +108,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
if err := relatedMediaFile.Move(destinationFilename); err != nil {
log.Errorf("could not move file to \"%s\": %s", destinationMainFilename, err.Error())
}
} else if i.removeExistingFiles {
} else if imp.removeExistingFiles {
if err := relatedMediaFile.Remove(); err != nil {
log.Errorf("could not delete file \"%s\": %s", relatedMediaFile.Filename(), err.Error())
} else {
@ -127,12 +127,12 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
}
if importedMainFile.IsRaw() {
if _, err := i.converter.ConvertToJpeg(importedMainFile); err != nil {
if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil {
log.Errorf("could not create jpeg from raw: %s", err)
}
}
if importedMainFile.IsHEIF() {
if _, err := i.converter.ConvertToJpeg(importedMainFile); err != nil {
if _, err := imp.converter.ConvertToJpeg(importedMainFile); err != nil {
log.Errorf("could not create jpeg from heif: %s", err)
}
}
@ -140,12 +140,12 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
if jpg, err := importedMainFile.Jpeg(); err != nil {
log.Error(err)
} else {
if err := jpg.CreateDefaultThumbnails(i.conf.ThumbnailsPath(), false); err != nil {
if err := jpg.CreateDefaultThumbnails(imp.conf.ThumbnailsPath(), false); err != nil {
log.Errorf("could not create default thumbnails: %s", err)
}
}
i.indexer.IndexRelated(importedMainFile, options)
imp.indexer.IndexRelated(importedMainFile, options)
}
return nil
@ -155,7 +155,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
return len(directories[i]) > len(directories[j])
})
if i.removeEmptyDirectories {
if imp.removeEmptyDirectories {
// Remove empty directories from import path
for _, directory := range directories {
if util.DirectoryIsEmpty(directory) {
@ -174,18 +174,18 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
}
// DestinationFilename get the destination of a media file.
func (i *Importer) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) {
func (imp *Importer) DestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) {
fileName := mainFile.CanonicalName()
fileExtension := mediaFile.Extension()
dateCreated := mainFile.DateCreated()
if file, err := entity.FindFileByHash(i.conf.Db(), mediaFile.Hash()); err == nil {
existingFilename := i.conf.OriginalsPath() + string(os.PathSeparator) + file.FileName
if file, err := entity.FindFileByHash(imp.conf.Db(), mediaFile.Hash()); err == nil {
existingFilename := imp.conf.OriginalsPath() + string(os.PathSeparator) + file.FileName
return existingFilename, fmt.Errorf("\"%s\" is identical to \"%s\" (%s)", mediaFile.Filename(), file.FileName, mediaFile.Hash())
}
// Mon Jan 2 15:04:05 -0700 MST 2006
pathName := i.originalsPath() + string(os.PathSeparator) + dateCreated.UTC().Format("2006/01")
pathName := imp.originalsPath() + string(os.PathSeparator) + dateCreated.UTC().Format("2006/01")
iteration := 0

96
internal/photoprism/indexer.go
View File

@ -1,12 +1,15 @@
package photoprism
import (
"errors"
"os"
"path/filepath"
"strings"
"sync"
"github.com/jinzhu/gorm"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/nsfw"
)
@ -16,6 +19,8 @@ type Indexer struct {
tensorFlow *TensorFlow
nsfwDetector *nsfw.Detector
db *gorm.DB
running bool
canceled bool
}
// NewIndexer returns a new indexer.
@ -31,55 +36,97 @@ func NewIndexer(conf *config.Config, tensorFlow *TensorFlow, nsfwDetector *nsfw.
return i
}
func (i *Indexer) originalsPath() string {
return i.conf.OriginalsPath()
func (ind *Indexer) originalsPath() string {
return ind.conf.OriginalsPath()
}
func (i *Indexer) thumbnailsPath() string {
return i.conf.ThumbnailsPath()
func (ind *Indexer) thumbnailsPath() string {
return ind.conf.ThumbnailsPath()
}
// IndexRelated will index all mediafiles which has relate to a given mediafile.
func (i *Indexer) IndexRelated(mediaFile *MediaFile, o IndexerOptions) map[string]bool {
func (ind *Indexer) IndexRelated(mediaFile *MediaFile, o IndexerOptions) map[string]bool {
indexed := make(map[string]bool)
related, err := mediaFile.RelatedFiles()
if err != nil {
log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(i.originalsPath()), err.Error())
log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(ind.originalsPath()), err.Error())
return indexed
}
mainIndexResult := i.indexMediaFile(related.main, o)
mainIndexResult := ind.indexMediaFile(related.main, o)
indexed[related.main.Filename()] = true
log.Infof("index: %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(i.originalsPath()))
log.Infof("index: %s main %s file \"%s\"", mainIndexResult, related.main.Type(), related.main.RelativeFilename(ind.originalsPath()))
for _, relatedMediaFile := range related.files {
if indexed[relatedMediaFile.Filename()] {
continue
}
indexResult := i.indexMediaFile(relatedMediaFile, o)
indexResult := ind.indexMediaFile(relatedMediaFile, o)
indexed[relatedMediaFile.Filename()] = true
log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(i.originalsPath()))
log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(ind.originalsPath()))
}
return indexed
}
// IndexOriginals will index mediafiles in the originals directory.
func (i *Indexer) IndexOriginals(o IndexerOptions) map[string]bool {
// Cancel stops the current indexing operation.
func (ind *Indexer) Cancel() {
ind.canceled = true
}
// Start will index mediafiles in the originals directory.
func (ind *Indexer) Start(o IndexerOptions) map[string]bool {
indexed := make(map[string]bool)
err := filepath.Walk(i.originalsPath(), func(filename string, fileInfo os.FileInfo, err error) error {
if ind.running {
event.Error("indexer already running")
return indexed
}
ind.running = true
ind.canceled = false
defer func() {
ind.running = false
ind.canceled = false
}()
if err := ind.tensorFlow.Init(); err != nil {
log.Errorf("index: %s", err.Error())
return indexed
}
jobs := make(chan IndexJob)
// Start a fixed number of goroutines to read and digest files.
var wg sync.WaitGroup
var numWorkers = ind.conf.Workers()
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
indexerWorker(jobs) // HLc
wg.Done()
}()
}
err := filepath.Walk(ind.originalsPath(), func(filename string, fileInfo os.FileInfo, err error) error {
defer func() {
if err := recover(); err != nil {
log.Errorf("index: panic %s", err)
}
}()
if ind.canceled {
return errors.New("indexing canceled")
}
if err != nil || indexed[filename] {
return nil
}
@ -94,15 +141,32 @@ func (i *Indexer) IndexOriginals(o IndexerOptions) map[string]bool {
return nil
}
for relatedFilename := range i.IndexRelated(mediaFile, o) {
indexed[relatedFilename] = true
related, err := mediaFile.RelatedFiles()
if err != nil {
log.Warnf("could not index \"%s\": %s", mediaFile.RelativeFilename(ind.originalsPath()), err.Error())
return nil
}
for _, f := range related.files {
indexed[f.Filename()] = true
}
jobs <- IndexJob{
r: related,
o: o,
i: ind,
}
return nil
})
close(jobs)
wg.Wait()
if err != nil {
log.Warn(err.Error())
log.Error(err.Error())
}
return indexed

74
internal/photoprism/indexer_mediafile.go
View File

@ -23,7 +23,7 @@ const (
type IndexResult string
func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
func (ind *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
start := time.Now()
var photo entity.Photo
@ -35,8 +35,8 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
labels := Labels{}
fileBase := m.Basename()
filePath := m.RelativePath(i.originalsPath())
fileName := m.RelativeFilename(i.originalsPath())
filePath := m.RelativePath(ind.originalsPath())
fileName := m.RelativeFilename(ind.originalsPath())
fileHash := m.Hash()
fileChanged := true
fileExists := false
@ -48,18 +48,18 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
"baseName": filepath.Base(fileName),
})
fileQuery = i.db.Unscoped().First(&file, "file_hash = ? OR file_name = ?", fileHash, fileName)
fileQuery = ind.db.Unscoped().First(&file, "file_hash = ? OR file_name = ?", fileHash, fileName)
fileExists = fileQuery.Error == nil
if !fileExists {
photoQuery = i.db.Unscoped().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase)
photoQuery = ind.db.Unscoped().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase)
if photoQuery.Error != nil && m.HasTimeAndPlace() {
exifData, _ = m.Exif()
photoQuery = i.db.Unscoped().First(&photo, "photo_lat = ? AND photo_lng = ? AND taken_at = ?", exifData.Lat, exifData.Lng, exifData.TakenAt)
photoQuery = ind.db.Unscoped().First(&photo, "photo_lat = ? AND photo_lng = ? AND taken_at = ?", exifData.Lat, exifData.Lng, exifData.TakenAt)
}
} else {
photoQuery = i.db.Unscoped().First(&photo, "id = ?", file.PhotoID)
photoQuery = ind.db.Unscoped().First(&photo, "id = ?", file.PhotoID)
fileChanged = file.FileHash != fileHash
}
@ -71,7 +71,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if !file.FilePrimary {
if photoExists {
if q := i.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
if q := ind.db.Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
file.FilePrimary = m.IsJpeg()
}
} else {
@ -89,7 +89,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if file.FilePrimary {
if fileChanged || o.UpdateKeywords || o.UpdateLabels || o.UpdateTitle {
// Image classification labels
labels, isNSFW = i.classifyImage(m)
labels, isNSFW = ind.classifyImage(m)
photo.PhotoNSFW = isNSFW
}
@ -114,8 +114,8 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if fileChanged || o.UpdateCamera {
// Set UpdateCamera, Lens, Focal Length and F Number
photo.Camera = entity.NewCamera(m.CameraModel(), m.CameraMake()).FirstOrCreate(i.db)
photo.Lens = entity.NewLens(m.LensModel(), m.LensMake()).FirstOrCreate(i.db)
photo.Camera = entity.NewCamera(m.CameraModel(), m.CameraMake()).FirstOrCreate(ind.db)
photo.Lens = entity.NewLens(m.LensModel(), m.LensMake()).FirstOrCreate(ind.db)
photo.PhotoFocalLength = m.FocalLength()
photo.PhotoFNumber = m.FNumber()
photo.PhotoIso = m.Iso()
@ -123,7 +123,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
}
if fileChanged || o.UpdateKeywords || o.UpdateLocation || o.UpdateTitle {
locKeywords, locLabels := i.indexLocation(m, &photo, labels, fileChanged, o)
locKeywords, locLabels := ind.indexLocation(m, &photo, labels, fileChanged, o)
keywords = append(keywords, locKeywords...)
labels = append(labels, locLabels...)
}
@ -164,10 +164,10 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if photoExists {
// Estimate location
if o.UpdateLocation && photo.NoLocation() {
i.estimateLocation(&photo)
ind.estimateLocation(&photo)
}
if err := i.db.Unscoped().Save(&photo).Error; err != nil {
if err := ind.db.Unscoped().Save(&photo).Error; err != nil {
log.Errorf("index: %s", err)
return indexResultFailed
}
@ -178,7 +178,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
photo.PhotoFavorite = false
if err := i.db.Create(&photo).Error; err != nil {
if err := ind.db.Create(&photo).Error; err != nil {
log.Errorf("index: %s", err)
return indexResultFailed
}
@ -186,7 +186,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if len(labels) > 0 {
log.Infof("index: adding labels %+v", labels)
i.addLabels(photo.ID, labels)
ind.addLabels(photo.ID, labels)
}
file.PhotoID = photo.ID
@ -202,7 +202,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if m.IsJpeg() && (fileChanged || o.UpdateColors) {
// Color information
if p, err := m.Colors(i.thumbnailsPath()); err == nil {
if p, err := m.Colors(ind.thumbnailsPath()); err == nil {
file.FileMainColor = p.MainColor.Name()
file.FileColors = p.Colors.Hex()
file.FileLuminance = p.Luminance.Hex()
@ -222,13 +222,13 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
if file.FilePrimary && (fileChanged || o.UpdateKeywords || o.UpdateTitle) {
keywords = append(keywords, file.FileMainColor)
keywords = append(keywords, labels.Keywords()...)
photo.IndexKeywords(keywords, i.db)
photo.IndexKeywords(keywords, ind.db)
}
if fileQuery.Error == nil {
file.UpdatedIn = int64(time.Since(start))
if err := i.db.Unscoped().Save(&file).Error; err != nil {
if err := ind.db.Unscoped().Save(&file).Error; err != nil {
log.Errorf("index: %s", err)
return indexResultFailed
}
@ -238,7 +238,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
file.CreatedIn = int64(time.Since(start))
if err := i.db.Create(&file).Error; err != nil {
if err := ind.db.Create(&file).Error; err != nil {
log.Errorf("index: %s", err)
return indexResultFailed
}
@ -247,7 +247,7 @@ func (i *Indexer) indexMediaFile(m *MediaFile, o IndexerOptions) IndexResult {
}
// classifyImage returns all matching labels for a media file.
func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
func (ind *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
start := time.Now()
var thumbs []string
@ -261,14 +261,14 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
var labels Labels
for _, thumb := range thumbs {
filename, err := jpeg.Thumbnail(i.thumbnailsPath(), thumb)
filename, err := jpeg.Thumbnail(ind.thumbnailsPath(), thumb)
if err != nil {
log.Error(err)
continue
}
imageLabels, err := i.tensorFlow.LabelsFromFile(filename)
imageLabels, err := ind.tensorFlow.LabelsFromFile(filename)
if err != nil {
log.Error(err)
@ -278,10 +278,10 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
labels = append(labels, imageLabels...)
}
if filename, err := jpeg.Thumbnail(i.thumbnailsPath(), "fit_720"); err != nil {
if filename, err := jpeg.Thumbnail(ind.thumbnailsPath(), "fit_720"); err != nil {
log.Error(err)
} else {
if nsfwLabels, err := i.nsfwDetector.LabelsFromFile(filename); err != nil {
if nsfwLabels, err := ind.nsfwDetector.LabelsFromFile(filename); err != nil {
log.Error(err)
} else {
log.Infof("nsfw: %+v", nsfwLabels)
@ -323,9 +323,9 @@ func (i *Indexer) classifyImage(jpeg *MediaFile) (results Labels, isNSFW bool) {
return results, isNSFW
}
func (i *Indexer) addLabels(photoId uint, labels Labels) {
func (ind *Indexer) addLabels(photoId uint, labels Labels) {
for _, label := range labels {
lm := entity.NewLabel(label.Name, label.Priority).FirstOrCreate(i.db)
lm := entity.NewLabel(label.Name, label.Priority).FirstOrCreate(ind.db)
if lm.New && label.Priority >= 0 {
event.Publish("count.labels", event.Data{
@ -336,17 +336,17 @@ func (i *Indexer) addLabels(photoId uint, labels Labels) {
if lm.LabelPriority != label.Priority {
lm.LabelPriority = label.Priority
if err := i.db.Save(&lm).Error; err != nil {
if err := ind.db.Save(&lm).Error; err != nil {
log.Errorf("index: %s", err)
}
}
plm := entity.NewPhotoLabel(photoId, lm.ID, label.Uncertainty, label.Source).FirstOrCreate(i.db)
plm := entity.NewPhotoLabel(photoId, lm.ID, label.Uncertainty, label.Source).FirstOrCreate(ind.db)
// Add categories
for _, category := range label.Categories {
sn := entity.NewLabel(category, -3).FirstOrCreate(i.db)
if err := i.db.Model(&lm).Association("LabelCategories").Append(sn).Error; err != nil {
sn := entity.NewLabel(category, -3).FirstOrCreate(ind.db)
if err := ind.db.Model(&lm).Association("LabelCategories").Append(sn).Error; err != nil {
log.Errorf("index: %s", err)
}
}
@ -354,18 +354,18 @@ func (i *Indexer) addLabels(photoId uint, labels Labels) {
if plm.LabelUncertainty > label.Uncertainty {
plm.LabelUncertainty = label.Uncertainty
plm.LabelSource = label.Source
if err := i.db.Save(&plm).Error; err != nil {
if err := ind.db.Save(&plm).Error; err != nil {
log.Errorf("index: %s", err)
}
}
}
}
func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexerOptions) ([]string, Labels) {
func (ind *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, labels Labels, fileChanged bool, o IndexerOptions) ([]string, Labels) {
var keywords []string
if location, err := mediaFile.Location(); err == nil {
err := location.Find(i.db)
err := location.Find(ind.db)
if err != nil {
log.Error(err)
@ -384,7 +384,7 @@ func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, label
photo.PlaceID = location.PlaceID
photo.LocationEstimated = false
country := entity.NewCountry(location.CountryCode(), location.CountryName()).FirstOrCreate(i.db)
country := entity.NewCountry(location.CountryCode(), location.CountryName()).FirstOrCreate(ind.db)
if country.New {
event.Publish("count.countries", event.Data{
@ -442,10 +442,10 @@ func (i *Indexer) indexLocation(mediaFile *MediaFile, photo *entity.Photo, label
return keywords, labels
}
func (i *Indexer) estimateLocation(photo *entity.Photo) {
func (ind *Indexer) estimateLocation(photo *entity.Photo) {
var recentPhoto entity.Photo
if result := i.db.Unscoped().Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Place").First(&recentPhoto); result.Error == nil {
if result := ind.db.Unscoped().Order(gorm.Expr("ABS(DATEDIFF(taken_at, ?)) ASC", photo.TakenAt)).Preload("Place").First(&recentPhoto); result.Error == nil {
if recentPhoto.HasPlace() {
photo.Place = recentPhoto.Place
photo.PhotoCountry = photo.Place.LocCountry

4
internal/photoprism/indexer_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/photoprism/photoprism/internal/nsfw"
)
func TestIndexer_IndexAll(t *testing.T) {
func TestIndexer_Start(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
@ -29,5 +29,5 @@ func TestIndexer_IndexAll(t *testing.T) {
options := IndexerOptionsAll()
indexer.IndexOriginals(options)
indexer.Start(options)
}

33
internal/photoprism/indexer_worker.go Normal file
View File

@ -0,0 +1,33 @@
package photoprism
type IndexJob struct {
r RelatedFiles
o IndexerOptions
i *Indexer
}
func indexerWorker(jobs <-chan IndexJob) {
for job := range jobs {
indexed := make(map[string]bool)
r := job.r
o := job.o
i := job.i
mainIndexResult := i.indexMediaFile(r.main, o)
indexed[r.main.Filename()] = true
log.Infof("index: %s main %s file \"%s\"", mainIndexResult, r.main.Type(), r.main.RelativeFilename(i.originalsPath()))
for _, relatedMediaFile := range r.files {
if indexed[relatedMediaFile.Filename()] {
continue
}
indexResult := i.indexMediaFile(relatedMediaFile, o)
indexed[relatedMediaFile.Filename()] = true
log.Infof("index: %s related %s file \"%s\"", indexResult, relatedMediaFile.Type(), relatedMediaFile.RelativeFilename(i.originalsPath()))
}
}
}

8
internal/photoprism/tensorflow.go
View File

@ -46,6 +46,14 @@ func NewTensorFlow(conf *config.Config) *TensorFlow {
return &TensorFlow{conf: conf, modelName: "nasnet", modelTags: []string{"photoprism"}}
}
func (t *TensorFlow) Init() (err error) {
if err := t.loadModel(); err != nil {
return err
}
return t.loadLabelRules()
}
func (t *TensorFlow) loadLabelRules() (err error) {
if len(t.labelRules) > 0 {
return nil

3
internal/server/routes.go
View File

@ -41,7 +41,8 @@ func registerRoutes(router *gin.Engine, conf *config.Config) {
api.Upload(v1, conf)
api.Import(v1, conf)
api.Index(v1, conf)
api.StartIndexing(v1, conf)
api.CancelIndexing(v1, conf)
api.BatchPhotosDelete(v1, conf)
api.BatchPhotosPrivate(v1, conf)