luisgulo/photoprism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Importer now instantly converts and indexes files

Browse Source
This commit is contained in:
Michael Mayer 2018-09-18 18:36:12 +02:00
parent de494fb8f8
commit 0148096c86
7 changed files with 194 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
cmd/photoprism/photoprism.go
View File

@ -46,21 +46,21 @@ func main() {
Usage: "Starts web server",
Flags: []cli.Flag{
cli.IntFlag{
Name: "server-port, p",
Usage: "HTTP server port",
Value: 80,
Name: "server-port, p",
Usage: "HTTP server port",
Value: 80,
EnvVar: "PHOTOPRISM_SERVER_PORT",
},
cli.StringFlag{
Name: "server-host, h",
Usage: "HTTP server host",
Value: "",
Name: "server-host, h",
Usage: "HTTP server host",
Value: "",
EnvVar: "PHOTOPRISM_SERVER_HOST",
},
cli.StringFlag{
Name: "server-mode, m",
Usage: "debug, release or test",
Value: "",
Name: "server-mode, m",
Usage: "debug, release or test",
Value: "",
EnvVar: "PHOTOPRISM_SERVER_MODE",
},
},
@ -131,7 +131,9 @@ func main() {
indexer := photoprism.NewIndexer(conf.OriginalsPath, tensorFlow, conf.GetDb())
importer := photoprism.NewImporter(conf.OriginalsPath, indexer)
converter := photoprism.NewConverter(conf.DarktableCli)
importer := photoprism.NewImporter(conf.OriginalsPath, indexer, converter)
importer.ImportPhotosFromDirectory(conf.ImportPath)
@ -305,62 +307,62 @@ func main() {
var globalCliFlags = []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "run in debug mode",
Name: "debug",
Usage: "run in debug mode",
EnvVar: "PHOTOPRISM_DEBUG",
},
cli.StringFlag{
Name: "config-file, c",
Usage: "load configuration from `FILENAME`",
Value: "/etc/photoprism/photoprism.yml",
Name: "config-file, c",
Usage: "load configuration from `FILENAME`",
Value: "/etc/photoprism/photoprism.yml",
EnvVar: "PHOTOPRISM_CONFIG_FILE",
},
cli.StringFlag{
Name: "darktable-cli",
Usage: "darktable command-line executable `FILENAME`",
Value: "/usr/bin/darktable-cli",
Name: "darktable-cli",
Usage: "darktable command-line executable `FILENAME`",
Value: "/usr/bin/darktable-cli",
EnvVar: "PHOTOPRISM_DARKTABLE_CLI",
},
cli.StringFlag{
Name: "originals-path",
Usage: "originals `PATH`",
Value: "/var/photoprism/photos/originals",
Name: "originals-path",
Usage: "originals `PATH`",
Value: "/var/photoprism/photos/originals",
EnvVar: "PHOTOPRISM_ORIGINALS_PATH",
},
cli.StringFlag{
Name: "thumbnails-path",
Usage: "thumbnails `PATH`",
Value: "/var/photoprism/photos/thumbnails",
Name: "thumbnails-path",
Usage: "thumbnails `PATH`",
Value: "/var/photoprism/photos/thumbnails",
EnvVar: "PHOTOPRISM_THUMBNAILS_PATH",
},
cli.StringFlag{
Name: "import-path",
Usage: "import `PATH`",
Value: "/var/photoprism/photos/import",
Name: "import-path",
Usage: "import `PATH`",
Value: "/var/photoprism/photos/import",
EnvVar: "PHOTOPRISM_IMPORT_PATH",
},
cli.StringFlag{
Name: "export-path",
Usage: "export `PATH`",
Value: "/var/photoprism/photos/export",
Name: "export-path",
Usage: "export `PATH`",
Value: "/var/photoprism/photos/export",
EnvVar: "PHOTOPRISM_EXPORT_PATH",
},
cli.StringFlag{
Name: "assets-path",
Usage: "assets `PATH`",
Value: "/var/photoprism",
Name: "assets-path",
Usage: "assets `PATH`",
Value: "/var/photoprism",
EnvVar: "PHOTOPRISM_ASSETS_PATH",
},
cli.StringFlag{
Name: "database-driver",
Usage: "database `DRIVER` (mysql, mssql, postgres or sqlite)",
Value: "mysql",
Name: "database-driver",
Usage: "database `DRIVER` (mysql, mssql, postgres or sqlite)",
Value: "mysql",
EnvVar: "PHOTOPRISM_DATABASE_DRIVER",
},
cli.StringFlag{
Name: "database-dsn",
Usage: "database data source name (`DSN`)",
Value: "photoprism:photoprism@tcp(localhost:3306)/photoprism",
Name: "database-dsn",
Usage: "database data source name (`DSN`)",
Value: "photoprism:photoprism@tcp(localhost:3306)/photoprism",
EnvVar: "PHOTOPRISM_DATABASE_DSN",
},
}

10
internal/photoprism/converter.go
View File

@ -1,7 +1,7 @@
package photoprism
import (
"errors"
"fmt"
"log"
"os"
"os/exec"
@ -54,16 +54,14 @@ func (c *Converter) ConvertAll(path string) {
func (c *Converter) ConvertToJpeg(image *MediaFile) (*MediaFile, error) {
if !image.Exists() {
return nil, errors.New("can not convert, file does not exist")
return nil, fmt.Errorf("can not convert, file does not exist: %s", image.GetFilename())
}
if image.IsJpeg() {
return image, nil
}
extension := image.GetExtension()
baseFilename := image.filename[0 : len(image.filename)-len(extension)]
baseFilename := image.GetCanonicalNameFromFileWithDirectory()
jpegFilename := baseFilename + ".jpg"
@ -73,7 +71,7 @@ func (c *Converter) ConvertToJpeg(image *MediaFile) (*MediaFile, error) {
return mediaFile, nil
}
log.Printf("Converting %s to %s \n", image.filename, jpegFilename)
log.Printf("Converting \"%s\" to \"%s\"\n", image.filename, jpegFilename)
xmpFilename := baseFilename + ".xmp"

55
internal/photoprism/importer.go
View File

@ -14,15 +14,17 @@ import (
type Importer struct {
originalsPath string
indexer *Indexer
converter *Converter
removeDotFiles bool
removeExistingFiles bool
removeEmptyDirectories bool
}
func NewImporter(originalsPath string, indexer *Indexer) *Importer {
func NewImporter(originalsPath string, indexer *Indexer, converter *Converter) *Importer {
instance := &Importer{
originalsPath: originalsPath,
indexer: indexer,
converter: converter,
removeDotFiles: true,
removeExistingFiles: true,
removeEmptyDirectories: true,
@ -35,6 +37,8 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
var directories []string
err := filepath.Walk(importPath, func(filename string, fileInfo os.FileInfo, err error) error {
var destinationMainFilename string
if err != nil {
// log.Print(err.Error())
return nil
@ -49,6 +53,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
if i.removeDotFiles && strings.HasPrefix(filepath.Base(filename), ".") {
os.Remove(filename)
return nil
}
@ -58,20 +63,48 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
return nil
}
relatedFiles, masterFile, _ := mediaFile.GetRelatedFiles()
relatedFiles, mainFile, err := mediaFile.GetRelatedFiles()
if err != nil {
log.Printf("Could not import \"%s\": %s", mediaFile.GetRelativeFilename(importPath), err.Error())
return nil
}
for _, relatedMediaFile := range relatedFiles {
if destinationFilename, err := i.GetDestinationFilename(masterFile, relatedMediaFile); err == nil {
if destinationFilename, err := i.GetDestinationFilename(mainFile, relatedMediaFile); err == nil {
os.MkdirAll(path.Dir(destinationFilename), os.ModePerm)
log.Printf("Moving file %s to %s", relatedMediaFile.GetFilename(), destinationFilename)
if mainFile.HasSameFilename(relatedMediaFile) {
destinationMainFilename = destinationFilename
log.Printf("Moving main %s file \"%s\" to \"%s\"", relatedMediaFile.GetType(), relatedMediaFile.GetRelativeFilename(importPath), destinationFilename)
} else {
log.Printf("Moving related %s file \"%s\" to \"%s\"", relatedMediaFile.GetType(), relatedMediaFile.GetRelativeFilename(importPath), destinationFilename)
}
relatedMediaFile.Move(destinationFilename)
i.indexer.IndexMediaFile(relatedMediaFile)
} else if i.removeExistingFiles {
relatedMediaFile.Remove()
log.Printf("Deleted %s (already exists)", relatedMediaFile.GetFilename())
log.Printf("Deleted \"%s\" (already exists)", relatedMediaFile.GetRelativeFilename(importPath))
}
}
if destinationMainFilename != "" {
importedMainFile, err := NewMediaFile(destinationMainFilename)
if err != nil {
log.Printf("Could not index \"%s\" after import: %s", destinationMainFilename, err.Error())
return nil
}
if importedMainFile.IsRaw() {
i.converter.ConvertToJpeg(importedMainFile)
}
i.indexer.IndexRelated(importedMainFile)
}
return nil
})
@ -84,7 +117,7 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
for _, directory := range directories {
if directoryIsEmpty(directory) {
os.Remove(directory)
log.Printf("Deleted empty directory %s", directory)
log.Printf("Deleted empty directory \"%s\"", directory)
}
}
}
@ -94,15 +127,15 @@ func (i *Importer) ImportPhotosFromDirectory(importPath string) {
}
}
func (i *Importer) GetDestinationFilename(masterFile *MediaFile, mediaFile *MediaFile) (string, error) {
canonicalName := masterFile.GetCanonicalName()
func (i *Importer) GetDestinationFilename(mainFile *MediaFile, mediaFile *MediaFile) (string, error) {
canonicalName := mainFile.GetCanonicalName()
fileExtension := mediaFile.GetExtension()
dateCreated := masterFile.GetDateCreated()
dateCreated := mainFile.GetDateCreated()
// Mon Jan 2 15:04:05 -0700 MST 2006
pathName := i.originalsPath + "/" + dateCreated.UTC().Format("2006/01")
iteration := 1
iteration := 0
result := pathName + "/" + canonicalName + fileExtension

12
internal/photoprism/importer_test.go
View File

@ -12,7 +12,9 @@ func TestNewImporter(t *testing.T) {
indexer := NewIndexer(conf.OriginalsPath, tensorFlow, conf.GetDb())
importer := NewImporter(conf.OriginalsPath, indexer)
converter := NewConverter(conf.DarktableCli)
importer := NewImporter(conf.OriginalsPath, indexer, converter)
assert.IsType(t, &Importer{}, importer)
}
@ -26,7 +28,9 @@ func TestImporter_ImportPhotosFromDirectory(t *testing.T) {
indexer := NewIndexer(conf.OriginalsPath, tensorFlow, conf.GetDb())
importer := NewImporter(conf.OriginalsPath, indexer)
converter := NewConverter(conf.DarktableCli)
importer := NewImporter(conf.OriginalsPath, indexer, converter)
importer.ImportPhotosFromDirectory(conf.ImportPath)
}
@ -39,7 +43,9 @@ func TestImporter_GetDestinationFilename(t *testing.T) {
indexer := NewIndexer(conf.OriginalsPath, tensorFlow, conf.GetDb())
importer := NewImporter(conf.OriginalsPath, indexer)
converter := NewConverter(conf.DarktableCli)
importer := NewImporter(conf.OriginalsPath, indexer, converter)
rawFile, err := NewMediaFile(conf.ImportPath + "/raw/IMG_1435.CR2")

60
internal/photoprism/indexer.go
View File

@ -186,7 +186,39 @@ func (i *Indexer) IndexMediaFile(mediaFile *MediaFile) {
}
}
func (i *Indexer) IndexAll() {
func (i *Indexer) IndexRelated(mediaFile *MediaFile) map[string]bool {
indexed := make(map[string]bool)
relatedFiles, mainFile, err := mediaFile.GetRelatedFiles()
if err != nil {
log.Printf("Could not index \"%s\": %s", mediaFile.GetRelativeFilename(i.originalsPath), err.Error())
return indexed
}
log.Printf("Indexing main %s file \"%s\"", mainFile.GetType(), mainFile.GetRelativeFilename(i.originalsPath))
i.IndexMediaFile(mainFile)
indexed[mainFile.GetFilename()] = true
for _, relatedMediaFile := range relatedFiles {
if indexed[relatedMediaFile.GetFilename()] {
continue
}
log.Printf("Indexing related %s file \"%s\"", relatedMediaFile.GetType(), relatedMediaFile.GetRelativeFilename(i.originalsPath))
i.IndexMediaFile(relatedMediaFile)
indexed[relatedMediaFile.GetFilename()] = true
}
return indexed
}
func (i *Indexer) IndexAll() map[string]bool {
indexed := make(map[string]bool)
err := filepath.Walk(i.originalsPath, func(filename string, fileInfo os.FileInfo, err error) error {
@ -204,28 +236,8 @@ func (i *Indexer) IndexAll() {
return nil
}
relatedFiles, masterFile, err := mediaFile.GetRelatedFiles()
if err != nil {
log.Printf("Could not import %s: %s", mediaFile.GetRelativeFilename(i.originalsPath), err.Error())
return nil
}
log.Printf("Indexing %s", masterFile.GetRelativeFilename(i.originalsPath))
i.IndexMediaFile(masterFile)
indexed[masterFile.GetFilename()] = true
for _, relatedMediaFile := range relatedFiles {
if indexed[relatedMediaFile.GetFilename()] {
continue
}
log.Printf(" + related %s file: %s", relatedMediaFile.GetType(), relatedMediaFile.GetRelativeFilename(i.originalsPath))
i.IndexMediaFile(relatedMediaFile)
indexed[relatedMediaFile.GetFilename()] = true
for relatedFilename := range i.IndexRelated(mediaFile) {
indexed[relatedFilename] = true
}
return nil
@ -234,4 +246,6 @@ func (i *Indexer) IndexAll() {
if err != nil {
log.Print(err.Error())
}
return indexed
}

104
internal/photoprism/mediafile.go
View File

@ -6,7 +6,6 @@ import (
"github.com/brett-lempereur/ish"
"github.com/djherbis/times"
. "github.com/photoprism/photoprism/internal/models"
"github.com/pkg/errors"
"github.com/steakknife/hamming"
"image"
_ "image/gif"
@ -53,43 +52,43 @@ var FileExtensions = map[string]string{
".aae": FileTypeAae,
".heif": FileTypeHEIF,
".heic": FileTypeHEIF,
".3fr": FileTypeRaw,
".ari": FileTypeRaw,
".bay": FileTypeRaw,
".cr3": FileTypeRaw,
".cap": FileTypeRaw,
".3fr": FileTypeRaw,
".ari": FileTypeRaw,
".bay": FileTypeRaw,
".cr3": FileTypeRaw,
".cap": FileTypeRaw,
".data": FileTypeRaw,
".dcs": FileTypeRaw,
".dcr": FileTypeRaw,
".drf": FileTypeRaw,
".eip": FileTypeRaw,
".erf": FileTypeRaw,
".fff": FileTypeRaw,
".gpr": FileTypeRaw,
".iiq": FileTypeRaw,
".k25": FileTypeRaw,
".kdc": FileTypeRaw,
".mdc": FileTypeRaw,
".mef": FileTypeRaw,
".mos": FileTypeRaw,
".mrw": FileTypeRaw,
".nrw": FileTypeRaw,
".obm": FileTypeRaw,
".orf": FileTypeRaw,
".pef": FileTypeRaw,
".ptx": FileTypeRaw,
".pxn": FileTypeRaw,
".r3d": FileTypeRaw,
".raf": FileTypeRaw,
".raw": FileTypeRaw,
".rwl": FileTypeRaw,
".rw2": FileTypeRaw,
".rwz": FileTypeRaw,
".sr2": FileTypeRaw,
".srf": FileTypeRaw,
".srw": FileTypeRaw,
".tif": FileTypeRaw,
".x3f": FileTypeRaw,
".dcs": FileTypeRaw,
".dcr": FileTypeRaw,
".drf": FileTypeRaw,
".eip": FileTypeRaw,
".erf": FileTypeRaw,
".fff": FileTypeRaw,
".gpr": FileTypeRaw,
".iiq": FileTypeRaw,
".k25": FileTypeRaw,
".kdc": FileTypeRaw,
".mdc": FileTypeRaw,
".mef": FileTypeRaw,
".mos": FileTypeRaw,
".mrw": FileTypeRaw,
".nrw": FileTypeRaw,
".obm": FileTypeRaw,
".orf": FileTypeRaw,
".pef": FileTypeRaw,
".ptx": FileTypeRaw,
".pxn": FileTypeRaw,
".r3d": FileTypeRaw,
".raf": FileTypeRaw,
".raw": FileTypeRaw,
".rwl": FileTypeRaw,
".rw2": FileTypeRaw,
".rwz": FileTypeRaw,
".sr2": FileTypeRaw,
".srf": FileTypeRaw,
".srw": FileTypeRaw,
".tif": FileTypeRaw,
".x3f": FileTypeRaw,
}
type MediaFile struct {
@ -189,6 +188,10 @@ func (m *MediaFile) GetCanonicalNameFromFile() string {
}
}
func (m *MediaFile) GetCanonicalNameFromFileWithDirectory() string {
return m.GetDirectory() + string(os.PathSeparator) + m.GetCanonicalNameFromFile()
}
func (m *MediaFile) GetPerceptualHash() (string, error) {
if m.perceptualHash != "" {
return m.perceptualHash, nil
@ -255,8 +258,8 @@ func (m *MediaFile) GetEditedFilename() (result string) {
return result
}
func (m *MediaFile) GetRelatedFiles() (result []*MediaFile, masterFile *MediaFile, err error) {
baseFilename := m.GetDirectory() + string(os.PathSeparator) + m.GetCanonicalNameFromFile()
func (m *MediaFile) GetRelatedFiles() (result []*MediaFile, mainFile *MediaFile, err error) {
baseFilename := m.GetCanonicalNameFromFileWithDirectory()
matches, err := filepath.Glob(baseFilename + "*")
@ -275,18 +278,18 @@ func (m *MediaFile) GetRelatedFiles() (result []*MediaFile, masterFile *MediaFil
continue
}
if masterFile == nil && resultFile.IsJpeg() {
masterFile = resultFile
if mainFile == nil && resultFile.IsJpeg() {
mainFile = resultFile
} else if resultFile.IsRaw() {
masterFile = resultFile
} else if resultFile.IsJpeg() && resultFile.IsJpeg() && len(masterFile.GetFilename()) > len(resultFile.GetFilename()) {
masterFile = resultFile
mainFile = resultFile
} else if resultFile.IsJpeg() && resultFile.IsJpeg() && len(mainFile.GetFilename()) > len(resultFile.GetFilename()) {
mainFile = resultFile
}
result = append(result, resultFile)
}
return result, masterFile, nil
return result, mainFile, nil
}
func (m *MediaFile) GetFilename() string {
@ -301,7 +304,8 @@ func (m *MediaFile) GetRelativeFilename(directory string) string {
index := strings.Index(m.filename, directory)
if index == 0 {
return m.filename[len(directory):]
pos := len(directory) + 1
return m.filename[pos:]
}
return m.filename
@ -361,6 +365,10 @@ func (m *MediaFile) Remove() error {
return os.Remove(m.GetFilename())
}
func (m *MediaFile) HasSameFilename(other *MediaFile) bool {
return m.GetFilename() == other.GetFilename()
}
func (m *MediaFile) Move(newFilename string) error {
if err := os.Rename(m.filename, newFilename); err != nil {
return err
@ -442,10 +450,10 @@ func (m *MediaFile) GetJpeg() (*MediaFile, error) {
return m, nil
}
jpegFilename := m.GetFilename()[0:len(m.GetFilename())-len(filepath.Ext(m.GetFilename()))] + ".jpg"
jpegFilename := m.GetCanonicalNameFromFileWithDirectory() + ".jpg"
if !fileExists(jpegFilename) {
return nil, errors.New("file does not exist")
return nil, fmt.Errorf("jpeg file does not exist: %s", jpegFilename)
}
return NewMediaFile(jpegFilename)

4
internal/photoprism/thumbnails_test.go
View File

@ -50,7 +50,9 @@ func TestCreateThumbnailsFromOriginals(t *testing.T) {
indexer := NewIndexer(conf.OriginalsPath, tensorFlow, conf.GetDb())
importer := NewImporter(conf.OriginalsPath, indexer)
converter := NewConverter(conf.DarktableCli)
importer := NewImporter(conf.OriginalsPath, indexer, converter)
importer.ImportPhotosFromDirectory(conf.ImportPath)