luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Indexer: Fix JSON sidecar creation using Exiftool

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-12-25 20:29:06 +01:00
parent 45c0e2f60d
commit d430ae24ee
10 changed files with 110 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
internal/meta/json.go
View file

@ -33,7 +33,6 @@ func (data *Data) JSON(jsonName, originalName string) (err error) {
jsonData, err := ioutil.ReadFile(jsonName)
if err != nil {
log.Warnf("metadata: %s (json)", err.Error())
return fmt.Errorf("can't read json file %s", quotedName)
}
@ -45,6 +44,7 @@ func (data *Data) JSON(jsonName, originalName string) (err error) {
return data.GPhoto(jsonData)
}
log.Warnf("metadata: unknown format in %s (json)", quotedName)
return fmt.Errorf("unknown json format in %s", quotedName)
log.Warnf("metadata: unknown json in %s", quotedName)
return fmt.Errorf("unknown json in %s", quotedName)
}

3
internal/meta/json_test.go
View file

@ -1,9 +1,10 @@
package meta
import (
"github.com/photoprism/photoprism/pkg/fs"
"testing"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/stretchr/testify/assert"
)

4
internal/photoprism/import_worker.go
View file

@ -105,7 +105,7 @@ func ImportWorker(jobs <-chan ImportJob) {
}
}
if imp.conf.ExifToolJson() && f.IsMedia() && !f.HasJson() {
if f.NeedsJson() {
if jsonFile, err := imp.convert.ToJson(f); err != nil {
log.Errorf("import: %s in %s (create json sidecar)", err.Error(), txt.Quote(f.BaseName()))
} else {
@ -167,7 +167,7 @@ func ImportWorker(jobs <-chan ImportJob) {
continue
}
if ind.conf.ExifToolJson() && f.IsMedia() && !f.HasJson() {
if f.NeedsJson() {
if jsonFile, err := ind.convert.ToJson(f); err != nil {
log.Errorf("import: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error())
} else {

4
internal/photoprism/index_related.go
View file

@ -46,7 +46,7 @@ func IndexMain(related *RelatedFiles, ind *Index, opt IndexOptions) (result Inde
}
}
if ind.conf.ExifToolJson() && f.IsMedia() && !f.HasJson() {
if f.NeedsJson() {
if jsonFile, err := ind.convert.ToJson(f); err != nil {
log.Errorf("index: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error())
} else {
@ -102,7 +102,7 @@ func IndexRelated(related RelatedFiles, ind *Index, opt IndexOptions) (result In
continue
}
if ind.conf.ExifToolJson() && f.IsMedia() && !f.HasJson() {
if f.NeedsJson() {
if jsonFile, err := ind.convert.ToJson(f); err != nil {
log.Errorf("index: failed creating json sidecar for %s (%s)", txt.Quote(f.BaseName()), err.Error())
} else {

7
internal/photoprism/mediafile.go
View file

@ -804,6 +804,11 @@ func (m *MediaFile) HasJson() bool {
return fs.FormatJson.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false) != ""
}
// NeedsJson tests if the media file needs a JSON sidecar file to be created.
func (m *MediaFile) NeedsJson() bool {
return Config().ExifToolJson() && m.Root() != entity.RootSidecar && m.IsMedia() && !m.HasJson()
}
func (m *MediaFile) decodeDimensions() error {
if !m.IsMedia() {
return fmt.Errorf("failed decoding dimensions for %s", txt.Quote(m.BaseName()))
@ -1026,7 +1031,7 @@ func (m *MediaFile) RenameSidecars(oldFileName string) (renamed map[string]strin
}
for _, srcName := range matches {
destName := filepath.Join(sidecarPath, newName+filepath.Ext(srcName))
destName := filepath.Join(sidecarPath, newName+fs.Ext(srcName))
if fs.FileExists(destName) {
renamed[fs.RelName(srcName, sidecarPath)] = fs.RelName(destName, sidecarPath)

40
internal/photoprism/mediafile_test.go
View file

@ -2097,6 +2097,42 @@ func TestMediaFile_HasJson(t *testing.T) {
})
}
func TestMediaFile_NeedsJson(t *testing.T) {
t.Run("false", func(t *testing.T) {
conf := config.TestConfig()
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg")
if err != nil {
t.Fatal(err)
}
assert.True(t, mediaFile.NeedsJson())
})
t.Run("true", func(t *testing.T) {
conf := config.TestConfig()
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4")
if err != nil {
t.Fatal(err)
}
assert.False(t, mediaFile.NeedsJson())
})
t.Run("true", func(t *testing.T) {
conf := config.TestConfig()
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/blue-go-video.mp4.json")
if err != nil {
t.Fatal(err)
}
assert.False(t, mediaFile.NeedsJson())
})
}
func TestMediaFile_RenameSidecars(t *testing.T) {
t.Run("success", func(t *testing.T) {
conf := config.TestConfig()
@ -2119,8 +2155,8 @@ func TestMediaFile_RenameSidecars(t *testing.T) {
t.Fatal(err)
}
srcName := filepath.Join(conf.SidecarPath(), "foo/bar.json")
dstName := filepath.Join(conf.SidecarPath(), "2020/12/foobar.json")
srcName := filepath.Join(conf.SidecarPath(), "foo/bar.jpg.json")
dstName := filepath.Join(conf.SidecarPath(), "2020/12/foobar.jpg.json")
if err := ioutil.WriteFile(srcName, []byte("{}"), 0666); err != nil {
t.Fatal(err)

22
internal/photoprism/metadata.go
View file

@ -20,17 +20,19 @@ func (m *MediaFile) MetaData() (result meta.Data) {
err = fmt.Errorf("exif not supported: %s", txt.Quote(m.BaseName()))
}
// Parse regular JSON sidecar files ("img_1234.json").
if jsonFiles := fs.FormatJson.FindAll(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false); len(jsonFiles) == 0 {
log.Debugf("media: no json sidecar file found for %s", txt.Quote(filepath.Base(m.FileName())))
} else {
for _, jsonFile := range jsonFiles {
jsonErr := m.metaData.JSON(jsonFile, m.BaseName())
// Parse regular JSON sidecar files ("img_1234.json")
if !m.IsSidecar() {
if jsonFiles := fs.FormatJson.FindAll(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false); len(jsonFiles) == 0 {
log.Debugf("media: no json sidecar file found for %s", txt.Quote(filepath.Base(m.FileName())))
} else {
for _, jsonFile := range jsonFiles {
jsonErr := m.metaData.JSON(jsonFile, m.BaseName())
if jsonErr != nil {
log.Debug(jsonErr)
} else {
err = nil
if jsonErr != nil {
log.Debug(jsonErr)
} else {
err = nil
}
}
}
}

12
pkg/fs/base.go
View file

@ -24,6 +24,18 @@ func StripKnownExt(name string) string {
return name
}
// Ext returns all extension of a file name including the dots.
func Ext(name string) string {
ext := filepath.Ext(name)
name = StripExt(name)
if FileExt.Known(name) {
ext = filepath.Ext(name) + ext
}
return ext
}
// StripSequence removes common sequence patterns at the end of file names.
func StripSequence(name string) string {
// Strip numeric extensions like .00000, .00001, .4542353245,.... (at least 5 digits).

22
pkg/fs/base_test.go
View file

@ -40,6 +40,28 @@ func TestStripKnownExt(t *testing.T) {
})
}
func TestExt(t *testing.T) {
t.Run("Test.jpg", func(t *testing.T) {
result := Ext("/testdata/Test.jpg")
assert.Equal(t, ".jpg", result)
})
t.Run("Test.jpg.json", func(t *testing.T) {
result := Ext("/testdata/Test.jpg.json")
assert.Equal(t, ".jpg.json", result)
})
t.Run("Test copy 3.foo", func(t *testing.T) {
result := Ext("/testdata/Test copy 3.foo")
assert.Equal(t, ".foo", result)
})
t.Run("Test", func(t *testing.T) {
result := Ext("/testdata/Test")
assert.Equal(t, "", result)
})
}
func TestBase(t *testing.T) {
t.Run("Screenshot 2019-05-21 at 10.45.52.png", func(t *testing.T) {
regular := BasePrefix("Screenshot 2019-05-21 at 10.45.52.png", false)

28
pkg/fs/filetype.go
View file

@ -236,19 +236,13 @@ func (t FileFormat) FindFirst(fileName string, dirs []string, baseDir string, st
}
if info, err := os.Stat(filepath.Join(dir, fileBase) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, info.Name())
}
if info, err := os.Stat(filepath.Join(dir, fileBasePrefix) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, info.Name())
}
if info, err := os.Stat(filepath.Join(dir, fileBaseLower) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, info.Name())
}
if info, err := os.Stat(filepath.Join(dir, fileBaseUpper) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, info.Name())
return filepath.Join(dir, fileBase) + ext
} else if info, err := os.Stat(filepath.Join(dir, fileBasePrefix) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, fileBasePrefix) + ext
} else if info, err := os.Stat(filepath.Join(dir, fileBaseLower) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, fileBaseLower) + ext
} else if info, err := os.Stat(filepath.Join(dir, fileBaseUpper) + ext); err == nil && info.Mode().IsRegular() {
return filepath.Join(dir, fileBaseUpper) + ext
}
}
}
@ -285,19 +279,19 @@ func (t FileFormat) FindAll(fileName string, dirs []string, baseDir string, stri
}
if info, err := os.Stat(filepath.Join(dir, fileBase) + ext); err == nil && info.Mode().IsRegular() {
results = append(results, filepath.Join(dir, info.Name()))
results = append(results, filepath.Join(dir, fileBase)+ext)
}
if info, err := os.Stat(filepath.Join(dir, fileBasePrefix) + ext); err == nil && info.Mode().IsRegular() {
results = append(results, filepath.Join(dir, info.Name()))
results = append(results, filepath.Join(dir, fileBasePrefix)+ext)
}
if info, err := os.Stat(filepath.Join(dir, fileBaseLower) + ext); err == nil && info.Mode().IsRegular() {
results = append(results, filepath.Join(dir, info.Name()))
results = append(results, filepath.Join(dir, fileBaseLower)+ext)
}
if info, err := os.Stat(filepath.Join(dir, fileBaseUpper) + ext); err == nil && info.Mode().IsRegular() {
results = append(results, filepath.Join(dir, info.Name()))
results = append(results, filepath.Join(dir, fileBaseUpper)+ext)
}
}
}