From 424c0ce6166d6a7160168b5d2e5923aca67f0c3b Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Dec 2020 19:17:07 +0100 Subject: [PATCH] Indexer: Automatically rename related sidecar files --- internal/config/filenames_test.go | 8 +++- internal/config/test.go | 2 +- internal/photoprism/convert_test.go | 26 +++++++----- internal/photoprism/filename_test.go | 2 +- internal/photoprism/index_mediafile.go | 15 +++++-- internal/photoprism/mediafile.go | 25 ++++++++++- internal/photoprism/mediafile_test.go | 59 +++++++++++++++++++++++--- internal/photoprism/metadata_test.go | 5 ++- pkg/fs/move.go | 59 ++++++++++++++++++++++++++ 9 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 pkg/fs/move.go diff --git a/internal/config/filenames_test.go b/internal/config/filenames_test.go index 98510c920..6e065b361 100644 --- a/internal/config/filenames_test.go +++ b/internal/config/filenames_test.go @@ -20,6 +20,7 @@ func TestConfig_SidecarJson(t *testing.T) { func TestConfig_SidecarYaml(t *testing.T) { c := NewConfig(CliTestContext()) + assert.Equal(t, false, c.SidecarYaml()) c.params.ReadOnly = true assert.Equal(t, false, c.SidecarJson()) @@ -27,14 +28,17 @@ func TestConfig_SidecarYaml(t *testing.T) { func TestConfig_SidecarPath(t *testing.T) { c := NewConfig(CliTestContext()) + + assert.Contains(t, c.SidecarPath(), "testdata/sidecar") + c.params.SidecarPath = ".photoprism" assert.Equal(t, ".photoprism", c.SidecarPath()) - c.params.SidecarPath = "" - assert.Equal(t, "/go/src/github.com/photoprism/photoprism/storage/testdata/sidecar", c.SidecarPath()) } func TestConfig_SidecarPathIsAbs(t *testing.T) { c := NewConfig(CliTestContext()) + assert.Equal(t, true, c.SidecarPathIsAbs()) + c.params.SidecarPath = ".photoprism" assert.Equal(t, false, c.SidecarPathIsAbs()) } diff --git a/internal/config/test.go b/internal/config/test.go index 293e3486b..917d36b88 100644 --- a/internal/config/test.go +++ b/internal/config/test.go @@ -59,7 +59,6 @@ func NewTestParams() *Params { ReadOnly: false, DetectNSFW: true, UploadNSFW: false, - SidecarPath: fs.HiddenPath, AssetsPath: assetsPath, StoragePath: testDataPath, CachePath: testDataPath + "/cache", @@ -67,6 +66,7 @@ func NewTestParams() *Params { ImportPath: testDataPath + "/import", TempPath: testDataPath + "/temp", SettingsPath: testDataPath + "/settings", + SidecarPath: testDataPath + "/sidecar", DatabaseDriver: dbDriver, DatabaseDsn: dbDsn, AdminPassword: "photoprism", diff --git a/internal/photoprism/convert_test.go b/internal/photoprism/convert_test.go index ad7660883..7952e406b 100644 --- a/internal/photoprism/convert_test.go +++ b/internal/photoprism/convert_test.go @@ -2,6 +2,7 @@ package photoprism import ( "os" + "path/filepath" "testing" "github.com/photoprism/photoprism/internal/config" @@ -27,8 +28,8 @@ func TestConvert_ToJpeg(t *testing.T) { convert := NewConvert(conf) t.Run("gopher-video.mp4", func(t *testing.T) { - fileName := conf.ExamplesPath() + "/gopher-video.mp4" - outputName := conf.ExamplesPath() + "/.photoprism/gopher-video.jpg" + fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4") + outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.jpg") _ = os.Remove(outputName) @@ -55,7 +56,7 @@ func TestConvert_ToJpeg(t *testing.T) { }) t.Run("fern_green.jpg", func(t *testing.T) { - jpegFilename := conf.ImportPath() + "/fern_green.jpg" + jpegFilename := filepath.Join(conf.ImportPath(), "fern_green.jpg") assert.Truef(t, fs.FileExists(jpegFilename), "file does not exist: %s", jpegFilename) @@ -79,7 +80,8 @@ func TestConvert_ToJpeg(t *testing.T) { assert.Equal(t, "Canon EOS 7D", infoJpeg.CameraModel) - rawFilename := conf.ImportPath() + "/raw/IMG_2567.CR2" + rawFilename := filepath.Join(conf.ImportPath(), "raw", "IMG_2567.CR2") + jpgFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "raw/IMG_2567.jpg") t.Logf("Testing RAW to JPEG convert with %s", rawFilename) @@ -95,7 +97,7 @@ func TestConvert_ToJpeg(t *testing.T) { t.Fatalf("%s for %s", err.Error(), rawFilename) } - assert.True(t, fs.FileExists(conf.ImportPath()+"/raw/.photoprism/IMG_2567.jpg"), "Jpeg file was not found - is Darktable installed?") + assert.True(t, fs.FileExists(jpgFilename), "Jpeg file was not found - is Darktable installed?") if imageRaw == nil { t.Fatal("imageRaw is nil") @@ -106,6 +108,8 @@ func TestConvert_ToJpeg(t *testing.T) { infoRaw := imageRaw.MetaData() assert.Equal(t, "Canon EOS 6D", infoRaw.CameraModel) + + _ = os.Remove(jpgFilename) }) } @@ -114,8 +118,8 @@ func TestConvert_ToJson(t *testing.T) { convert := NewConvert(conf) t.Run("gopher-video.mp4", func(t *testing.T) { - fileName := conf.ExamplesPath() + "/gopher-video.mp4" - outputName := conf.ExamplesPath() + "/.photoprism/gopher-video.json" + fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4") + outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.json") _ = os.Remove(outputName) @@ -149,8 +153,8 @@ func TestConvert_ToJson(t *testing.T) { }) t.Run("IMG_4120.JPG", func(t *testing.T) { - fileName := conf.ExamplesPath() + "/IMG_4120.JPG" - outputName := conf.ExamplesPath() + "/.photoprism/IMG_4120.json" + fileName := filepath.Join(conf.ExamplesPath(), "IMG_4120.JPG") + outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "IMG_4120.json") _ = os.Remove(outputName) @@ -228,7 +232,7 @@ func TestConvert_Start(t *testing.T) { t.Fatal(err) } - jpegFilename := conf.ImportPath() + "/raw/.photoprism/canon_eos_6d.jpg" + jpegFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "raw/canon_eos_6d.jpg") assert.True(t, fs.FileExists(jpegFilename), "Jpeg file was not found - is Darktable installed?") @@ -244,7 +248,7 @@ func TestConvert_Start(t *testing.T) { assert.Equal(t, "Canon EOS 6D", infoRaw.CameraModel, "UpdateCamera model should be Canon EOS M10") - existingJpegFilename := conf.ImportPath() + "/raw/.photoprism/IMG_2567.jpg" + existingJpegFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "/raw/IMG_2567.jpg") oldHash := fs.Hash(existingJpegFilename) diff --git a/internal/photoprism/filename_test.go b/internal/photoprism/filename_test.go index 35a04e4d9..316d5b716 100644 --- a/internal/photoprism/filename_test.go +++ b/internal/photoprism/filename_test.go @@ -10,7 +10,7 @@ import ( func TestFileName(t *testing.T) { conf := config.TestConfig() t.Run("sidecar", func(t *testing.T) { - assert.Equal(t, ".photoprism/test.jpg", FileName("sidecar", "test.jpg")) + assert.Equal(t, conf.SidecarPath()+"/test.jpg", FileName("sidecar", "test.jpg")) }) t.Run("import", func(t *testing.T) { assert.Equal(t, conf.ImportPath()+"/test.jpg", FileName("import", "test.jpg")) diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index cd0c81fab..d6e8a0035 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -132,11 +132,17 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( if !fileExists && !m.IsSidecar() && m.Root() == entity.RootOriginals { fileHash = m.Hash() fileQuery = entity.UnscopedDb().First(&file, "file_hash = ?", fileHash) - fileExists = fileQuery.Error == nil + + indFileName := "" + + if fileQuery.Error == nil { + fileExists = true + indFileName = FileName(file.FileRoot, file.FileName) + } if !fileExists { // Do nothing. - } else if fs.FileExists(FileName(file.FileRoot, file.FileName)) { + } else if fs.FileExists(indFileName) { if err := entity.AddDuplicate(m.RootRelName(), m.Root(), m.Hash(), m.FileSize(), m.ModTime().Unix()); err != nil { log.Error(err) } @@ -151,6 +157,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( result.Err = err return result + } else if err := m.RenameSidecars(indFileName); err != nil { + log.Errorf("index: %s in %s (rename)", err.Error(), logName) + fileRenamed = true } else { fileRenamed = true } @@ -242,7 +251,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) ( // Flag first JPEG as primary file for this photo. if !file.FilePrimary { if photoExists { - if q := entity.UnscopedDb().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil { + if q := entity.Db().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil { file.FilePrimary = m.IsJpeg() } } else { diff --git a/internal/photoprism/mediafile.go b/internal/photoprism/mediafile.go index 1d37b75b9..74309d4a2 100644 --- a/internal/photoprism/mediafile.go +++ b/internal/photoprism/mediafile.go @@ -918,7 +918,7 @@ func (m *MediaFile) Thumbnail(path string, typeName string) (filename string, er return thumbnail, nil } -// Thumbnail returns a resampled image of the file. +// Resample returns a resampled image of the file. func (m *MediaFile) Resample(path string, typeName string) (img image.Image, err error) { filename, err := m.Thumbnail(path, typeName) @@ -929,6 +929,7 @@ func (m *MediaFile) Resample(path string, typeName string) (img image.Image, err return imaging.Open(filename, imaging.AutoOrientation(true)) } +// ResampleDefault pre-renders default thumbnails. func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) { count := 0 start := time.Now() @@ -1000,3 +1001,25 @@ func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) { return nil } + +// RenameSidecars moves related sidecar files. +func (m *MediaFile) RenameSidecars(oldFileName string) (err error) { + oldRelPrefix := fs.RelPrefix(oldFileName, Config().OriginalsPath(), false) + newRelPrefix := m.RelPrefix(Config().OriginalsPath(), false) + globPrefix := filepath.Join(Config().SidecarPath(), oldRelPrefix) + "." + matches, err := filepath.Glob(regexp.QuoteMeta(globPrefix) + "*") + + if err != nil { + return err + } + + for _, fileName := range matches { + destName := filepath.Join(Config().SidecarPath(), newRelPrefix + filepath.Ext(fileName)) + + if err := fs.Move(fileName, destName); err != nil { + return err + } + } + + return nil +} \ No newline at end of file diff --git a/internal/photoprism/mediafile_test.go b/internal/photoprism/mediafile_test.go index 0dd47dc24..b1a8475a8 100644 --- a/internal/photoprism/mediafile_test.go +++ b/internal/photoprism/mediafile_test.go @@ -1,6 +1,7 @@ package photoprism import ( + "io/ioutil" "os" "path/filepath" "strings" @@ -488,18 +489,20 @@ func TestMediaFile_RelatedFiles(t *testing.T) { t.Fatal(err) } - assert.Len(t, related.Files, 3) + assert.GreaterOrEqual(t, len(related.Files), 3) for _, result := range related.Files { t.Logf("FileName: %s", result.FileName()) filename := result.FileName() - extension := result.Extension() - baseFilename := filename[0 : len(filename)-len(extension)] - assert.Equal(t, expectedBaseFilename, baseFilename) + if result.IsJpeg() { + assert.Contains(t, expectedBaseFilename, "examples/iphone_7") + } else { + assert.Equal(t, expectedBaseFilename, baseFilename) + } } }) @@ -1459,7 +1462,7 @@ func TestMediaFile_Jpeg(t *testing.T) { t.Run("iphone_7.json", func(t *testing.T) { conf := config.TestConfig() - mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/iphone_7.json") + mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/test.md") if err != nil { t.Fatal(err) @@ -1920,10 +1923,10 @@ func TestMediaFile_PathNameInfo(t *testing.T) { mediaFile.SetFileName(".photoprism/beach_sand.jpg") root, base, path, name := mediaFile.PathNameInfo() - assert.Equal(t, "sidecar", root) + assert.Equal(t, "", root) assert.Equal(t, "beach_sand", base) assert.Equal(t, "", path) - assert.Equal(t, "beach_sand.jpg", name) + assert.Equal(t, ".photoprism/beach_sand.jpg", name) mediaFile.SetFileName(initialName) }) @@ -2093,3 +2096,45 @@ func TestMediaFile_HasJson(t *testing.T) { assert.True(t, mediaFile.HasJson()) }) } + +func TestMediaFile_RenameSidecars(t *testing.T) { + t.Run("success", func(t *testing.T) { + conf := config.TestConfig() + + jpegExample := filepath.Join(conf.ExamplesPath(), "/limes.jpg") + jpegPath := filepath.Join(conf.OriginalsPath(), "2020", "12") + jpegName := filepath.Join(jpegPath, "foobar.jpg") + + if err := fs.Copy(jpegExample, jpegName); err != nil { + t.Fatal(err) + } + + mf, err := NewMediaFile(jpegName) + + if err != nil { + t.Fatal(err) + } + + srcName := filepath.Join(conf.SidecarPath(), "foo/bar.json") + dstName := filepath.Join(conf.SidecarPath(), "2020/12/foobar.json") + + if err := ioutil.WriteFile(srcName, []byte("{}"), os.ModePerm); err != nil { + t.Fatal(err) + } + + if err := mf.RenameSidecars(filepath.Join(conf.OriginalsPath(), "foo/bar.jpg")); err != nil { + t.Fatal(err) + } + + if fs.FileExists(srcName) { + t.Errorf("src file still exists: %s", srcName) + } + + if !fs.FileExists(dstName) { + t.Errorf("dst file not found: %s", srcName) + } + + _ = os.Remove(srcName) + _ = os.Remove(dstName) + }) +} \ No newline at end of file diff --git a/internal/photoprism/metadata_test.go b/internal/photoprism/metadata_test.go index ac541aa41..e4c68aa6a 100644 --- a/internal/photoprism/metadata_test.go +++ b/internal/photoprism/metadata_test.go @@ -2,6 +2,7 @@ package photoprism import ( "os" + "path/filepath" "testing" "github.com/photoprism/photoprism/internal/config" @@ -215,7 +216,7 @@ func TestMediaFile_Exif_HEIF(t *testing.T) { conf := config.TestConfig() - img, err := NewMediaFile(conf.ExamplesPath() + "/iphone_7.heic") + img, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "iphone_7.heic")) if err != nil { t.Fatal(err) @@ -265,7 +266,7 @@ func TestMediaFile_Exif_HEIF(t *testing.T) { assert.Equal(t, false, jpegInfo.Flash) assert.Equal(t, "", jpegInfo.Description) - if err := os.Remove(conf.ExamplesPath() + "/.photoprism/iphone_7.jpg"); err != nil { + if err := os.Remove(filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "iphone_7.jpg")); err != nil { t.Error(err) } } diff --git a/pkg/fs/move.go b/pkg/fs/move.go new file mode 100644 index 000000000..83c7b677f --- /dev/null +++ b/pkg/fs/move.go @@ -0,0 +1,59 @@ +package fs + +import ( + "io" + "os" + "path/filepath" +) + +// Moves a file to a new destination. +func Move(src, dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + return err + } + + if err := os.Rename(src, dest); err == nil { + return nil + } + + if err := Copy(src, dest); err != nil { + return err + } + + if err := os.Remove(src); err != nil { + return err + } + + return nil +} + +// Copies a file to a destination. +func Copy(src, dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + return err + } + + thisFile, err := os.Open(src) + + if err != nil { + return err + } + + defer thisFile.Close() + + destFile, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, os.ModePerm) + + if err != nil { + return err + } + + defer destFile.Close() + + _, err = io.Copy(destFile, thisFile) + + if err != nil { + return err + } + + return nil +} \ No newline at end of file