Indexer: Automatically rename related sidecar files

internal/config/filenames_test.go

@@ -20,6 +20,7 @@ func TestConfig_SidecarJson(t *testing.T) {
 
 func TestConfig_SidecarYaml(t *testing.T) {
 	c := NewConfig(CliTestContext())
+
 	assert.Equal(t, false, c.SidecarYaml())
 	c.params.ReadOnly = true
 	assert.Equal(t, false, c.SidecarJson())
@@ -27,14 +28,17 @@ func TestConfig_SidecarYaml(t *testing.T) {
 
 func TestConfig_SidecarPath(t *testing.T) {
 	c := NewConfig(CliTestContext())
+
+	assert.Contains(t, c.SidecarPath(), "testdata/sidecar")
+	c.params.SidecarPath = ".photoprism"
 	assert.Equal(t, ".photoprism", c.SidecarPath())
-	c.params.SidecarPath = ""
-	assert.Equal(t, "/go/src/github.com/photoprism/photoprism/storage/testdata/sidecar", c.SidecarPath())
 }
 
 func TestConfig_SidecarPathIsAbs(t *testing.T) {
 	c := NewConfig(CliTestContext())
 
+	assert.Equal(t, true, c.SidecarPathIsAbs())
+	c.params.SidecarPath = ".photoprism"
 	assert.Equal(t, false, c.SidecarPathIsAbs())
 }
 

internal/config/test.go

@@ -59,7 +59,6 @@ func NewTestParams() *Params {
 		ReadOnly:       false,
 		DetectNSFW:     true,
 		UploadNSFW:     false,
-		SidecarPath:    fs.HiddenPath,
 		AssetsPath:     assetsPath,
 		StoragePath:    testDataPath,
 		CachePath:      testDataPath + "/cache",
@@ -67,6 +66,7 @@ func NewTestParams() *Params {
 		ImportPath:     testDataPath + "/import",
 		TempPath:       testDataPath + "/temp",
 		SettingsPath:   testDataPath + "/settings",
+		SidecarPath:    testDataPath + "/sidecar",
 		DatabaseDriver: dbDriver,
 		DatabaseDsn:    dbDsn,
 		AdminPassword:  "photoprism",

internal/photoprism/convert_test.go

@@ -2,6 +2,7 @@ package photoprism
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/photoprism/photoprism/internal/config"
@@ -27,8 +28,8 @@ func TestConvert_ToJpeg(t *testing.T) {
 	convert := NewConvert(conf)
 
 	t.Run("gopher-video.mp4", func(t *testing.T) {
-		fileName := conf.ExamplesPath() + "/gopher-video.mp4"
-		outputName := conf.ExamplesPath() + "/.photoprism/gopher-video.jpg"
+		fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4")
+		outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.jpg")
 
 		_ = os.Remove(outputName)
 
@@ -55,7 +56,7 @@ func TestConvert_ToJpeg(t *testing.T) {
 	})
 
 	t.Run("fern_green.jpg", func(t *testing.T) {
-		jpegFilename := conf.ImportPath() + "/fern_green.jpg"
+		jpegFilename := filepath.Join(conf.ImportPath(), "fern_green.jpg")
 
 		assert.Truef(t, fs.FileExists(jpegFilename), "file does not exist: %s", jpegFilename)
 
@@ -79,7 +80,8 @@ func TestConvert_ToJpeg(t *testing.T) {
 
 		assert.Equal(t, "Canon EOS 7D", infoJpeg.CameraModel)
 
-		rawFilename := conf.ImportPath() + "/raw/IMG_2567.CR2"
+		rawFilename := filepath.Join(conf.ImportPath(), "raw", "IMG_2567.CR2")
+		jpgFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "raw/IMG_2567.jpg")
 
 		t.Logf("Testing RAW to JPEG convert with %s", rawFilename)
 
@@ -95,7 +97,7 @@ func TestConvert_ToJpeg(t *testing.T) {
 			t.Fatalf("%s for %s", err.Error(), rawFilename)
 		}
 
-		assert.True(t, fs.FileExists(conf.ImportPath()+"/raw/.photoprism/IMG_2567.jpg"), "Jpeg file was not found - is Darktable installed?")
+		assert.True(t, fs.FileExists(jpgFilename), "Jpeg file was not found - is Darktable installed?")
 
 		if imageRaw == nil {
 			t.Fatal("imageRaw is nil")
@@ -106,6 +108,8 @@ func TestConvert_ToJpeg(t *testing.T) {
 		infoRaw := imageRaw.MetaData()
 
 		assert.Equal(t, "Canon EOS 6D", infoRaw.CameraModel)
+
+		_ = os.Remove(jpgFilename)
 	})
 }
 
@@ -114,8 +118,8 @@ func TestConvert_ToJson(t *testing.T) {
 	convert := NewConvert(conf)
 
 	t.Run("gopher-video.mp4", func(t *testing.T) {
-		fileName := conf.ExamplesPath() + "/gopher-video.mp4"
-		outputName := conf.ExamplesPath() + "/.photoprism/gopher-video.json"
+		fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4")
+		outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.json")
 
 		_ = os.Remove(outputName)
 
@@ -149,8 +153,8 @@ func TestConvert_ToJson(t *testing.T) {
 	})
 
 	t.Run("IMG_4120.JPG", func(t *testing.T) {
-		fileName := conf.ExamplesPath() + "/IMG_4120.JPG"
-		outputName := conf.ExamplesPath() + "/.photoprism/IMG_4120.json"
+		fileName := filepath.Join(conf.ExamplesPath(), "IMG_4120.JPG")
+		outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "IMG_4120.json")
 
 		_ = os.Remove(outputName)
 
@@ -228,7 +232,7 @@ func TestConvert_Start(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	jpegFilename := conf.ImportPath() + "/raw/.photoprism/canon_eos_6d.jpg"
+	jpegFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "raw/canon_eos_6d.jpg")
 
 	assert.True(t, fs.FileExists(jpegFilename), "Jpeg file was not found - is Darktable installed?")
 
@@ -244,7 +248,7 @@ func TestConvert_Start(t *testing.T) {
 
 	assert.Equal(t, "Canon EOS 6D", infoRaw.CameraModel, "UpdateCamera model should be Canon EOS M10")
 
-	existingJpegFilename := conf.ImportPath() + "/raw/.photoprism/IMG_2567.jpg"
+	existingJpegFilename := filepath.Join(conf.SidecarPath(), conf.ImportPath(), "/raw/IMG_2567.jpg")
 
 	oldHash := fs.Hash(existingJpegFilename)
 

internal/photoprism/filename_test.go

@@ -10,7 +10,7 @@ import (
 func TestFileName(t *testing.T) {
 	conf := config.TestConfig()
 	t.Run("sidecar", func(t *testing.T) {
-		assert.Equal(t, ".photoprism/test.jpg", FileName("sidecar", "test.jpg"))
+		assert.Equal(t, conf.SidecarPath()+"/test.jpg", FileName("sidecar", "test.jpg"))
 	})
 	t.Run("import", func(t *testing.T) {
 		assert.Equal(t, conf.ImportPath()+"/test.jpg", FileName("import", "test.jpg"))

internal/photoprism/index_mediafile.go

@@ -132,11 +132,17 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 	if !fileExists && !m.IsSidecar() && m.Root() == entity.RootOriginals {
 		fileHash = m.Hash()
 		fileQuery = entity.UnscopedDb().First(&file, "file_hash = ?", fileHash)
-		fileExists = fileQuery.Error == nil
+
+		indFileName := ""
+
+		if fileQuery.Error == nil {
+			fileExists = true
+			indFileName = FileName(file.FileRoot, file.FileName)
+		}
 
 		if !fileExists {
 			// Do nothing.
-		} else if fs.FileExists(FileName(file.FileRoot, file.FileName)) {
+		} else if fs.FileExists(indFileName) {
 			if err := entity.AddDuplicate(m.RootRelName(), m.Root(), m.Hash(), m.FileSize(), m.ModTime().Unix()); err != nil {
 				log.Error(err)
 			}
@@ -151,6 +157,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 			result.Err = err
 
 			return result
+		} else if err := m.RenameSidecars(indFileName); err != nil {
+			log.Errorf("index: %s in %s (rename)", err.Error(), logName)
+			fileRenamed = true
 		} else {
 			fileRenamed = true
 		}
@@ -242,7 +251,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 	// Flag first JPEG as primary file for this photo.
 	if !file.FilePrimary {
 		if photoExists {
-			if q := entity.UnscopedDb().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
+			if q := entity.Db().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
 				file.FilePrimary = m.IsJpeg()
 			}
 		} else {

internal/photoprism/mediafile.go

@@ -918,7 +918,7 @@ func (m *MediaFile) Thumbnail(path string, typeName string) (filename string, er
 	return thumbnail, nil
 }
 
-// Thumbnail returns a resampled image of the file.
+// Resample returns a resampled image of the file.
 func (m *MediaFile) Resample(path string, typeName string) (img image.Image, err error) {
 	filename, err := m.Thumbnail(path, typeName)
 
@@ -929,6 +929,7 @@ func (m *MediaFile) Resample(path string, typeName string) (img image.Image, err
 	return imaging.Open(filename, imaging.AutoOrientation(true))
 }
 
+// ResampleDefault pre-renders default thumbnails.
 func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
 	count := 0
 	start := time.Now()
@@ -1000,3 +1001,25 @@ func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
 
 	return nil
 }
+
+// RenameSidecars moves related sidecar files.
+func (m *MediaFile) RenameSidecars(oldFileName string) (err error) {
+	oldRelPrefix := fs.RelPrefix(oldFileName, Config().OriginalsPath(), false)
+	newRelPrefix := m.RelPrefix(Config().OriginalsPath(), false)
+	globPrefix := filepath.Join(Config().SidecarPath(), oldRelPrefix) + "."
+	matches, err := filepath.Glob(regexp.QuoteMeta(globPrefix) + "*")
+
+	if err != nil {
+		return err
+	}
+
+	for _, fileName := range matches {
+		destName := filepath.Join(Config().SidecarPath(), newRelPrefix + filepath.Ext(fileName))
+
+		if err := fs.Move(fileName, destName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/photoprism/mediafile_test.go

@@ -1,6 +1,7 @@
 package photoprism
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -488,18 +489,20 @@ func TestMediaFile_RelatedFiles(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		assert.Len(t, related.Files, 3)
+		assert.GreaterOrEqual(t, len(related.Files), 3)
 
 		for _, result := range related.Files {
 			t.Logf("FileName: %s", result.FileName())
 
 			filename := result.FileName()
-
 			extension := result.Extension()
-
 			baseFilename := filename[0 : len(filename)-len(extension)]
 
-			assert.Equal(t, expectedBaseFilename, baseFilename)
+			if result.IsJpeg() {
+				assert.Contains(t, expectedBaseFilename, "examples/iphone_7")
+			} else {
+				assert.Equal(t, expectedBaseFilename, baseFilename)
+			}
 		}
 	})
 
@@ -1459,7 +1462,7 @@ func TestMediaFile_Jpeg(t *testing.T) {
 	t.Run("iphone_7.json", func(t *testing.T) {
 		conf := config.TestConfig()
 
-		mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/iphone_7.json")
+		mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/test.md")
 
 		if err != nil {
 			t.Fatal(err)
@@ -1920,10 +1923,10 @@ func TestMediaFile_PathNameInfo(t *testing.T) {
 		mediaFile.SetFileName(".photoprism/beach_sand.jpg")
 
 		root, base, path, name := mediaFile.PathNameInfo()
-		assert.Equal(t, "sidecar", root)
+		assert.Equal(t, "", root)
 		assert.Equal(t, "beach_sand", base)
 		assert.Equal(t, "", path)
-		assert.Equal(t, "beach_sand.jpg", name)
+		assert.Equal(t, ".photoprism/beach_sand.jpg", name)
 		mediaFile.SetFileName(initialName)
 	})
 
@@ -2093,3 +2096,45 @@ func TestMediaFile_HasJson(t *testing.T) {
 		assert.True(t, mediaFile.HasJson())
 	})
 }
+
+func TestMediaFile_RenameSidecars(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		conf := config.TestConfig()
+
+		jpegExample := filepath.Join(conf.ExamplesPath(), "/limes.jpg")
+		jpegPath := filepath.Join(conf.OriginalsPath(), "2020", "12")
+		jpegName := filepath.Join(jpegPath, "foobar.jpg")
+
+		if err := fs.Copy(jpegExample, jpegName); err != nil {
+			t.Fatal(err)
+		}
+
+		mf, err := NewMediaFile(jpegName)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		srcName := filepath.Join(conf.SidecarPath(), "foo/bar.json")
+		dstName := filepath.Join(conf.SidecarPath(), "2020/12/foobar.json")
+
+		if err := ioutil.WriteFile(srcName, []byte("{}"), os.ModePerm); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := mf.RenameSidecars(filepath.Join(conf.OriginalsPath(), "foo/bar.jpg")); err != nil {
+			t.Fatal(err)
+		}
+
+		if fs.FileExists(srcName) {
+			t.Errorf("src file still exists: %s", srcName)
+		}
+
+		if !fs.FileExists(dstName) {
+			t.Errorf("dst file not found: %s", srcName)
+		}
+
+		_ = os.Remove(srcName)
+		_ = os.Remove(dstName)
+	})
+}

internal/photoprism/metadata_test.go

@@ -2,6 +2,7 @@ package photoprism
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/photoprism/photoprism/internal/config"
@@ -215,7 +216,7 @@ func TestMediaFile_Exif_HEIF(t *testing.T) {
 
 	conf := config.TestConfig()
 
-	img, err := NewMediaFile(conf.ExamplesPath() + "/iphone_7.heic")
+	img, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "iphone_7.heic"))
 
 	if err != nil {
 		t.Fatal(err)
@@ -265,7 +266,7 @@ func TestMediaFile_Exif_HEIF(t *testing.T) {
 	assert.Equal(t, false, jpegInfo.Flash)
 	assert.Equal(t, "", jpegInfo.Description)
 
-	if err := os.Remove(conf.ExamplesPath() + "/.photoprism/iphone_7.jpg"); err != nil {
+	if err := os.Remove(filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "iphone_7.jpg")); err != nil {
 		t.Error(err)
 	}
 }

pkg/fs/move.go

@@ -0,0 +1,59 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Moves a file to a new destination.
+func Move(src, dest string) error {
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+	return err
+	}
+
+	if err := os.Rename(src, dest); err == nil {
+		return nil
+	}
+
+	if err := Copy(src, dest); err != nil {
+		return err
+	}
+
+	if err := os.Remove(src); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Copies a file to a destination.
+func Copy(src, dest string) error {
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		return err
+	}
+
+	thisFile, err := os.Open(src)
+
+	if err != nil {
+		return err
+	}
+
+	defer thisFile.Close()
+
+	destFile, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, os.ModePerm)
+
+	if err != nil {
+		return err
+	}
+
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, thisFile)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}