Live Photos: Add Support for Samsung Motion Photos #439 #3588

* Save mp4 files generated from samsung motion photos
* Parse exiftool payload to identify videos
* Detect embedded video file type
* Extract embedded video in RelatedFiles

assets/examples/beach_sand.json

@@ -0,0 +1,70 @@
+[{
+  "SourceFile": "beach_sand.jpg",
+  "ExifToolVersion": 12.56,
+  "FileName": "beach_sand.jpg",
+  "Directory": ".",
+  "FileSize": 105321,
+  "FileModifyDate": "2023:06:20 04:43:41+00:00",
+  "FileAccessDate": "2023:08:06 16:38:50+00:00",
+  "FileInodeChangeDate": "2023:08:04 05:10:37+00:00",
+  "FilePermissions": 100644,
+  "FileType": "JPEG",
+  "FileTypeExtension": "JPG",
+  "MIMEType": "image/jpeg",
+  "JFIFVersion": "1 2",
+  "ExifByteOrder": "MM",
+  "Make": "Apple",
+  "Model": "iPhone SE",
+  "XResolution": 72,
+  "YResolution": 72,
+  "ResolutionUnit": 2,
+  "Software": "10.2.1",
+  "ModifyDate": "2017:02:15 14:13:40",
+  "YCbCrPositioning": 1,
+  "ExposureTime": 0.0004940711462,
+  "ExposureProgram": 2,
+  "ISO": 25,
+  "ExifVersion": "0231",
+  "DateTimeOriginal": "2017:02:15 14:13:40",
+  "CreateDate": "2017:02:15 14:13:40",
+  "ComponentsConfiguration": "1 2 3 0",
+  "ApertureValue": 2.19999999733148,
+  "MeteringMode": 5,
+  "Flash": 16,
+  "FocalLength": 4.2,
+  "SubSecTimeOriginal": 249,
+  "SubSecTimeDigitized": 249,
+  "FlashpixVersion": "0100",
+  "ColorSpace": 65535,
+  "SensingMethod": 2,
+  "ExposureMode": 0,
+  "WhiteBalance": 0,
+  "SceneCaptureType": 0,
+  "LensModel": "iPhone SE back camera 4.15mm f/2.2",
+  "GPSVersionID": "2 3 0 0",
+  "GPSLatitudeRef": "S",
+  "GPSLongitudeRef": "E",
+  "GPSAltitudeRef": 1,
+  "CurrentIPTCDigest": "804bedc723e0e6cd3a41d0a44b074d19",
+  "DocumentNotes": "https://flickr.com/e/Rl7qi7oH%2BSEGDuwWBbZYaBQMEB5oNfPvQ6m3aMrPQ64%3D",
+  "ApplicationRecordVersion": 4,
+  "ImageWidth": 640,
+  "ImageHeight": 480,
+  "EncodingProcess": 2,
+  "BitsPerSample": 8,
+  "ColorComponents": 3,
+  "YCbCrSubSampling": "2 2",
+  "Aperture": 2.19999999733148,
+  "ImageSize": "640 480",
+  "Megapixels": 0.3072,
+  "ShutterSpeed": 0.0004940711462,
+  "SubSecCreateDate": "2017:02:15 14:13:40.249",
+  "SubSecDateTimeOriginal": "2017:02:15 14:13:40.249",
+  "GPSAltitude": -1.990417522,
+  "GPSLatitude": -29.2824777777778,
+  "GPSLongitude": 31.4436361111111,
+  "FocalLength35efl": 4.2,
+  "GPSPosition": "-29.2824777777778 31.4436361111111",
+  "LightValue": 15.2580006188259,
+  "LensID": "iPhone SE back camera 4.15mm f/2.2"
+}]

assets/examples/samsung-motion-photo.json

@@ -0,0 +1,82 @@
+[{
+  "SourceFile": "/go/src/github.com/photoprism/photoprism/storage/import/samsung-motion-photo.jpg",
+  "ExifToolVersion": 12.56,
+  "FileName": "samsung-motion-photo.jpg",
+  "Directory": "/go/src/github.com/photoprism/photoprism/storage/import",
+  "FileSize": 7221645,
+  "FileModifyDate": "2023:07:27 02:03:13+00:00",
+  "FileAccessDate": "2023:08:13 17:11:04+00:00",
+  "FileInodeChangeDate": "2023:08:13 17:11:02+00:00",
+  "FilePermissions": 100644,
+  "FileType": "JPEG",
+  "FileTypeExtension": "JPG",
+  "MIMEType": "image/jpeg",
+  "ExifByteOrder": "II",
+  "Make": "samsung",
+  "Model": "SM-G973F",
+  "Orientation": 1,
+  "XResolution": 72,
+  "YResolution": 72,
+  "ResolutionUnit": 2,
+  "Software": "G973FXXU4CTC9",
+  "ModifyDate": "2020:04:23 18:33:41",
+  "YCbCrPositioning": 1,
+  "ExposureTime": 0.0007575757576,
+  "FNumber": 2.4,
+  "ExposureProgram": 2,
+  "ISO": 50,
+  "ExifVersion": "0220",
+  "DateTimeOriginal": "2020:04:23 18:33:41",
+  "CreateDate": "2020:04:23 18:33:41",
+  "ShutterSpeedValue": 0.999475026346474,
+  "ApertureValue": 2.39495740923786,
+  "BrightnessValue": 22.58,
+  "ExposureCompensation": 0,
+  "MaxApertureValue": 2.39495740923786,
+  "MeteringMode": 2,
+  "Flash": 0,
+  "FocalLength": 4.32,
+  "ColorSpace": 1,
+  "ExifImageWidth": 4032,
+  "ExifImageHeight": 3024,
+  "ExposureMode": 0,
+  "WhiteBalance": 0,
+  "DigitalZoomRatio": 1,
+  "FocalLengthIn35mmFormat": 26,
+  "SceneCaptureType": 0,
+  "ImageUniqueID": "L12XLLD01VM",
+  "GPSLatitudeRef": "N",
+  "GPSLongitudeRef": "W",
+  "Compression": 6,
+  "ThumbnailOffset": 888,
+  "ThumbnailLength": 50555,
+  "XMPToolkit": "Adobe XMP Core 5.1.0-jc003",
+  "MicroVideo": 1,
+  "MicroVideoVersion": 1,
+  "MicroVideoOffset": 4535831,
+  "MicroVideoPresentationTimestampUs": -1,
+  "ImageWidth": 4032,
+  "ImageHeight": 3024,
+  "EncodingProcess": 0,
+  "BitsPerSample": 8,
+  "ColorComponents": 3,
+  "YCbCrSubSampling": "2 2",
+  "TimeStamp": "2020:04:23 17:33:41.809+00:00",
+  "MCCData": 234,
+  "EmbeddedVideoType": "MotionPhoto_Data",
+  "EmbeddedVideoFile": "(Binary data 4535775 bytes, use -b option to extract)",
+  "Aperture": 2.4,
+  "ImageSize": "4032 3024",
+  "Megapixels": 12.192768,
+  "ScaleFactor35efl": 6.01851851851852,
+  "ShutterSpeed": 0.0007575757576,
+  "ThumbnailImage": "(Binary data 50555 bytes, use -b option to extract)",
+  "GPSLatitude": 51.5049828,
+  "GPSLongitude": -0.0787347997222222,
+  "CircleOfConfusion": "0.00499230176602706",
+  "FOV": 69.3903656740024,
+  "FocalLength35efl": 26,
+  "GPSPosition": "51.5049828 -0.0787347997222222",
+  "HyperfocalDistance": 1.55759815100044,
+  "LightValue": 13.8923910258672
+}]

internal/meta/data.go

@@ -65,6 +65,7 @@ type Data struct {
 	Rotation      int           `meta:"Rotation"`
 	Views         int           `meta:"-"`
 	Albums        []string      `meta:"-"`
+	EmbeddedVideo string        `meta:"EmbeddedVideo"`
 	Error         error         `meta:"-"`
 	json          map[string]string
 	exif          map[string]string

internal/meta/json_exiftool.go

@@ -365,5 +365,12 @@ func (data *Data) Exiftool(jsonData []byte, originalName string) (err error) {
 	data.Subject = SanitizeMeta(data.Subject)
 	data.Artist = SanitizeMeta(data.Artist)
 
+	// set the embedded video file name, if it exists
+	if embeddedVideo, ok := data.json["EmbeddedVideoFile"]; ok && embeddedVideo != "" {
+		data.EmbeddedVideo = "EmbeddedVideoFile"
+	} else if embeddedVideo, ok := data.json["MotionPhotoVideo"]; ok && embeddedVideo != "" {
+		data.EmbeddedVideo = "MotionPhotoVideo"
+	}
+
 	return nil
 }

internal/photoprism/mediafile.go

@@ -1,6 +1,8 @@
 package photoprism
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"image"
 	_ "image/gif"
@@ -9,6 +11,7 @@ import (
 	"io"
 	"math"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -314,6 +317,82 @@ func (m *MediaFile) EditedName() string {
 	return ""
 }
 
+func (m *MediaFile) ExtractEmbeddedVideo() (string, error) {
+	if m == nil {
+		return "", fmt.Errorf(
+			"mediafile: file is nil - you may have found a" +
+				" bug",
+		)
+	}
+
+	// Abort if the source media file does not exist.
+	if !m.Exists() {
+		return "", fmt.Errorf(
+			"mediafile: %s not found", clean.Log(m.RootRelName()),
+		)
+	} else if m.Empty() {
+		return "", fmt.Errorf(
+			"mediafile: %s is empty", clean.Log(m.RootRelName()),
+		)
+	}
+
+	// get the embedded video field name from the file metadata
+	if metaData := m.MetaData(); metaData.Error == nil && metaData.
+		EmbeddedVideo != "" {
+		outputPath := filepath.Join(Config().TempPath(), m.RootRelPath(), "%f")
+		cmd := exec.Command(
+			Config().ExifToolBin(), fmt.Sprintf("-%s", metaData.EmbeddedVideo),
+			"-b", "-w",
+			outputPath, m.FileName(),
+		)
+
+		var out bytes.Buffer
+		var stderr bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Stderr = &stderr
+		cmd.Env = []string{fmt.Sprintf("HOME=%s", Config().TempPath())}
+		if err := cmd.Run(); err != nil {
+			log.Debugf("Error running exiftool on video file: ", err)
+			if stderr.String() != "" {
+				return "", errors.New(stderr.String())
+			} else {
+				return "", err
+			}
+		}
+
+		// find the extracted video path
+		outputPath = strings.Replace(outputPath, "%f", m.BasePrefix(false), 1)
+
+		// detect mime type of the extracted video
+		mimeType := fs.MimeType(outputPath)
+
+		if len := len(strings.Split(mimeType, "/")); len <= 1 {
+			log.Debugf(
+				"Error detecting the mime type of video file at %s",
+				outputPath,
+			)
+			return "", nil
+		} else if extension := strings.Split(
+			mimeType, "/",
+		)[len-1]; extension != "" {
+			// rename the extracted video file with the correct extension and
+			// move it to the sidecar path
+			_, file := filepath.Split(outputPath)
+			newFileName := fmt.Sprintf("%s.%s", file, extension)
+			dstPath := filepath.Join(
+				Config().SidecarPath(), m.RootRelPath(), newFileName,
+			)
+			if err := fs.Move(outputPath, dstPath); err != nil {
+				log.Debugf("Error moving the video file at %s", outputPath)
+				return "", err
+			}
+			return dstPath, nil
+		}
+	}
+
+	return "", nil
+}
+
 // PathNameInfo returns file name infos for indexing.
 func (m *MediaFile) PathNameInfo(stripSequence bool) (fileRoot, fileBase, relativePath, relativeName string) {
 	fileRoot = m.Root()
@@ -946,13 +1025,25 @@ func (m *MediaFile) HasPreviewImage() bool {
 		return true
 	}
 
-	jpegName := fs.ImageJPEG.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false)
+	jpegName := fs.ImageJPEG.FindFirst(
+		m.FileName(),
+		[]string{
+			Config().SidecarPath(),
+			fs.HiddenPath,
+		},
+		Config().OriginalsPath(), false,
+	)
 
 	if m.hasPreviewImage = fs.MimeType(jpegName) == fs.MimeTypeJPEG; m.hasPreviewImage {
 		return true
 	}
 
-	pngName := fs.ImagePNG.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false)
+	pngName := fs.ImagePNG.FindFirst(
+		m.FileName(),
+		[]string{
+			Config().SidecarPath(), fs.HiddenPath,
+		}, Config().OriginalsPath(), false,
+	)
 
 	if m.hasPreviewImage = fs.MimeType(pngName) == fs.MimeTypePNG; m.hasPreviewImage {
 		return true

internal/photoprism/mediafile_meta_test.go

@@ -18,7 +18,7 @@ func TestMediaFile_HasSidecarJson(t *testing.T) {
 	t.Run("false", func(t *testing.T) {
 		conf := config.TestConfig()
 
-		mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg")
+		mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_wood.jpg")
 
 		if err != nil {
 			t.Fatal(err)

internal/photoprism/mediafile_related.go

@@ -55,6 +55,13 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e
 		matches = append(matches, name)
 	}
 
+	// check for an embedded video in the media file
+	if embeddedVideoName, err := m.ExtractEmbeddedVideo(); err != nil {
+		return result, err
+	} else if embeddedVideoName != "" {
+		matches = append(matches, embeddedVideoName)
+	}
+
 	isHEIC := false
 
 	for _, fileName := range matches {

internal/photoprism/mediafile_test.go

@@ -2610,3 +2610,66 @@ func TestMediaFile_Duration(t *testing.T) {
 		}
 	})
 }
+
+func TestMediaFile_ExtractEmbeddedVideo(t *testing.T) {
+	conf := config.TestConfig()
+	t.Run(
+		"samsung-motion-photo.jpg", func(t *testing.T) {
+			// input image file
+			fileName := filepath.Join(
+				conf.ExamplesPath(),
+				"samsung-motion-photo.jpg",
+			)
+			// expected output video file
+			outputName := filepath.Join(
+				conf.SidecarPath(), "samsung-motion-photo.mp4",
+			)
+
+			_ = os.Remove(outputName)
+
+			mf, err := NewMediaFile(fileName)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// extract the video
+			if embeddedVideoName, err := mf.ExtractEmbeddedVideo(); err != nil {
+				t.Fatal(err)
+			} else if embeddedVideoName == "" {
+				t.Errorf("embeddedVideoName should not be empty")
+			} else {
+				t.Logf(embeddedVideoName)
+				assert.Equal(t, embeddedVideoName, outputName)
+				assert.Truef(
+					t, fs.FileExists(embeddedVideoName),
+					"output file does not exist: %s", embeddedVideoName,
+				)
+
+				_ = os.Remove(outputName)
+			}
+		},
+	)
+
+	t.Run(
+		"beach_sand.jpg", func(t *testing.T) {
+			// input image file
+			fileName := filepath.Join(
+				conf.ExamplesPath(),
+				"beach_sand.jpg",
+			)
+
+			mf, err := NewMediaFile(fileName)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if embeddedVideoName, err := mf.ExtractEmbeddedVideo(); err != nil {
+				t.Fatal(err)
+			} else if embeddedVideoName != "" {
+				t.Errorf("expected embeddedVideoName to be empty")
+			} else {
+				t.Logf(embeddedVideoName)
+			}
+		},
+	)
+}