Live Photos: Add support for playing videos embedded in HEIC images #439

Related Issues:
- Samsung: Initial support for Motion Photos (#439)
- Google: Initial support for Motion Photos (#1739)
- Metadata: Flag Samsung/Google Motion Photos as Live Photos (#2788)

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/api/video.go

@@ -68,8 +68,8 @@ func GetVideo(router *gin.RouterGroup) {
 		conf := get.Config()
 		fileName := photoprism.FileName(f.FileRoot, f.FileName)
 
-		// If file is not a video, try to find and stream embedded video data.
-		if f.MediaType != entity.MediaVideo {
+		// If the file has a hybrid photo/video format, try to find and send the embedded video data.
+		if f.MediaType == entity.MediaLive {
 			if info, videoErr := video.ProbeFile(fileName); info.VideoOffset < 0 || !info.Compatible || videoErr != nil {
 				logError("video", videoErr)
 				log.Warnf("video: no data found in %s", clean.Log(f.FileName))

internal/search/photos.go

@@ -30,7 +30,7 @@ var PhotosColsAll = SelectString(Photo{}, []string{"*"})
 var PhotosColsView = SelectString(Photo{}, SelectCols(GeoResult{}, []string{"*"}))
 
 // FileTypes contains a list of browser-compatible file formats returned by search queries.
-var FileTypes = []string{fs.ImageJPEG.String(), fs.ImagePNG.String(), fs.ImageGIF.String(), fs.ImageAVIF.String(), fs.ImageAVIFS.String(), fs.ImageWebP.String(), fs.VectorSVG.String()}
+var FileTypes = []string{fs.ImageJPEG.String(), fs.ImagePNG.String(), fs.ImageGIF.String(), fs.ImageHEIC.String(), fs.ImageAVIF.String(), fs.ImageAVIFS.String(), fs.ImageWebP.String(), fs.VectorSVG.String()}
 
 // Photos finds PhotoResults based on the search form without checking rights or permissions.
 func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) {

pkg/video/brands.go

@@ -1,9 +1,17 @@
 package video
 
-// ChunkMP4 specifies the start chunk of MP4 video files,
-// it must be followed by a valid subtype chunk.
+import (
+	"errors"
+	"os"
+
+	"github.com/photoprism/photoprism/pkg/fs"
+)
+
+// ChunkFTYP specifies the start chunk of the ISO base media
+// format, it must be followed by a valid subtype chunk.
+// https://en.wikipedia.org/wiki/ISO_base_media_file_format
 var (
-	ChunkMP4  = Chunk{'f', 't', 'y', 'p'}
+	ChunkFTYP = Chunk{'f', 't', 'y', 'p'}
 	ChunkQT   = Chunk{'q', 't', ' ', ' '}
 	ChunkISOM = Chunk{'i', 's', 'o', 'm'}
 	ChunkISO2 = Chunk{'i', 's', 'o', '2'}
@@ -50,3 +58,22 @@ var CompatibleBrands = Chunks{
 	ChunkMP42,
 	ChunkMP71,
 }
+
+// FileTypeOffset returns the file type start offset, or -1 if it was not found.
+func FileTypeOffset(fileName string, brands Chunks) (int, error) {
+	if !fs.FileExists(fileName) {
+		return -1, errors.New("file not found")
+	}
+
+	file, err := os.Open(fileName)
+
+	if err != nil {
+		return -1, err
+	}
+
+	defer file.Close()
+
+	index, err := brands.FileTypeOffset(file)
+
+	return index, err
+}

pkg/video/brands_test.go

@@ -0,0 +1,26 @@
+package video
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileTypeOffset(t *testing.T) {
+	t.Run("mp4v-avc1.mp4", func(t *testing.T) {
+		index, err := FileTypeOffset("testdata/mp4v-avc1.mp4", CompatibleBrands)
+		require.NoError(t, err)
+		assert.Equal(t, 0, index)
+	})
+	t.Run("isom-avc1.mp4", func(t *testing.T) {
+		index, err := FileTypeOffset("testdata/isom-avc1.mp4", CompatibleBrands)
+		require.NoError(t, err)
+		assert.Equal(t, 0, index)
+	})
+	t.Run("image-isom-avc1.jpg", func(t *testing.T) {
+		index, err := FileTypeOffset("testdata/image-isom-avc1.jpg", CompatibleBrands)
+		require.NoError(t, err)
+		assert.Equal(t, 23209, index)
+	})
+}

pkg/video/chunk.go

@@ -9,7 +9,6 @@ import (
 	"os"
 
 	"github.com/photoprism/photoprism/pkg/fs"
-
 	"github.com/sunfish-shogi/bufseekio"
 )
 
@@ -46,7 +45,7 @@ func (c Chunk) Equal(b []byte) bool {
 	return bytes.Equal(c.Bytes(), b)
 }
 
-// FileOffset returns the index of the chunk in the specified file, or -1 if it was not found.
+// FileOffset returns the index of the chunk, or -1 if it was not found.
 func (c Chunk) FileOffset(fileName string) (int, error) {
 	if !fs.FileExists(fileName) {
 		return -1, errors.New("file not found")
@@ -65,7 +64,7 @@ func (c Chunk) FileOffset(fileName string) (int, error) {
 	return index, err
 }
 
-// DataOffset returns the index of the chunk in f, or -1 if it was not found.
+// DataOffset returns the index of the chunk in file, or -1 if it was not found.
 func (c Chunk) DataOffset(file io.ReadSeeker) (int, error) {
 	if file == nil {
 		return -1, errors.New("file is nil")

pkg/video/chunks.go

@@ -1,9 +1,33 @@
 package video
 
+import (
+	"bytes"
+	"errors"
+	"io"
+
+	"github.com/sunfish-shogi/bufseekio"
+)
+
 // Chunks represents a list of file chunks.
 type Chunks []Chunk
 
-// ContainsAny checks if at least one common chunk exists.
+// Contains tests if the chunk is contained in this list.
+func (c Chunks) Contains(s [4]byte) bool {
+	if len(c) == 0 {
+		return false
+	}
+
+	// Find matches.
+	for i := range c {
+		if s == c[i] {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ContainsAny checks if at least one common chunk exists in this list.
 func (c Chunks) ContainsAny(b [][4]byte) bool {
 	if len(c) == 0 || len(b) == 0 {
 		return false
@@ -21,3 +45,49 @@ func (c Chunks) ContainsAny(b [][4]byte) bool {
 	// Not found.
 	return false
 }
+
+// FileTypeOffset returns the file type start offset in f, or -1 if it was not found.
+func (c Chunks) FileTypeOffset(file io.ReadSeeker) (int, error) {
+	if file == nil {
+		return -1, errors.New("file is nil")
+	}
+
+	ftyp := ChunkFTYP.Bytes()
+	blockSize := 128 * 1024
+	buffer := make([]byte, blockSize)
+
+	// Create buffered read seeker.
+	r := bufseekio.NewReadSeeker(file, blockSize, 8)
+
+	// Index offset.
+	var offset int
+
+	// Search in batches.
+	for {
+		n, err := r.Read(buffer)
+		buffer = buffer[:n]
+
+		if err != nil {
+			if err != io.EOF {
+				return -1, err
+			}
+
+			break
+		} else if n == 0 {
+			break
+		}
+
+		// Find ftyp chunk.
+		if i := bytes.Index(buffer, ftyp); i < 0 {
+			// Not found.
+		} else if j := i + 4; j < 8 || len(buffer) < j+4 {
+			// Skip.
+		} else if k := j + 4; c.Contains(*(*[4]byte)(buffer[j:k])) {
+			return offset + i - 4, nil
+		}
+
+		offset += n
+	}
+
+	return -1, nil
+}

pkg/video/chunks_test.go

@@ -13,29 +13,29 @@ import (
 
 func TestChunk_TypeCast(t *testing.T) {
 	t.Run("String", func(t *testing.T) {
-		assert.Equal(t, "ftyp", ChunkMP4.String())
+		assert.Equal(t, "ftyp", ChunkFTYP.String())
 	})
 	t.Run("Hex", func(t *testing.T) {
-		assert.Equal(t, "0x66747970", ChunkMP4.Hex())
+		assert.Equal(t, "0x66747970", ChunkFTYP.Hex())
 	})
 	t.Run("Uint32", func(t *testing.T) {
-		assert.Equal(t, uint32(0x66747970), ChunkMP4.Uint32())
+		assert.Equal(t, uint32(0x66747970), ChunkFTYP.Uint32())
 	})
 }
 
 func TestChunk_FileOffset(t *testing.T) {
 	t.Run("mp4v-avc1.mp4", func(t *testing.T) {
-		index, err := ChunkMP4.FileOffset("testdata/mp4v-avc1.mp4")
+		index, err := ChunkFTYP.FileOffset("testdata/mp4v-avc1.mp4")
 		require.NoError(t, err)
 		assert.Equal(t, 4, index)
 	})
 	t.Run("isom-avc1.mp4", func(t *testing.T) {
-		index, err := ChunkMP4.FileOffset("testdata/isom-avc1.mp4")
+		index, err := ChunkFTYP.FileOffset("testdata/isom-avc1.mp4")
 		require.NoError(t, err)
 		assert.Equal(t, 4, index)
 	})
 	t.Run("image-isom-avc1.jpg", func(t *testing.T) {
-		index, err := ChunkMP4.FileOffset("testdata/image-isom-avc1.jpg")
+		index, err := ChunkFTYP.FileOffset("testdata/image-isom-avc1.jpg")
 		require.NoError(t, err)
 		assert.Equal(t, 23213, index)
 	})
@@ -69,7 +69,7 @@ func TestChunks(t *testing.T) {
 			t.Fatal("expected to read 4 bytes")
 		}
 
-		assert.Equal(t, ChunkMP4.Bytes(), startChunk[:4])
+		assert.Equal(t, ChunkFTYP.Bytes(), startChunk[:4])
 		assert.Equal(t, ChunkMP4V.Bytes(), subType[:4])
 	})
 	t.Run("isom-avc1.mp4", func(t *testing.T) {
@@ -86,7 +86,7 @@ func TestChunks(t *testing.T) {
 			t.Fatalf("expected to read 12 bytes instead of %d", n)
 		}
 
-		assert.Equal(t, ChunkMP4[:], b[4:8])
+		assert.Equal(t, ChunkFTYP[:], b[4:8])
 		assert.Equal(t, ChunkISOM[:], b[8:12])
 	})
 	t.Run("image-isom-avc1.jpg", func(t *testing.T) {
@@ -103,18 +103,27 @@ func TestChunks(t *testing.T) {
 			t.Fatalf("expected to read 12 bytes instead of %d", n)
 		}
 
-		assert.NotEqual(t, ChunkMP4, *(*[4]byte)(b[4:8]))
+		assert.NotEqual(t, ChunkFTYP, *(*[4]byte)(b[4:8]))
 		assert.NotEqual(t, ChunkISOM, *(*[4]byte)(b[8:12]))
 	})
 }
 
+func TestChunks_Contains(t *testing.T) {
+	t.Run("Found", func(t *testing.T) {
+		assert.True(t, CompatibleBrands.Contains(ChunkMP41))
+	})
+	t.Run("NotFound", func(t *testing.T) {
+		assert.False(t, CompatibleBrands.Contains(ChunkFTYP))
+	})
+}
+
 func TestChunks_ContainsAny(t *testing.T) {
 	t.Run("Found", func(t *testing.T) {
 		chunks := [][4]byte{ChunkMP41, ChunkMP42}
 		assert.True(t, CompatibleBrands.ContainsAny(chunks))
 	})
 	t.Run("NotFound", func(t *testing.T) {
-		chunks := [][4]byte{ChunkMP4}
+		chunks := [][4]byte{ChunkFTYP}
 		assert.False(t, CompatibleBrands.ContainsAny(chunks))
 	})
 }

pkg/video/probe.go

@@ -71,13 +71,13 @@ func Probe(file io.ReadSeeker) (info Info, err error) {
 		return info, err
 	}
 
-	// Find start chunk.
-	if offset, findErr := ChunkMP4.DataOffset(file); findErr != nil {
+	// Find file type start offset.
+	if offset, findErr := CompatibleBrands.FileTypeOffset(file); findErr != nil {
 		return info, findErr
-	} else if offset < 4 {
+	} else if offset < 0 {
 		return info, nil
 	} else {
-		info.VideoOffset = int64(offset) - 4
+		info.VideoOffset = int64(offset)
 	}
 
 	// Ignore any data before the video offset.

pkg/video/probe_test.go

@@ -24,33 +24,6 @@ func TestProbeFile(t *testing.T) {
 		require.Error(t, err)
 		require.NotNil(t, info)
 	})
-	/*t.Run("motion-photo.heif", func(t *testing.T) {
-		fileName := "testdata/motion-photo.heif"
-		info, err := ProbeFile(fileName)
-		require.NoError(t, err)
-		require.NotNil(t, info)
-
-		assert.Equal(t, fileName, info.FileName)
-		assert.Equal(t, int64(3366300), info.FileSize)
-		assert.Equal(t, fs.ImageHEIC, info.FileType)
-		assert.Equal(t, MOV, info.VideoType)
-		assert.Equal(t, int64(0), info.VideoOffset)
-		assert.Equal(t, int64(-1), info.ThumbOffset)
-		assert.Equal(t, media.Image, info.MediaType)
-		assert.Equal(t, CodecHVC, info.VideoCodec)
-		assert.Equal(t, fs.MimeTypeMOV, info.VideoMimeType)
-		assert.Equal(t, "", info.VideoContentType())
-		assert.Equal(t, "1.166666666s", info.Duration.String())
-		assert.InEpsilon(t, 1.166, info.Duration.Seconds(), 0.01)
-		assert.Equal(t, 5, info.Tracks)
-		assert.Equal(t, 0, info.VideoWidth)
-		assert.Equal(t, 0, info.VideoHeight)
-		assert.Equal(t, 35, info.Frames)
-		assert.Equal(t, 30.0, info.FPS)
-		assert.Equal(t, false, info.Encrypted)
-		assert.Equal(t, false, info.FastStart)
-		assert.Equal(t, true, info.Compatible)
-	})*/
 	t.Run("mp4v-avc1.mp4", func(t *testing.T) {
 		fileName := "testdata/mp4v-avc1.mp4"
 		info, err := ProbeFile(fileName)