diff --git a/internal/api/video.go b/internal/api/video.go index 68ed438f5..75dd321fa 100644 --- a/internal/api/video.go +++ b/internal/api/video.go @@ -68,8 +68,8 @@ func GetVideo(router *gin.RouterGroup) { conf := get.Config() fileName := photoprism.FileName(f.FileRoot, f.FileName) - // If file is not a video, try to find and stream embedded video data. - if f.MediaType != entity.MediaVideo { + // If the file has a hybrid photo/video format, try to find and send the embedded video data. + if f.MediaType == entity.MediaLive { if info, videoErr := video.ProbeFile(fileName); info.VideoOffset < 0 || !info.Compatible || videoErr != nil { logError("video", videoErr) log.Warnf("video: no data found in %s", clean.Log(f.FileName)) diff --git a/internal/search/photos.go b/internal/search/photos.go index 884f9544d..3399f4f54 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -30,7 +30,7 @@ var PhotosColsAll = SelectString(Photo{}, []string{"*"}) var PhotosColsView = SelectString(Photo{}, SelectCols(GeoResult{}, []string{"*"})) // FileTypes contains a list of browser-compatible file formats returned by search queries. -var FileTypes = []string{fs.ImageJPEG.String(), fs.ImagePNG.String(), fs.ImageGIF.String(), fs.ImageAVIF.String(), fs.ImageAVIFS.String(), fs.ImageWebP.String(), fs.VectorSVG.String()} +var FileTypes = []string{fs.ImageJPEG.String(), fs.ImagePNG.String(), fs.ImageGIF.String(), fs.ImageHEIC.String(), fs.ImageAVIF.String(), fs.ImageAVIFS.String(), fs.ImageWebP.String(), fs.VectorSVG.String()} // Photos finds PhotoResults based on the search form without checking rights or permissions. func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { diff --git a/pkg/video/chunks_mp4.go b/pkg/video/brands.go similarity index 63% rename from pkg/video/chunks_mp4.go rename to pkg/video/brands.go index f6722aaa3..ae1cf7dd9 100644 --- a/pkg/video/chunks_mp4.go +++ b/pkg/video/brands.go @@ -1,9 +1,17 @@ package video -// ChunkMP4 specifies the start chunk of MP4 video files, -// it must be followed by a valid subtype chunk. +import ( + "errors" + "os" + + "github.com/photoprism/photoprism/pkg/fs" +) + +// ChunkFTYP specifies the start chunk of the ISO base media +// format, it must be followed by a valid subtype chunk. +// https://en.wikipedia.org/wiki/ISO_base_media_file_format var ( - ChunkMP4 = Chunk{'f', 't', 'y', 'p'} + ChunkFTYP = Chunk{'f', 't', 'y', 'p'} ChunkQT = Chunk{'q', 't', ' ', ' '} ChunkISOM = Chunk{'i', 's', 'o', 'm'} ChunkISO2 = Chunk{'i', 's', 'o', '2'} @@ -50,3 +58,22 @@ var CompatibleBrands = Chunks{ ChunkMP42, ChunkMP71, } + +// FileTypeOffset returns the file type start offset, or -1 if it was not found. +func FileTypeOffset(fileName string, brands Chunks) (int, error) { + if !fs.FileExists(fileName) { + return -1, errors.New("file not found") + } + + file, err := os.Open(fileName) + + if err != nil { + return -1, err + } + + defer file.Close() + + index, err := brands.FileTypeOffset(file) + + return index, err +} diff --git a/pkg/video/brands_test.go b/pkg/video/brands_test.go new file mode 100644 index 000000000..45866b536 --- /dev/null +++ b/pkg/video/brands_test.go @@ -0,0 +1,26 @@ +package video + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFileTypeOffset(t *testing.T) { + t.Run("mp4v-avc1.mp4", func(t *testing.T) { + index, err := FileTypeOffset("testdata/mp4v-avc1.mp4", CompatibleBrands) + require.NoError(t, err) + assert.Equal(t, 0, index) + }) + t.Run("isom-avc1.mp4", func(t *testing.T) { + index, err := FileTypeOffset("testdata/isom-avc1.mp4", CompatibleBrands) + require.NoError(t, err) + assert.Equal(t, 0, index) + }) + t.Run("image-isom-avc1.jpg", func(t *testing.T) { + index, err := FileTypeOffset("testdata/image-isom-avc1.jpg", CompatibleBrands) + require.NoError(t, err) + assert.Equal(t, 23209, index) + }) +} diff --git a/pkg/video/chunk.go b/pkg/video/chunk.go index 45b2d76bf..65a52483e 100644 --- a/pkg/video/chunk.go +++ b/pkg/video/chunk.go @@ -9,7 +9,6 @@ import ( "os" "github.com/photoprism/photoprism/pkg/fs" - "github.com/sunfish-shogi/bufseekio" ) @@ -46,7 +45,7 @@ func (c Chunk) Equal(b []byte) bool { return bytes.Equal(c.Bytes(), b) } -// FileOffset returns the index of the chunk in the specified file, or -1 if it was not found. +// FileOffset returns the index of the chunk, or -1 if it was not found. func (c Chunk) FileOffset(fileName string) (int, error) { if !fs.FileExists(fileName) { return -1, errors.New("file not found") @@ -65,7 +64,7 @@ func (c Chunk) FileOffset(fileName string) (int, error) { return index, err } -// DataOffset returns the index of the chunk in f, or -1 if it was not found. +// DataOffset returns the index of the chunk in file, or -1 if it was not found. func (c Chunk) DataOffset(file io.ReadSeeker) (int, error) { if file == nil { return -1, errors.New("file is nil") diff --git a/pkg/video/chunks.go b/pkg/video/chunks.go index fa91ecca3..13c25d549 100644 --- a/pkg/video/chunks.go +++ b/pkg/video/chunks.go @@ -1,9 +1,33 @@ package video +import ( + "bytes" + "errors" + "io" + + "github.com/sunfish-shogi/bufseekio" +) + // Chunks represents a list of file chunks. type Chunks []Chunk -// ContainsAny checks if at least one common chunk exists. +// Contains tests if the chunk is contained in this list. +func (c Chunks) Contains(s [4]byte) bool { + if len(c) == 0 { + return false + } + + // Find matches. + for i := range c { + if s == c[i] { + return true + } + } + + return false +} + +// ContainsAny checks if at least one common chunk exists in this list. func (c Chunks) ContainsAny(b [][4]byte) bool { if len(c) == 0 || len(b) == 0 { return false @@ -21,3 +45,49 @@ func (c Chunks) ContainsAny(b [][4]byte) bool { // Not found. return false } + +// FileTypeOffset returns the file type start offset in f, or -1 if it was not found. +func (c Chunks) FileTypeOffset(file io.ReadSeeker) (int, error) { + if file == nil { + return -1, errors.New("file is nil") + } + + ftyp := ChunkFTYP.Bytes() + blockSize := 128 * 1024 + buffer := make([]byte, blockSize) + + // Create buffered read seeker. + r := bufseekio.NewReadSeeker(file, blockSize, 8) + + // Index offset. + var offset int + + // Search in batches. + for { + n, err := r.Read(buffer) + buffer = buffer[:n] + + if err != nil { + if err != io.EOF { + return -1, err + } + + break + } else if n == 0 { + break + } + + // Find ftyp chunk. + if i := bytes.Index(buffer, ftyp); i < 0 { + // Not found. + } else if j := i + 4; j < 8 || len(buffer) < j+4 { + // Skip. + } else if k := j + 4; c.Contains(*(*[4]byte)(buffer[j:k])) { + return offset + i - 4, nil + } + + offset += n + } + + return -1, nil +} diff --git a/pkg/video/chunks_test.go b/pkg/video/chunks_test.go index c9efb8a2b..08f80c0cc 100644 --- a/pkg/video/chunks_test.go +++ b/pkg/video/chunks_test.go @@ -13,29 +13,29 @@ import ( func TestChunk_TypeCast(t *testing.T) { t.Run("String", func(t *testing.T) { - assert.Equal(t, "ftyp", ChunkMP4.String()) + assert.Equal(t, "ftyp", ChunkFTYP.String()) }) t.Run("Hex", func(t *testing.T) { - assert.Equal(t, "0x66747970", ChunkMP4.Hex()) + assert.Equal(t, "0x66747970", ChunkFTYP.Hex()) }) t.Run("Uint32", func(t *testing.T) { - assert.Equal(t, uint32(0x66747970), ChunkMP4.Uint32()) + assert.Equal(t, uint32(0x66747970), ChunkFTYP.Uint32()) }) } func TestChunk_FileOffset(t *testing.T) { t.Run("mp4v-avc1.mp4", func(t *testing.T) { - index, err := ChunkMP4.FileOffset("testdata/mp4v-avc1.mp4") + index, err := ChunkFTYP.FileOffset("testdata/mp4v-avc1.mp4") require.NoError(t, err) assert.Equal(t, 4, index) }) t.Run("isom-avc1.mp4", func(t *testing.T) { - index, err := ChunkMP4.FileOffset("testdata/isom-avc1.mp4") + index, err := ChunkFTYP.FileOffset("testdata/isom-avc1.mp4") require.NoError(t, err) assert.Equal(t, 4, index) }) t.Run("image-isom-avc1.jpg", func(t *testing.T) { - index, err := ChunkMP4.FileOffset("testdata/image-isom-avc1.jpg") + index, err := ChunkFTYP.FileOffset("testdata/image-isom-avc1.jpg") require.NoError(t, err) assert.Equal(t, 23213, index) }) @@ -69,7 +69,7 @@ func TestChunks(t *testing.T) { t.Fatal("expected to read 4 bytes") } - assert.Equal(t, ChunkMP4.Bytes(), startChunk[:4]) + assert.Equal(t, ChunkFTYP.Bytes(), startChunk[:4]) assert.Equal(t, ChunkMP4V.Bytes(), subType[:4]) }) t.Run("isom-avc1.mp4", func(t *testing.T) { @@ -86,7 +86,7 @@ func TestChunks(t *testing.T) { t.Fatalf("expected to read 12 bytes instead of %d", n) } - assert.Equal(t, ChunkMP4[:], b[4:8]) + assert.Equal(t, ChunkFTYP[:], b[4:8]) assert.Equal(t, ChunkISOM[:], b[8:12]) }) t.Run("image-isom-avc1.jpg", func(t *testing.T) { @@ -103,18 +103,27 @@ func TestChunks(t *testing.T) { t.Fatalf("expected to read 12 bytes instead of %d", n) } - assert.NotEqual(t, ChunkMP4, *(*[4]byte)(b[4:8])) + assert.NotEqual(t, ChunkFTYP, *(*[4]byte)(b[4:8])) assert.NotEqual(t, ChunkISOM, *(*[4]byte)(b[8:12])) }) } +func TestChunks_Contains(t *testing.T) { + t.Run("Found", func(t *testing.T) { + assert.True(t, CompatibleBrands.Contains(ChunkMP41)) + }) + t.Run("NotFound", func(t *testing.T) { + assert.False(t, CompatibleBrands.Contains(ChunkFTYP)) + }) +} + func TestChunks_ContainsAny(t *testing.T) { t.Run("Found", func(t *testing.T) { chunks := [][4]byte{ChunkMP41, ChunkMP42} assert.True(t, CompatibleBrands.ContainsAny(chunks)) }) t.Run("NotFound", func(t *testing.T) { - chunks := [][4]byte{ChunkMP4} + chunks := [][4]byte{ChunkFTYP} assert.False(t, CompatibleBrands.ContainsAny(chunks)) }) } diff --git a/pkg/video/probe.go b/pkg/video/probe.go index ac04c2945..83c1c3ca0 100644 --- a/pkg/video/probe.go +++ b/pkg/video/probe.go @@ -71,13 +71,13 @@ func Probe(file io.ReadSeeker) (info Info, err error) { return info, err } - // Find start chunk. - if offset, findErr := ChunkMP4.DataOffset(file); findErr != nil { + // Find file type start offset. + if offset, findErr := CompatibleBrands.FileTypeOffset(file); findErr != nil { return info, findErr - } else if offset < 4 { + } else if offset < 0 { return info, nil } else { - info.VideoOffset = int64(offset) - 4 + info.VideoOffset = int64(offset) } // Ignore any data before the video offset. diff --git a/pkg/video/probe_test.go b/pkg/video/probe_test.go index 82bd5d8ce..2c28cd2a1 100644 --- a/pkg/video/probe_test.go +++ b/pkg/video/probe_test.go @@ -24,33 +24,6 @@ func TestProbeFile(t *testing.T) { require.Error(t, err) require.NotNil(t, info) }) - /*t.Run("motion-photo.heif", func(t *testing.T) { - fileName := "testdata/motion-photo.heif" - info, err := ProbeFile(fileName) - require.NoError(t, err) - require.NotNil(t, info) - - assert.Equal(t, fileName, info.FileName) - assert.Equal(t, int64(3366300), info.FileSize) - assert.Equal(t, fs.ImageHEIC, info.FileType) - assert.Equal(t, MOV, info.VideoType) - assert.Equal(t, int64(0), info.VideoOffset) - assert.Equal(t, int64(-1), info.ThumbOffset) - assert.Equal(t, media.Image, info.MediaType) - assert.Equal(t, CodecHVC, info.VideoCodec) - assert.Equal(t, fs.MimeTypeMOV, info.VideoMimeType) - assert.Equal(t, "", info.VideoContentType()) - assert.Equal(t, "1.166666666s", info.Duration.String()) - assert.InEpsilon(t, 1.166, info.Duration.Seconds(), 0.01) - assert.Equal(t, 5, info.Tracks) - assert.Equal(t, 0, info.VideoWidth) - assert.Equal(t, 0, info.VideoHeight) - assert.Equal(t, 35, info.Frames) - assert.Equal(t, 30.0, info.FPS) - assert.Equal(t, false, info.Encrypted) - assert.Equal(t, false, info.FastStart) - assert.Equal(t, true, info.Compatible) - })*/ t.Run("mp4v-avc1.mp4", func(t *testing.T) { fileName := "testdata/mp4v-avc1.mp4" info, err := ProbeFile(fileName)