luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

WebP: Add support for indexing and playing animations #3197 #668

Browse source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2023-02-22 20:58:21 +01:00
parent 4c90ba84e2
commit b44b8d52c1
20 changed files with 139 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frontend/src/common/caniuse.js
View file

@ -39,7 +39,7 @@ export const canUseVP9 = canUseVideo // WebM VP9
export const canUseAv1 = canUseVideo // AV1, Main Profile, Level 4.0 Main Tier, 8-bit
? !!document.createElement("video").canPlayType('video/webm; codecs="av01.0.08M.08"')
: false;
export const canUseWebm = canUseVideo
export const canUseWebM = canUseVideo
? !!document.createElement("video").canPlayType("video/webm")
: false;
export const canUseHevc = canUseVideo

4
frontend/src/model/photo.js
View file

@ -37,7 +37,7 @@ import { $gettext } from "common/vm";
import Clipboard from "common/clipboard";
import download from "common/download";
import * as src from "common/src";
import { canUseOGV, canUseVP8, canUseVP9, canUseAv1, canUseWebm, canUseHevc } from "common/caniuse";
import { canUseOGV, canUseVP8, canUseVP9, canUseAv1, canUseWebM, canUseHevc } from "common/caniuse";
export const CodecOGV = "ogv";
export const CodecVP8 = "vp8";
@ -500,7 +500,7 @@ export class Photo extends RestModel {
videoFormat = CodecVP9;
} else if (canUseAv1 && file.Codec === CodecAv1) {
videoFormat = FormatAv1;
} else if (canUseWebm && file.FileType === FormatWebM) {
} else if (canUseWebM && file.FileType === FormatWebM) {
videoFormat = FormatWebM;
}

6
frontend/tests/unit/common/caniuse_test.js
View file

@ -7,7 +7,7 @@ import {
canUseVideo,
canUseVP8,
canUseVP9,
canUseWebm,
canUseWebM,
} from "common/caniuse";
let chai = require("chai/chai");
@ -38,8 +38,8 @@ describe("common/caniuse", () => {
assert.equal(canUseAv1, true);
});
it("canUseWebm", () => {
assert.equal(canUseWebm, true);
it("canUseWebM", () => {
assert.equal(canUseWebM, true);
});
it("canUseHevc", () => {

2
go.sum
View file

@ -117,6 +117,8 @@ github.com/dsoprea/go-utility/v2 v2.0.0-20221003142440-7a1927d49d9d/go.mod h1:LV
github.com/dsoprea/go-utility/v2 v2.0.0-20221003160719-7bc88537c05e/go.mod h1:VZ7cB0pTjm1ADBWhJUOHESu4ZYy9JN+ZPqjfiW09EPU=
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 h1:DilThiXje0z+3UQ5YjYiSRRzVdtamFpvBQXKwMglWqw=
github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349/go.mod h1:4GC5sXji84i/p+irqghpPFZBF8tRN/Q7+700G0/DLe8=
github.com/dsoprea/go-webp-image-structure v0.0.0 h1:n7yGn01OL0U1M494TDY7Hwr5b1180AUyhOCwZiwlZUs=
github.com/dsoprea/go-webp-image-structure v0.0.0/go.mod h1:gnr1kACXpLVe19Swg3zgmGhTCj3cp+MU3RkQYiGh47Q=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

7
internal/entity/file.go
View file

@ -555,6 +555,11 @@ func (m *File) NoPNG() bool {
return fs.ImagePNG.NotEqual(m.FileType)
}
// Type returns the file type.
func (m *File) Type() fs.Type {
return fs.Type(m.FileType)
}
// Links returns all share links for this entity.
func (m *File) Links() Links {
return FindLinks("", m.FileUID)
@ -617,7 +622,7 @@ func (m *File) HasWatermark() bool {
// IsAnimated returns true if the file has animated image frames.
func (m *File) IsAnimated() bool {
return m.FileFrames > 1 && media.Image.Equal(m.MediaType)
return (m.FileFrames > 1 || m.FileDuration > 0) && media.Image.Equal(m.MediaType)
}
// ColorProfile returns the ICC color profile name if any.

2
internal/meta/data.go
View file

@ -26,7 +26,7 @@ type Data struct {
Duration time.Duration `meta:"Duration,MediaDuration,TrackDuration,PreviewDuration"`
FPS float64 `meta:"VideoFrameRate,VideoAvgFrameRate"`
Frames int `meta:"FrameCount,AnimationFrames"`
Codec string `meta:"CompressorID,VideoCodecID,CodecID,OtherFormat,MajorBrand,FileType"`
Codec string `meta:"CompressorID,VideoCodecID,CodecID,OtherFormat,FileType"`
Title string `meta:"Headline,Title" xmp:"dc:title" dc:"title,title.Alt"`
Subject string `meta:"Subject,PersonInImage,ObjectName,HierarchicalSubject,CatalogSets" xmp:"Subject"`
Keywords Keywords `meta:"Keywords"`

2
internal/photoprism/convert_image.go
View file

@ -99,7 +99,7 @@ func (c *Convert) ToImage(f *MediaFile, force bool) (*MediaFile, error) {
if err == nil {
log.Infof("convert: %s created in %s (%s)", clean.Log(filepath.Base(imageName)), time.Since(start), f.FileType())
return NewMediaFile(imageName)
} else if !f.IsTIFF() {
} else if !f.IsTIFF() && !f.IsWebP() {
// See https://github.com/photoprism/photoprism/issues/1612
// for TIFF file format compatibility.
return nil, err

6
internal/photoprism/convert_image_jpeg.go
View file

@ -22,12 +22,12 @@ func (c *Convert) JpegConvertCommands(f *MediaFile, jpegName string, xmpName str
maxSize := strconv.Itoa(c.conf.JpegSize())
// Apple Scriptable image processing system: https://ss64.com/osx/sips.html
if (f.IsRaw() || f.IsHEIC() || f.IsAVIF()) && c.conf.SipsEnabled() && c.sipsBlacklist.Allow(fileExt) {
if (f.IsRaw() || f.IsHEIF()) && c.conf.SipsEnabled() && c.sipsBlacklist.Allow(fileExt) {
result = append(result, exec.Command(c.conf.SipsBin(), "-Z", maxSize, "-s", "format", "jpeg", "--out", jpegName, f.FileName()))
}
// Extract a still image to be used as preview.
if f.IsAnimated() && c.conf.FFmpegEnabled() {
if f.IsAnimated() && !f.IsWebP() && c.conf.FFmpegEnabled() {
// Use "ffmpeg" to extract a JPEG still image from the video.
result = append(result, exec.Command(c.conf.FFmpegBin(), "-y", "-i", f.FileName(), "-ss", ffmpeg.PreviewTimeOffset(f.Duration()), "-vframes", "1", jpegName))
}
@ -95,7 +95,7 @@ func (c *Convert) JpegConvertCommands(f *MediaFile, jpegName string, xmpName str
// Try ImageMagick for other image file formats if allowed.
if c.conf.ImageMagickEnabled() && c.imagemagickBlacklist.Allow(fileExt) &&
(f.IsImage() && !f.IsJpegXL() && !f.IsRaw() && !f.IsAnimated() || f.IsVector() && c.conf.VectorEnabled()) {
(f.IsImage() && !f.IsJpegXL() && !f.IsRaw() && !f.IsHEIF() || f.IsVector() && c.conf.VectorEnabled()) {
quality := fmt.Sprintf("%d", c.conf.JpegQuality())
resize := fmt.Sprintf("%dx%d>", c.conf.JpegSize(), c.conf.JpegSize())
args := []string{f.FileName(), "-flatten", "-resize", resize, "-quality", quality, jpegName}

6
internal/photoprism/convert_image_png.go
View file

@ -21,12 +21,12 @@ func (c *Convert) PngConvertCommands(f *MediaFile, pngName string) (result []*ex
maxSize := strconv.Itoa(c.conf.PngSize())
// Apple Scriptable image processing system: https://ss64.com/osx/sips.html
if (f.IsRaw() || f.IsHEIC() || f.IsAVIF()) && c.conf.SipsEnabled() && c.sipsBlacklist.Allow(fileExt) {
if (f.IsRaw() || f.IsHEIF()) && c.conf.SipsEnabled() && c.sipsBlacklist.Allow(fileExt) {
result = append(result, exec.Command(c.conf.SipsBin(), "-Z", maxSize, "-s", "format", "png", "--out", pngName, f.FileName()))
}
// Extract a video still image that can be used as preview.
if f.IsAnimated() && c.conf.FFmpegEnabled() {
if f.IsAnimated() && !f.IsWebP() && c.conf.FFmpegEnabled() {
// Use "ffmpeg" to extract a PNG still image from the video.
result = append(result, exec.Command(c.conf.FFmpegBin(), "-y", "-i", f.FileName(), "-ss", ffmpeg.PreviewTimeOffset(f.Duration()), "-vframes", "1", pngName))
}
@ -43,7 +43,7 @@ func (c *Convert) PngConvertCommands(f *MediaFile, pngName string) (result []*ex
// Try ImageMagick for other image file formats if allowed.
if c.conf.ImageMagickEnabled() && c.imagemagickBlacklist.Allow(fileExt) &&
(f.IsImage() && !f.IsJpegXL() && !f.IsRaw() && !f.IsAnimated() || f.IsVector() && c.conf.VectorEnabled()) {
(f.IsImage() && !f.IsJpegXL() && !f.IsRaw() && !f.IsHEIF() || f.IsVector() && c.conf.VectorEnabled()) {
resize := fmt.Sprintf("%dx%d>", c.conf.PngSize(), c.conf.PngSize())
args := []string{f.FileName(), "-flatten", "-resize", resize, pngName}
result = append(result, exec.Command(c.conf.ImageMagickBin(), args...))

28
internal/photoprism/convert_video_avc.go
View file

@ -41,12 +41,13 @@ func (c *Convert) ToAvc(f *MediaFile, encoder ffmpeg.AvcEncoder, noMutex, force
return nil, fmt.Errorf("convert: transcoding disabled in read-only mode (%s)", f.RootRelName())
}
if c.conf.DisableFFmpeg() {
return nil, fmt.Errorf("convert: ffmpeg is disabled for transcoding %s", f.RootRelName())
}
fileName := f.RelName(c.conf.OriginalsPath())
avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.ExtAVC)
if f.IsAnimatedImage() {
avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.ExtMP4)
} else {
avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.ExtAVC)
}
cmd, useMutex, err := c.AvcConvertCommand(f, avcName, encoder)
@ -131,6 +132,7 @@ func (c *Convert) ToAvc(f *MediaFile, encoder ffmpeg.AvcEncoder, noMutex, force
// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC.
func (c *Convert) AvcConvertCommand(f *MediaFile, avcName string, encoder ffmpeg.AvcEncoder) (result *exec.Cmd, useMutex bool, err error) {
fileExt := f.Extension()
fileName := f.FileName()
bitrate := c.AvcBitrate(f)
ffmpegBin := c.conf.FFmpegBin()
@ -140,14 +142,22 @@ func (c *Convert) AvcConvertCommand(f *MediaFile, avcName string, encoder ffmpeg
return nil, false, fmt.Errorf("convert: %s video filename is empty - possible bug", f.FileType())
case bitrate == "":
return nil, false, fmt.Errorf("convert: transcoding bitrate is empty - possible bug")
case ffmpegBin == "":
return nil, false, fmt.Errorf("convert: ffmpeg must be installed to transcode %s", clean.Log(f.BaseName()))
case c.conf.DisableFFmpeg():
return nil, false, fmt.Errorf("convert: ffmpeg must be enabled to transcode %s", clean.Log(f.BaseName()))
case !f.IsAnimated():
return nil, false, fmt.Errorf("convert: file type %s of %s cannot be transcoded", f.FileType(), clean.Log(f.BaseName()))
}
// Transcode animated WebP images with ImageMagick.
if f.IsWebP() && c.conf.ImageMagickEnabled() && c.imagemagickBlacklist.Allow(fileExt) {
return exec.Command(c.conf.ImageMagickBin(), f.FileName(), avcName), false, nil
}
// Transcode all other formats with FFmpeg.
if ffmpegBin == "" {
return nil, false, fmt.Errorf("convert: ffmpeg must be installed to transcode %s", clean.Log(f.BaseName()))
} else if c.conf.DisableFFmpeg() {
return nil, false, fmt.Errorf("convert: ffmpeg must be enabled to transcode %s", clean.Log(f.BaseName()))
}
return ffmpeg.AvcConvertCommand(fileName, avcName, ffmpegBin, c.AvcBitrate(f), encoder)
}

30
internal/photoprism/convert_video_avc_test.go
View file

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/ffmpeg"
"github.com/photoprism/photoprism/pkg/fs"
)
@ -140,7 +141,7 @@ func TestConvert_AvcConvertCommand(t *testing.T) {
conf := config.TestConfig()
convert := NewConvert(conf)
t.Run(".mp4", func(t *testing.T) {
t.Run("MP4", func(t *testing.T) {
fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4")
mf, err := NewMediaFile(fileName)
@ -153,10 +154,11 @@ func TestConvert_AvcConvertCommand(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assert.Contains(t, r.Path, "ffmpeg")
assert.Contains(t, r.Args, "mp4")
})
t.Run(".jpg", func(t *testing.T) {
t.Run("JPEG", func(t *testing.T) {
fileName := filepath.Join(conf.ExamplesPath(), "cat_black.jpg")
mf, err := NewMediaFile(fileName)
@ -164,8 +166,30 @@ func TestConvert_AvcConvertCommand(t *testing.T) {
t.Fatal(err)
}
r, _, err := convert.AvcConvertCommand(mf, "avc1", "")
r, useMutex, err := convert.AvcConvertCommand(mf, "avc1", "")
assert.False(t, useMutex)
assert.Error(t, err)
assert.Nil(t, r)
})
t.Run("WebP", func(t *testing.T) {
webpName := "testdata/windows95.webp"
avcName := "windows95.mp4"
mf, err := NewMediaFile(webpName)
if err != nil {
t.Fatal(err)
}
r, useMutex, err := convert.AvcConvertCommand(mf, avcName, ffmpeg.SoftwareEncoder)
if err != nil {
t.Fatal(err)
}
assert.False(t, useMutex)
assert.Contains(t, r.Path, "convert")
assert.Contains(t, r.Args, webpName)
assert.Contains(t, r.Args, avcName)
})
}

11
internal/photoprism/mediafile.go
View file

@ -678,7 +678,12 @@ func (m *MediaFile) IsDNG() bool {
return m.MimeType() == fs.MimeTypeDNG
}
// IsHEIC checks if the file is a High Efficiency Image File Format (HEIC/HEIF) image with a supported file type extension.
// IsHEIF checks if the file is a High Efficiency Image File Format (HEIF) container with a supported file type extension.
func (m *MediaFile) IsHEIF() bool {
return m.IsHEIC() || m.IsHEICS() || m.IsAVIF() || m.IsAVIFS()
}
// IsHEIC checks if the file is a High Efficiency Image Container (HEIC) image with a supported file type extension.
func (m *MediaFile) IsHEIC() bool {
if t := fs.FileType(m.fileName); t != fs.ImageHEIF && t != fs.ImageHEIC {
return false
@ -735,7 +740,7 @@ func (m *MediaFile) Duration() time.Duration {
// IsAnimatedImage checks if the file is an animated image.
func (m *MediaFile) IsAnimatedImage() bool {
return fs.FileAnimated(m.fileName) && (m.MetaData().Frames > 1 || m.MetaData().Duration > 0)
return fs.IsAnimatedImage(m.fileName) && (m.MetaData().Frames > 1 || m.MetaData().Duration > 0)
}
// IsJSON checks if the file is a JSON sidecar file with a supported file type extension.
@ -869,7 +874,7 @@ func (m *MediaFile) IsLive() bool {
// ExifSupported returns true if parsing exif metadata is supported for the media file type.
func (m *MediaFile) ExifSupported() bool {
return m.IsJpeg() || m.IsRaw() || m.IsHEIC() || m.IsHEICS() || m.IsAVIF() || m.IsAVIFS() || m.IsPNG() || m.IsTIFF()
return m.IsJpeg() || m.IsRaw() || m.IsHEIF() || m.IsPNG() || m.IsTIFF()
}
// IsMedia returns true if this is a media file (photo or video, not sidecar or other).

2
internal/photoprism/mediafile_related.go
View file

@ -87,7 +87,7 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e
} else if f.IsHEIC() {
isHEIC = true
result.Main = f
} else if f.IsAVIF() {
} else if f.IsHEIF() {
result.Main = f
} else if f.IsImage() && !f.IsPreviewImage() {
result.Main = f

19
internal/photoprism/mediafile_test.go
View file

@ -1507,12 +1507,31 @@ func TestMediaFile_IsAnimated(t *testing.T) {
assert.Equal(t, true, f.ExifSupported())
assert.Equal(t, false, f.IsVideo())
assert.Equal(t, false, f.IsGIF())
assert.Equal(t, false, f.IsWebP())
assert.Equal(t, false, f.IsAVIF())
assert.Equal(t, false, f.IsHEIC())
assert.Equal(t, false, f.IsHEICS())
assert.Equal(t, false, f.IsSidecar())
}
})
t.Run("windows95.webp", func(t *testing.T) {
if f, err := NewMediaFile("testdata/windows95.webp"); err != nil {
t.Fatal(err)
} else {
assert.Equal(t, true, f.IsImage())
assert.Equal(t, true, f.IsWebP())
assert.Equal(t, true, f.IsAnimated())
assert.Equal(t, true, f.IsAnimatedImage())
assert.Equal(t, false, f.ExifSupported())
assert.Equal(t, false, f.IsVideo())
assert.Equal(t, false, f.IsGIF())
assert.Equal(t, false, f.IsAVIF())
assert.Equal(t, false, f.IsAVIFS())
assert.Equal(t, false, f.IsHEIC())
assert.Equal(t, false, f.IsHEICS())
assert.Equal(t, false, f.IsSidecar())
}
})
t.Run("example.gif", func(t *testing.T) {
if f, err := NewMediaFile(filepath.Join(cnf.ExamplesPath(), "example.gif")); err != nil {
t.Fatal(err)

BIN
internal/photoprism/testdata/windows95.webp vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 46 KiB

21
internal/photoprism/testdata/windows95.webp.json vendored Normal file
View file

@ -0,0 +1,21 @@
[{
"SourceFile": "windows95.webp",
"ExifToolVersion": 12.40,
"FileName": "windows95.webp",
"Directory": ".",
"FileSize": 47544,
"FileModifyDate": "2023:02:21 16:33:52+00:00",
"FileAccessDate": "2023:02:22 17:55:58+00:00",
"FileInodeChangeDate": "2023:02:22 17:55:58+00:00",
"FilePermissions": 100664,
"FileType": "WEBP",
"FileTypeExtension": "WEBP",
"MIMEType": "image/webp",
"ImageWidth": 500,
"ImageHeight": 313,
"BackgroundColor": "255 255 255 0",
"AnimationLoopCount": 0,
"Duration": 4,
"ImageSize": "500 313",
"Megapixels": 0.1565
}]

1
pkg/fs/file_ext.go
View file

@ -12,6 +12,7 @@ const (
ExtDNG = ".dng"
ExtTHM = ".thm"
ExtAVC = ".avc"
ExtMP4 = ".mp4"
)
// Ext returns all extension of a file name including the dots.

4
pkg/fs/file_type.go
View file

@ -17,8 +17,8 @@ func FileType(fileName string) Type {
return UnknownType
}
// FileAnimated checks if the type associated with the specified filename may be animated.
func FileAnimated(fileName string) bool {
// IsAnimatedImage checks if the type associated with the specified filename may be animated.
func IsAnimatedImage(fileName string) bool {
if t, found := Extensions[LowerExt(fileName)]; found {
return TypeAnimated[t] != ""
}

33
pkg/fs/file_type_test.go
View file

@ -170,35 +170,38 @@ func TestFileType(t *testing.T) {
})
}
func TestFileAnimated(t *testing.T) {
func TestIsAnimatedImage(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
assert.False(t, FileAnimated(""))
assert.False(t, IsAnimatedImage(""))
})
t.Run("JPEG", func(t *testing.T) {
assert.False(t, FileAnimated("testdata/test.jpg"))
assert.False(t, IsAnimatedImage("testdata/test.jpg"))
})
t.Run("RawCRW", func(t *testing.T) {
assert.False(t, FileAnimated("testdata/test (jpg).crw"))
assert.False(t, IsAnimatedImage("testdata/test (jpg).crw"))
})
t.Run("MP4", func(t *testing.T) {
assert.False(t, FileAnimated("file.mp"))
assert.False(t, FileAnimated("file.mp4"))
assert.False(t, IsAnimatedImage("file.mp"))
assert.False(t, IsAnimatedImage("file.mp4"))
})
t.Run("GIF", func(t *testing.T) {
assert.True(t, FileAnimated("file.gif"))
assert.True(t, IsAnimatedImage("file.gif"))
})
t.Run("WebP", func(t *testing.T) {
assert.True(t, IsAnimatedImage("file.webp"))
})
t.Run("PNG", func(t *testing.T) {
assert.True(t, FileAnimated("file.png"))
assert.True(t, FileAnimated("file.apng"))
assert.True(t, FileAnimated("file.pnga"))
assert.True(t, IsAnimatedImage("file.png"))
assert.True(t, IsAnimatedImage("file.apng"))
assert.True(t, IsAnimatedImage("file.pnga"))
})
t.Run("AVIF", func(t *testing.T) {
assert.True(t, FileAnimated("file.avif"))
assert.True(t, FileAnimated("file.avis"))
assert.True(t, FileAnimated("file.avifs"))
assert.True(t, IsAnimatedImage("file.avif"))
assert.True(t, IsAnimatedImage("file.avis"))
assert.True(t, IsAnimatedImage("file.avifs"))
})
t.Run("HEIC", func(t *testing.T) {
assert.True(t, FileAnimated("file.heic"))
assert.True(t, FileAnimated("file.heics"))
assert.True(t, IsAnimatedImage("file.heic"))
assert.True(t, IsAnimatedImage("file.heics"))
})
}

1
pkg/fs/file_types.go
View file

@ -66,6 +66,7 @@ const (
var TypeAnimated = TypeMap{
ImageGIF: MimeTypeGIF,
ImagePNG: MimeTypeAPNG,
ImageWebP: MimeTypeWebP,
ImageAVIF: MimeTypeAVIFS,
ImageAVIFS: MimeTypeAVIFS,
ImageHEIC: MimeTypeHEICS,