luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Videos: Improve Nvidia hardware transcoding support #2125

Browse source 
- successfully tested with NVIDIA Quadro P620 and driver v470.103.01
- the host Linux kernel should run the same driver version Make sure to
- driver names in PHOTOPRISM_FFMPEG_ENCODER have been simplified
- share /dev/nvidia* as shown in our new docker-compose.yml example
This commit is contained in:
Michael Mayer 2022-03-23 13:27:25 +01:00
parent 7c5d6007a0
commit 8c589e3649
8 changed files with 358 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Dockerfile
View file

@ -1,6 +1,8 @@
FROM photoprism/develop:220317-bullseye
# Debian 12, Codename "Bookworm"
FROM photoprism/develop:220323-bookworm
## alternative base images
# FROM photoprism/develop:bullseye # Debian 11, Codename "Bullseye"
# FROM photoprism/develop:buster # Debian 10, Codename "Buster"
# FROM photoprism/develop:impish # Ubuntu 21.10, Codename "Impish Indri"

17
docker-compose.yml
View file

@ -110,16 +110,19 @@ services:
## Run/install on first startup (options: update, gpu, tensorflow, davfs, clitools, clean):
# PHOTOPRISM_INIT: "gpu tensorflow"
## Hardware video transcoding config (optional):
# PHOTOPRISM_FFMPEG_BUFFERS: "64" # FFmpeg capture buffers (default: 32)
# PHOTOPRISM_FFMPEG_BITRATE: "32" # FFmpeg encoding bitrate limit in Mbit/s (default: 50)
# PHOTOPRISM_FFMPEG_ENCODER: "h264_v4l2m2m" # use Video4Linux for AVC transcoding (default: libx264)
# PHOTOPRISM_FFMPEG_ENCODER: "h264_qsv" # use Intel Quick Sync Video for AVC transcoding (default: libx264)
# PHOTOPRISM_FFMPEG_ENCODER: "nvidia" # FFmpeg Encoders ("software", "intel", "nvidia", "apple", "v4l2", "vaapi")
# PHOTOPRISM_FFMPEG_BUFFERS: "64" # FFmpeg capture buffers (default: 32)
# PHOTOPRISM_FFMPEG_BITRATE: "32" # FFmpeg encoding bitrate limit in Mbit/s (default: 50)
## Share hardware devices with FFmpeg and TensorFlow (optional):
# devices:
# - "/dev/dri:/dev/dri"
# - "/dev/nvidia0:/dev/nvidia0"
# - "/dev/dri:/dev/dri" # Intel (h264_qsv)
# - "/dev/nvidia0:/dev/nvidia0" # Nvidia (h264_nvenc)
# - "/dev/nvidiactl:/dev/nvidiactl"
# - "/dev/video11:/dev/video11" # Video4Linux (h264_v4l2m2m)
# - "/dev/nvidia-modeset:/dev/nvidia-modeset"
# - "/dev/nvidia-nvswitchctl:/dev/nvidia-nvswitchctl"
# - "/dev/nvidia-uvm:/dev/nvidia-uvm"
# - "/dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools"
# - "/dev/video11:/dev/video11" # Video4Linux (h264_v4l2m2m)
working_dir: "/go/src/github.com/photoprism/photoprism"
volumes:
- ".:/go/src/github.com/photoprism/photoprism"

10
docker/examples/docker-compose.yml
View file

@ -95,10 +95,14 @@ services:
# user: "1000:1000"
## Share hardware devices with FFmpeg and TensorFlow (optional):
# devices:
# - "/dev/dri:/dev/dri"
# - "/dev/nvidia0:/dev/nvidia0"
# - "/dev/dri:/dev/dri" # Intel (h264_qsv)
# - "/dev/nvidia0:/dev/nvidia0" # Nvidia (h264_nvenc)
# - "/dev/nvidiactl:/dev/nvidiactl"
# - "/dev/video11:/dev/video11" # Video4Linux (h264_v4l2m2m)
# - "/dev/nvidia-modeset:/dev/nvidia-modeset"
# - "/dev/nvidia-nvswitchctl:/dev/nvidia-nvswitchctl"
# - "/dev/nvidia-uvm:/dev/nvidia-uvm"
# - "/dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools"
# - "/dev/video11:/dev/video11" # Video4Linux (h264_v4l2m2m)
working_dir: "/photoprism"
## Storage Folders: "~" is a shortcut for your home directory, "." for the current directory
volumes:

175
internal/photoprism/convert.go
View file

@ -24,10 +24,6 @@ import (
"github.com/photoprism/photoprism/pkg/sanitize"
)
const DefaultAvcEncoder = "libx264" // Default FFmpeg AVC software encoder.
const IntelQsvEncoder = "h264_qsv"
const AppleVideoToolbox = "h264_videotoolbox"
// Convert represents a converter that can convert RAW/HEIF images to JPEG.
type Convert struct {
conf *config.Config
@ -365,174 +361,3 @@ func (c *Convert) AvcBitrate(f *MediaFile) string {
return fmt.Sprintf("%dM", bitrate)
}
// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC.
func (c *Convert) AvcConvertCommand(f *MediaFile, avcName, codecName string) (result *exec.Cmd, useMutex bool, err error) {
if f.IsVideo() {
// Don't transcode more than one video at the same time.
useMutex = true
if codecName == IntelQsvEncoder {
format := "format=rgb32"
result = exec.Command(
c.conf.FFmpegBin(),
"-qsv_device", "/dev/dri/renderD128",
"-init_hw_device", "qsv=hw",
"-filter_hw_device", "hw",
"-i", f.FileName(),
"-c:a", "aac",
"-vf", format,
"-c:v", codecName,
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-maxrate", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
} else if codecName == AppleVideoToolbox {
format := "format=yuv420p"
result = exec.Command(
c.conf.FFmpegBin(),
"-i", f.FileName(),
"-c:v", codecName,
"-c:a", "aac",
"-vf", format,
"-profile", "high",
"-level", "51",
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
} else {
format := "format=yuv420p"
result = exec.Command(
c.conf.FFmpegBin(),
"-i", f.FileName(),
"-c:v", codecName,
"-c:a", "aac",
"-vf", format,
"-num_output_buffers", strconv.Itoa(c.conf.FFmpegBuffers()+8),
"-num_capture_buffers", strconv.Itoa(c.conf.FFmpegBuffers()),
"-max_muxing_queue_size", "1024",
"-crf", "23",
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
}
} else {
return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", f.FileType(), sanitize.Log(f.BaseName()))
}
return result, useMutex, nil
}
// ToAvc converts a single video file to MPEG-4 AVC.
func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err error) {
if encoderName == "" {
encoderName = DefaultAvcEncoder
}
if f == nil {
return nil, fmt.Errorf("convert: file is nil - you might have found a bug")
}
if !f.Exists() {
return nil, fmt.Errorf("convert: %s not found", f.RelName(c.conf.OriginalsPath()))
}
avcName := fs.FormatAvc.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false)
mediaFile, err := NewMediaFile(avcName)
if err == nil && mediaFile.IsVideo() {
return mediaFile, nil
}
if !c.conf.SidecarWritable() {
return nil, fmt.Errorf("convert: transcoding disabled in read only mode (%s)", f.RelName(c.conf.OriginalsPath()))
}
if c.conf.DisableFFmpeg() {
return nil, fmt.Errorf("convert: ffmpeg is disabled for transcoding %s", f.RelName(c.conf.OriginalsPath()))
}
avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.AvcExt)
fileName := f.RelName(c.conf.OriginalsPath())
cmd, useMutex, err := c.AvcConvertCommand(f, avcName, encoderName)
if err != nil {
log.Error(err)
return nil, err
}
if useMutex {
// Make sure only one command is executed at a time.
// See https://photo.stackexchange.com/questions/105969/darktable-cli-fails-because-of-locked-database-file
c.cmdMutex.Lock()
defer c.cmdMutex.Unlock()
}
if fs.FileExists(avcName) {
return NewMediaFile(avcName)
}
// Fetch command output.
var out bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
event.Publish("index.converting", event.Data{
"fileType": f.FileType(),
"fileName": fileName,
"baseName": filepath.Base(fileName),
"xmpName": "",
})
log.Infof("%s: transcoding %s to %s", encoderName, fileName, fs.FormatAvc)
// Log exact command for debugging in trace mode.
log.Trace(cmd.String())
// Run convert command.
start := time.Now()
if err = cmd.Run(); err != nil {
_ = os.Remove(avcName)
if stderr.String() != "" {
err = errors.New(stderr.String())
}
// Log ffmpeg output for debugging.
if err.Error() != "" {
log.Debug(err)
}
// Log filename and transcoding time.
log.Warnf("%s: failed transcoding %s [%s]", encoderName, fileName, time.Since(start))
if encoderName != DefaultAvcEncoder {
return c.ToAvc(f, DefaultAvcEncoder)
} else {
return nil, err
}
}
// Log transcoding time.
log.Infof("%s: created %s [%s]", encoderName, filepath.Base(avcName), time.Since(start))
return NewMediaFile(avcName)
}

267
internal/photoprism/convert_avc.go Normal file
View file

@ -0,0 +1,267 @@
package photoprism
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"time"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/pkg/sanitize"
)
// FFmpegSoftwareEncoder see https://trac.ffmpeg.org/wiki/HWAccelIntro.
const FFmpegSoftwareEncoder = "libx264"
// FFmpegIntelEncoder is the Intel Quick Sync H.264 encoder.
const FFmpegIntelEncoder = "h264_qsv"
// FFmpegAppleEncoder is the Apple Video Toolboar H.264 encoder.
const FFmpegAppleEncoder = "h264_videotoolbox"
// FFmpegVAAPIEncoder is the Video Acceleration API H.264 encoder.
const FFmpegVAAPIEncoder = "h264_vaapi"
// FFmpegNvidiaEncoder is the NVIDIA H.264 encoder.
const FFmpegNvidiaEncoder = "h264_nvenc"
// FFmpegV4L2Encoder is the Video4Linux H.264 encoder.
const FFmpegV4L2Encoder = "h264_v4l2m2m"
// FFmpegAvcEncoders is the list of supported H.264 encoders with aliases.
var FFmpegAvcEncoders = map[string]string{
"": FFmpegSoftwareEncoder,
"default": FFmpegSoftwareEncoder,
"software": FFmpegSoftwareEncoder,
FFmpegSoftwareEncoder: FFmpegSoftwareEncoder,
"intel": FFmpegIntelEncoder,
"qsv": FFmpegIntelEncoder,
FFmpegIntelEncoder: FFmpegIntelEncoder,
"apple": FFmpegAppleEncoder,
"osx": FFmpegAppleEncoder,
"mac": FFmpegAppleEncoder,
"macos": FFmpegAppleEncoder,
FFmpegAppleEncoder: FFmpegAppleEncoder,
"vaapi": FFmpegVAAPIEncoder,
"libva": FFmpegVAAPIEncoder,
FFmpegVAAPIEncoder: FFmpegVAAPIEncoder,
"nvidia": FFmpegNvidiaEncoder,
"nvenc": FFmpegNvidiaEncoder,
"cuda": FFmpegNvidiaEncoder,
FFmpegNvidiaEncoder: FFmpegNvidiaEncoder,
"v4l2": FFmpegV4L2Encoder,
"video4linux": FFmpegV4L2Encoder,
"rp4": FFmpegV4L2Encoder,
"raspberry": FFmpegV4L2Encoder,
"raspberrypi": FFmpegV4L2Encoder,
FFmpegV4L2Encoder: FFmpegV4L2Encoder,
}
// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC.
func (c *Convert) AvcConvertCommand(f *MediaFile, avcName, encoderName string) (result *exec.Cmd, useMutex bool, err error) {
if f.IsVideo() {
// Don't transcode more than one video at the same time.
useMutex = true
// Display encoder info.
if encoderName != FFmpegSoftwareEncoder {
log.Infof("convert: ffmpeg encoder %s selected", encoderName)
}
if encoderName == FFmpegIntelEncoder {
format := "format=rgb32"
result = exec.Command(
c.conf.FFmpegBin(),
"-qsv_device", "/dev/dri/renderD128",
"-init_hw_device", "qsv=hw",
"-filter_hw_device", "hw",
"-i", f.FileName(),
"-c:a", "aac",
"-vf", format,
"-c:v", encoderName,
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-maxrate", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
} else if encoderName == FFmpegAppleEncoder {
format := "format=yuv420p"
result = exec.Command(
c.conf.FFmpegBin(),
"-i", f.FileName(),
"-c:v", encoderName,
"-c:a", "aac",
"-vf", format,
"-profile", "high",
"-level", "51",
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
} else if encoderName == FFmpegNvidiaEncoder {
// to show options: ffmpeg -hide_banner -h encoder=h264_nvenc
result = exec.Command(
c.conf.FFmpegBin(),
"-r", "30",
"-i", f.FileName(),
"-pix_fmt", "yuv420p",
"-c:v", encoderName,
"-c:a", "aac",
"-preset", "15",
"-pixel_format", "yuv420p",
"-gpu", "any",
"-vf", "format=yuv420p",
"-rc:v", "constqp",
"-cq", "0",
"-tune", "2",
"-b:v", c.AvcBitrate(f),
"-profile:v", "1",
"-level:v", "41",
"-coder:v", "1",
"-f", "mp4",
"-y",
avcName,
)
} else {
format := "format=yuv420p"
result = exec.Command(
c.conf.FFmpegBin(),
"-i", f.FileName(),
"-c:v", encoderName,
"-c:a", "aac",
"-vf", format,
"-num_output_buffers", strconv.Itoa(c.conf.FFmpegBuffers()+8),
"-num_capture_buffers", strconv.Itoa(c.conf.FFmpegBuffers()),
"-max_muxing_queue_size", "1024",
"-crf", "23",
"-vsync", "vfr",
"-r", "30",
"-b:v", c.AvcBitrate(f),
"-f", "mp4",
"-y",
avcName,
)
}
} else {
return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", f.FileType(), sanitize.Log(f.BaseName()))
}
return result, useMutex, nil
}
// ToAvc converts a single video file to MPEG-4 AVC.
func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err error) {
if n := FFmpegAvcEncoders[encoderName]; n != "" {
encoderName = n
} else {
log.Warnf("convert: unsupported ffmpeg encoder %s", encoderName)
encoderName = FFmpegSoftwareEncoder
}
if f == nil {
return nil, fmt.Errorf("convert: file is nil - you might have found a bug")
}
if !f.Exists() {
return nil, fmt.Errorf("convert: %s not found", f.RelName(c.conf.OriginalsPath()))
}
avcName := fs.FormatAvc.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false)
mediaFile, err := NewMediaFile(avcName)
if err == nil && mediaFile.IsVideo() {
return mediaFile, nil
}
if !c.conf.SidecarWritable() {
return nil, fmt.Errorf("convert: transcoding disabled in read only mode (%s)", f.RelName(c.conf.OriginalsPath()))
}
if c.conf.DisableFFmpeg() {
return nil, fmt.Errorf("convert: ffmpeg is disabled for transcoding %s", f.RelName(c.conf.OriginalsPath()))
}
avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.AvcExt)
fileName := f.RelName(c.conf.OriginalsPath())
cmd, useMutex, err := c.AvcConvertCommand(f, avcName, encoderName)
if err != nil {
log.Error(err)
return nil, err
}
if useMutex {
// Make sure only one command is executed at a time.
// See https://photo.stackexchange.com/questions/105969/darktable-cli-fails-because-of-locked-database-file
c.cmdMutex.Lock()
defer c.cmdMutex.Unlock()
}
if fs.FileExists(avcName) {
return NewMediaFile(avcName)
}
// Fetch command output.
var out bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
event.Publish("index.converting", event.Data{
"fileType": f.FileType(),
"fileName": fileName,
"baseName": filepath.Base(fileName),
"xmpName": "",
})
log.Infof("%s: transcoding %s to %s", encoderName, fileName, fs.FormatAvc)
// Log exact command for debugging in trace mode.
log.Trace(cmd.String())
// Run convert command.
start := time.Now()
if err = cmd.Run(); err != nil {
_ = os.Remove(avcName)
if stderr.String() != "" {
err = errors.New(stderr.String())
}
// Log ffmpeg output for debugging.
if err.Error() != "" {
log.Debug(err)
}
// Log filename and transcoding time.
log.Warnf("%s: failed transcoding %s [%s]", encoderName, fileName, time.Since(start))
if encoderName != FFmpegSoftwareEncoder {
return c.ToAvc(f, FFmpegSoftwareEncoder)
} else {
return nil, err
}
}
// Log transcoding time.
log.Infof("%s: created %s [%s]", encoderName, filepath.Base(avcName), time.Since(start))
return NewMediaFile(avcName)
}

66
internal/photoprism/convert_avc_test.go Normal file
View file

@ -0,0 +1,66 @@
package photoprism
import (
"os"
"path/filepath"
"testing"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/stretchr/testify/assert"
)
func TestConvert_ToAvc(t *testing.T) {
t.Run("gopher-video.mp4", func(t *testing.T) {
conf := config.TestConfig()
convert := NewConvert(conf)
fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4")
outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.mp4.avc")
_ = os.Remove(outputName)
assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName)
mf, err := NewMediaFile(fileName)
if err != nil {
t.Fatal(err)
}
avcFile, err := convert.ToAvc(mf, "")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, avcFile.FileName(), outputName)
assert.Truef(t, fs.FileExists(avcFile.FileName()), "output file does not exist: %s", avcFile.FileName())
t.Logf("video metadata: %+v", avcFile.MetaData())
_ = os.Remove(outputName)
})
t.Run("jpg", func(t *testing.T) {
conf := config.TestConfig()
convert := NewConvert(conf)
fileName := filepath.Join(conf.ExamplesPath(), "cat_black.jpg")
outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "cat_black.jpg.avc")
_ = os.Remove(outputName)
assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName)
mf, err := NewMediaFile(fileName)
if err != nil {
t.Fatal(err)
}
avcFile, err := convert.ToAvc(mf, "")
assert.Error(t, err)
assert.Nil(t, avcFile)
})
}

55
internal/photoprism/convert_test.go
View file

@ -348,58 +348,3 @@ func TestConvert_AvcConvertCommand(t *testing.T) {
assert.Nil(t, r)
})
}
func TestConvert_ToAvc(t *testing.T) {
t.Run("gopher-video.mp4", func(t *testing.T) {
conf := config.TestConfig()
convert := NewConvert(conf)
fileName := filepath.Join(conf.ExamplesPath(), "gopher-video.mp4")
outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "gopher-video.mp4.avc")
_ = os.Remove(outputName)
assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName)
mf, err := NewMediaFile(fileName)
if err != nil {
t.Fatal(err)
}
avcFile, err := convert.ToAvc(mf, "")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, avcFile.FileName(), outputName)
assert.Truef(t, fs.FileExists(avcFile.FileName()), "output file does not exist: %s", avcFile.FileName())
t.Logf("video metadata: %+v", avcFile.MetaData())
_ = os.Remove(outputName)
})
t.Run("jpg", func(t *testing.T) {
conf := config.TestConfig()
convert := NewConvert(conf)
fileName := filepath.Join(conf.ExamplesPath(), "cat_black.jpg")
outputName := filepath.Join(conf.SidecarPath(), conf.ExamplesPath(), "cat_black.jpg.avc")
_ = os.Remove(outputName)
assert.Truef(t, fs.FileExists(fileName), "input file does not exist: %s", fileName)
mf, err := NewMediaFile(fileName)
if err != nil {
t.Fatal(err)
}
avcFile, err := convert.ToAvc(mf, "")
assert.Error(t, err)
assert.Nil(t, avcFile)
})
}

6
scripts/dist/install-gpu.sh vendored
View file

@ -35,7 +35,11 @@ for t in ${GPU_DETECTED[@]}; do
;;
nvidia)
apt-get -qq install nvidia-opencl-icd nvidia-vdpau-driver nvidia-driver-libs nvidia-kernel-dkms libva2 vainfo libva-wayland2
apt-get -qq install nvidia-opencl-icd nvidia-vdpau-driver nvidia-driver-libs nvidia-kernel-dkms libva2 vainfo libva-wayland2 libnvidia-encode1
;;
null)
# ignore
;;
*)