CLI: Create thumbs and convert files in deterministic order #3194

This also adds support for specifying a path to the thumbs command.

Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2023-02-14 11:37:22 +01:00
parent 4e0f38881d
commit 0c4aa86f85
30 changed files with 161 additions and 83 deletions

View file

@ -57,7 +57,7 @@ func StartImport(router *gin.RouterGroup) {
srcFolder := ""
importPath := conf.ImportPath()
// Import from sub-folder?
// Import from subfolder?
if srcFolder = c.Param("path"); srcFolder != "" && srcFolder != "/" {
srcFolder = clean.UserPath(srcFolder)
} else if f.Path != "" {

View file

@ -24,7 +24,7 @@ const backupDescription = "A user-defined SQL dump FILENAME or - for stdout can
" Make sure to run the command with exec -T when using Docker to prevent log messages from being sent to stdout.\n" +
" The index backup and album file paths are automatically detected if not specified explicitly."
// BackupCommand configures the backup cli command.
// BackupCommand configures the command name, flags, and action.
var BackupCommand = cli.Command{
Name: "backup",
Description: backupDescription,

View file

@ -11,7 +11,7 @@ import (
"github.com/photoprism/photoprism/internal/photoprism"
)
// CleanUpCommand registers the cleanup command.
// CleanUpCommand configures the command name, flags, and action.
var CleanUpCommand = cli.Command{
Name: "cleanup",
Usage: "Removes orphaned index entries, sidecar and thumbnail files",

View file

@ -13,11 +13,11 @@ import (
"github.com/photoprism/photoprism/pkg/clean"
)
// ConvertCommand registers the convert cli command.
// ConvertCommand configures the command name, flags, and action.
var ConvertCommand = cli.Command{
Name: "convert",
Usage: "Converts files in other formats to JPEG and AVC as needed",
ArgsUsage: "[sub-folder]",
ArgsUsage: "[subfolder]",
Flags: []cli.Flag{
cli.StringSliceFlag{
Name: "ext, e",

View file

@ -15,7 +15,7 @@ import (
"github.com/photoprism/photoprism/pkg/clean"
)
// CopyCommand registers the copy cli command.
// CopyCommand configures the command name, flags, and action.
var CopyCommand = cli.Command{
Name: "cp",
Aliases: []string{"copy"},

View file

@ -18,7 +18,7 @@ import (
"github.com/photoprism/photoprism/pkg/fs"
)
// FacesCommand registers the face recognition subcommands.
// FacesCommand configures the command name, flags, and action.
var FacesCommand = cli.Command{
Name: "faces",
Usage: "Face recognition subcommands",
@ -53,7 +53,7 @@ var FacesCommand = cli.Command{
{
Name: "index",
Usage: "Searches originals for faces",
ArgsUsage: "[sub-folder]",
ArgsUsage: "[subfolder]",
Action: facesIndexAction,
},
{

View file

@ -15,7 +15,7 @@ import (
"github.com/photoprism/photoprism/pkg/clean"
)
// ImportCommand registers the import cli command.
// ImportCommand configures the command name, flags, and action.
var ImportCommand = cli.Command{
Name: "mv",
Aliases: []string{"import"},

View file

@ -19,7 +19,7 @@ import (
var IndexCommand = cli.Command{
Name: "index",
Usage: "Indexes original media files",
ArgsUsage: "[sub-folder]",
ArgsUsage: "[subfolder]",
Flags: indexFlags,
Action: indexAction,
}

View file

@ -9,7 +9,7 @@ import (
"github.com/photoprism/photoprism/internal/get"
)
// MomentsCommand registers the moments command.
// MomentsCommand configures the command name, flags, and action.
var MomentsCommand = cli.Command{
Name: "moments",
Usage: "Creates albums of special moments, trips, and places",

View file

@ -10,7 +10,7 @@ import (
"github.com/photoprism/photoprism/internal/workers"
)
// OptimizeCommand registers the index cli command.
// OptimizeCommand configures the command name, flags, and action.
var OptimizeCommand = cli.Command{
Name: "optimize",
Usage: "Maintains titles, estimates, and other metadata",

View file

@ -13,7 +13,7 @@ import (
"github.com/photoprism/photoprism/internal/query"
)
// PlacesCommand registers the places subcommands.
// PlacesCommand configures the command name, flags, and action.
var PlacesCommand = cli.Command{
Name: "places",
Usage: "Maps and location details subcommands",

View file

@ -15,7 +15,7 @@ import (
"github.com/photoprism/photoprism/pkg/fs"
)
// PurgeCommand registers the index cli command.
// PurgeCommand configures the command name, flags, and action.
var PurgeCommand = cli.Command{
Name: "purge",
Usage: "Updates missing files, photo counts, and album covers",

View file

@ -18,7 +18,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
)
// ResetCommand resets the index, clears the cache, and removes sidecar files after confirmation.
// ResetCommand configures the command name, flags, and action.
var ResetCommand = cli.Command{
Name: "reset",
Usage: "Resets the index, clears the cache, and removes sidecar files",

View file

@ -26,7 +26,7 @@ const restoreDescription = "A user-defined SQL dump FILENAME can be passed as th
"The -i parameter can be omitted in this case.\n" +
" The index backup and album file paths are automatically detected if not specified explicitly."
// RestoreCommand configures the backup cli command.
// RestoreCommand configures the command name, flags, and action.
var RestoreCommand = cli.Command{
Name: "restore",
Description: restoreDescription,

View file

@ -4,7 +4,7 @@ import (
"github.com/urfave/cli"
)
// ShowCommand registers the show subcommands.
// ShowCommand configures the show subcommands.
var ShowCommand = cli.Command{
Name: "show",
Usage: "Shows supported formats, features, and config options",

View file

@ -24,7 +24,7 @@ import (
"github.com/photoprism/photoprism/pkg/fs"
)
// StartCommand registers the start cli command.
// StartCommand configures the command name, flags, and action.
var StartCommand = cli.Command{
Name: "start",
Aliases: []string{"up"},

View file

@ -12,7 +12,7 @@ import (
"github.com/photoprism/photoprism/internal/config"
)
// StatusCommand registers the status command.
// StatusCommand configures the command name, flags, and action.
var StatusCommand = cli.Command{
Name: "status",
Usage: "Checks if the Web server is running",

View file

@ -9,7 +9,7 @@ import (
"github.com/photoprism/photoprism/pkg/clean"
)
// StopCommand registers the stop cli command.
// StopCommand configures the command name, flags, and action.
var StopCommand = cli.Command{
Name: "stop",
Aliases: []string{"down"},

View file

@ -2,6 +2,7 @@ package commands
import (
"context"
"strings"
"time"
"github.com/urfave/cli"
@ -10,24 +11,25 @@ import (
"github.com/photoprism/photoprism/pkg/clean"
)
// ThumbsCommand registers the resample cli command.
// ThumbsCommand configures the command name, flags, and action.
var ThumbsCommand = cli.Command{
Name: "thumbs",
Usage: "Generates thumbnails using the current settings",
Name: "thumbs",
Usage: "Generates thumbnails using the current settings",
ArgsUsage: "[subfolder]",
Flags: []cli.Flag{
cli.BoolFlag{
Name: "force, f",
Usage: "replace existing thumbnails",
Usage: "replace existing thumbnail files",
},
cli.BoolFlag{
Name: "originals, o",
Usage: "originals only, skip sidecar files",
Usage: "scan originals only, skip sidecar folder",
},
},
Action: thumbsAction,
}
// thumbsAction pre-renders thumbnail images.
// thumbsAction generates thumbnails using the current settings.
func thumbsAction(ctx *cli.Context) error {
start := time.Now()
@ -43,16 +45,41 @@ func thumbsAction(ctx *cli.Context) error {
conf.RegisterDb()
defer conf.Shutdown()
log.Infof("creating thumbs in %s", clean.Log(conf.ThumbCachePath()))
dir := strings.TrimSpace(ctx.Args().First())
force := ctx.Bool("force")
originals := ctx.Bool("originals")
rs := get.Thumbs()
var action, ack string
if force {
action = "replacing"
ack = "replaced"
} else {
action = "creating"
ack = "created"
}
if err := rs.Start(ctx.Bool("force"), ctx.Bool("originals")); err != nil {
log.Error(err)
// Display info.
if dir == "" {
if originals {
log.Infof("%s thumbnails for originals only", action)
} else {
log.Infof("%s thumbnails for originals and sidecar files", action)
}
} else {
if originals {
log.Infof("%s thumbnails for originals in %s", action, clean.LogQuote(dir))
} else {
log.Infof("%s thumbnails for originals and sidecar files in %s", action, clean.LogQuote(dir))
}
}
w := get.Thumbs()
if err = w.Start(dir, ctx.Bool("force"), ctx.Bool("originals")); err != nil {
return err
}
log.Infof("thumbs created in %s", time.Since(start))
log.Infof("thumbnails %s in %s", ack, time.Since(start))
return nil
}

View file

@ -18,7 +18,7 @@ const (
UserWebDAVUsage = "allow to sync files via WebDAV"
)
// UsersCommand registers the user management subcommands.
// UsersCommand configures the user management subcommands.
var UsersCommand = cli.Command{
Name: "users",
Aliases: []string{"user"},

View file

@ -8,7 +8,7 @@ import (
"github.com/photoprism/photoprism/internal/config"
)
// VersionCommand registers the version cli command.
// VersionCommand configures the command name, flags, and action.
var VersionCommand = cli.Command{
Name: "version",
Usage: "Shows version information",

View file

@ -40,7 +40,7 @@ func NewConvert(conf *config.Config) *Convert {
}
// Start converts all files in a directory to JPEG if possible.
func (c *Convert) Start(path string, ext []string, force bool) (err error) {
func (c *Convert) Start(dir string, ext []string, force bool) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("convert: %s (panic)\nstack: %s", r, debug.Stack())
@ -48,7 +48,7 @@ func (c *Convert) Start(path string, ext []string, force bool) (err error) {
}
}()
if err := mutex.MainWorker.Start(); err != nil {
if err = mutex.MainWorker.Start(); err != nil {
return err
}
@ -70,7 +70,7 @@ func (c *Convert) Start(path string, ext []string, force bool) (err error) {
done := make(fs.Done)
ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
if err := ignore.Dir(path); err != nil {
if err = ignore.Dir(dir); err != nil {
log.Infof("convert: %s", err)
}
@ -78,7 +78,7 @@ func (c *Convert) Start(path string, ext []string, force bool) (err error) {
log.Infof("convert: ignoring %s", clean.Log(filepath.Base(fileName)))
}
err = godirwalk.Walk(path, &godirwalk.Options{
err = godirwalk.Walk(dir, &godirwalk.Options{
ErrorCallback: func(fileName string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
@ -122,7 +122,7 @@ func (c *Convert) Start(path string, ext []string, force bool) (err error) {
return nil
},
Unsorted: true,
Unsorted: false,
FollowSymbolicLinks: true,
})

View file

@ -70,7 +70,7 @@ func (imp *Import) Start(opt ImportOptions) fs.Done {
// Check if the import folder exists.
if !fs.PathExists(importPath) {
event.Error(fmt.Sprintf("import: %s does not exist", importPath))
event.Error(fmt.Sprintf("import: directory %s not found", importPath))
return done
}

View file

@ -90,7 +90,7 @@ func (ind *Index) Start(o IndexOptions) fs.Done {
optionsPath := filepath.Join(originalsPath, o.Path)
if !fs.PathExists(optionsPath) {
event.Error(fmt.Sprintf("%s does not exist", clean.Log(optionsPath)))
event.Error(fmt.Sprintf("index: directory %s not found", clean.Log(optionsPath)))
return done
} else if fs.DirIsEmpty(originalsPath) {
event.InfoMsg(i18n.ErrOriginalsEmpty)

View file

@ -721,53 +721,96 @@ func (m *MediaFile) IsPreviewImage() bool {
return m.IsJpeg() || m.IsPng()
}
// IsJpeg return true if this media file is a JPEG image.
// IsJpeg checks if the file is a JPEG image with a supported file type extension.
func (m *MediaFile) IsJpeg() bool {
// Don't import/use existing thumbnail files (we create our own)
if m.Extension() == fs.ExtTHM {
// Ignore .thm files, as some cameras automatically
// create them as thumbnails.
return false
} else if fs.FileType(m.fileName) != fs.ImageJPEG {
// Files with an incorrect file extension are no longer
// recognized as JPEG to improve indexing performance.
return false
}
// Since mime type detection is expensive, it is only
// performed after other checks have passed.
return m.MimeType() == fs.MimeTypeJpeg
}
// IsPng returns true if this is a PNG image.
// IsPng checks if the file is a PNG image with a supported file type extension.
func (m *MediaFile) IsPng() bool {
if fs.FileType(m.fileName) != fs.ImagePNG {
// Files with an incorrect file extension are no longer
// recognized as PNG to improve indexing performance.
return false
}
// Since mime type detection is expensive, it is only
// performed after other checks have passed.
return m.MimeType() == fs.MimeTypePng
}
// IsGif returns true if this is a GIF image.
// IsGif checks if the file is a GIF image with a supported file type extension.
func (m *MediaFile) IsGif() bool {
if fs.FileType(m.fileName) != fs.ImageGIF {
return false
}
return m.MimeType() == fs.MimeTypeGif
}
// IsTiff returns true if this is a TIFF image.
// IsTiff checks if the file is a TIFF image with a supported file type extension.
func (m *MediaFile) IsTiff() bool {
return m.HasFileType(fs.ImageTIFF) && m.MimeType() == fs.MimeTypeTiff
if fs.FileType(m.fileName) != fs.ImageTIFF {
return false
}
return m.MimeType() == fs.MimeTypeTiff
}
// IsDNG returns true if this is a Adobe Digital Negative image.
// IsDNG checks if the file is a Adobe Digital Negative (DNG) image with a supported file type extension.
func (m *MediaFile) IsDNG() bool {
if fs.FileType(m.fileName) != fs.ImageDNG {
return false
}
return m.MimeType() == fs.MimeTypeDNG
}
// IsHEIC returns true if this is a High Efficiency Image File Format image.
// IsHEIC checks if the file is a High Efficiency Image File Format (HEIC/HEIF) image with a supported file type extension.
func (m *MediaFile) IsHEIC() bool {
if t := fs.FileType(m.fileName); t != fs.ImageHEIC && t != fs.ImageHEIF {
return false
}
return m.MimeType() == fs.MimeTypeHEIC
}
// IsAVIF returns true if this is an AV1 Image File Format image.
// IsAVIF checks if the file is an AV1 Image File Format image with a supported file type extension.
func (m *MediaFile) IsAVIF() bool {
if fs.FileType(m.fileName) != fs.ImageAVIF {
return false
}
return m.MimeType() == fs.MimeTypeAVIF
}
// IsBitmap returns true if this is a bitmap image.
// IsBitmap checks if the file is a bitmap image with a supported file type extension.
func (m *MediaFile) IsBitmap() bool {
if fs.FileType(m.fileName) != fs.ImageBMP {
return false
}
return m.MimeType() == fs.MimeTypeBitmap
}
// IsWebP returns true if this is a WebP image file.
// IsWebP checks if the file is a WebP image file with a supported file type extension.
func (m *MediaFile) IsWebP() bool {
if fs.FileType(m.fileName) != fs.ImageWebP {
return false
}
return m.MimeType() == fs.MimeTypeWebP
}
@ -780,12 +823,12 @@ func (m *MediaFile) Duration() time.Duration {
return m.MetaData().Duration
}
// IsAnimatedGif returns true if it is an animated GIF.
// IsAnimatedGif checks if the file is an animated GIF with a supported file type extension.
func (m *MediaFile) IsAnimatedGif() bool {
return m.IsGif() && m.MetaData().Frames > 1
}
// IsJson return true if this media file is a json sidecar file.
// IsJson checks if the file is a JSON sidecar file with a supported file type extension.
func (m *MediaFile) IsJson() bool {
return m.HasFileType(fs.SidecarJSON)
}
@ -848,7 +891,7 @@ func (m *MediaFile) IsAnimated() bool {
// IsVideo returns true if this is a video file.
func (m *MediaFile) IsVideo() bool {
return strings.HasPrefix(m.MimeType(), "video/") || m.Media() == media.Video
return m.HasMediaType(media.Video)
}
// IsVector returns true if this is a vector graphics.
@ -863,7 +906,7 @@ func (m *MediaFile) IsSidecar() bool {
// IsSVG returns true if this is a SVG vector graphics.
func (m *MediaFile) IsSVG() bool {
return m.HasFileType(fs.VectorSVG)
return m.FileType() == fs.VectorSVG
}
// IsXMP returns true if this is a XMP sidecar file.
@ -964,25 +1007,17 @@ func (m *MediaFile) HasPreviewImage() bool {
jpegName := fs.ImageJPEG.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false)
if jpegName == "" {
m.hasPreviewImage = false
} else {
m.hasPreviewImage = fs.MimeType(jpegName) == fs.MimeTypeJpeg
}
if m.hasPreviewImage {
if m.hasPreviewImage = fs.MimeType(jpegName) == fs.MimeTypeJpeg; m.hasPreviewImage {
return true
}
pngName := fs.ImagePNG.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false)
if pngName == "" {
m.hasPreviewImage = false
} else {
m.hasPreviewImage = fs.MimeType(pngName) == fs.MimeTypePng
if m.hasPreviewImage = fs.MimeType(pngName) == fs.MimeTypePng; m.hasPreviewImage {
return true
}
return m.hasPreviewImage
return false
}
func (m *MediaFile) decodeDimensions() error {

View file

@ -2139,9 +2139,12 @@ func TestMediaFile_FileType(t *testing.T) {
t.Fatal(err)
}
assert.True(t, m.IsJpeg())
assert.Equal(t, "jpg", string(m.FileType()))
assert.Equal(t, fs.ImageJPEG, m.FileType())
// No longer recognized as JPEG to improve indexing performance (skips mime type detection).
assert.False(t, m.IsJpeg())
assert.False(t, m.IsPng())
assert.Equal(t, "png", string(m.FileType()))
assert.Equal(t, "image/jpeg", m.MimeType())
assert.Equal(t, fs.ImagePNG, m.FileType())
assert.Equal(t, ".png", m.Extension())
}

View file

@ -26,8 +26,8 @@ func NewThumbs(conf *config.Config) *Thumbs {
return &Thumbs{conf: conf}
}
// Start creates thumbnail images for all files found in the originals and sidecar folders.
func (w *Thumbs) Start(force, originalsOnly bool) (err error) {
// Start creates thumbnails for files in the originals and sidecar folders.
func (w *Thumbs) Start(dir string, force, originalsOnly bool) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("thumbs: %s (panic)\nstack: %s", r, debug.Stack())
@ -36,13 +36,22 @@ func (w *Thumbs) Start(force, originalsOnly bool) (err error) {
}()
originalsPath := w.conf.OriginalsPath()
originalsDir := filepath.Join(originalsPath, dir)
sidecarPath := w.conf.SidecarPath()
sidecarDir := filepath.Join(sidecarPath, dir)
originalsOnly = originalsOnly || sidecarPath == "" || sidecarPath == originalsPath
// Valid path provided?
if !fs.PathExists(originalsDir) {
return fmt.Errorf("thumbs: directory %s not found", clean.Log(originalsDir))
}
if _, err = w.Dir(originalsPath, force); err != nil || originalsOnly {
// Scan sidecar folder?
originalsOnly = originalsOnly || sidecarPath == "" || sidecarPath == originalsPath || !fs.PathExists(sidecarDir)
// Start creating thumbnails.
if _, err = w.Dir(originalsDir, force); err != nil || originalsOnly {
return err
} else if _, err = w.Dir(sidecarPath, force); err != nil {
} else if _, err = w.Dir(sidecarDir, force); err != nil {
return err
}
@ -50,10 +59,10 @@ func (w *Thumbs) Start(force, originalsOnly bool) (err error) {
}
// Dir creates thumbnail images for files found in a given path.
func (w *Thumbs) Dir(dir string, force bool) (done fs.Done, err error) {
done = make(fs.Done)
func (w *Thumbs) Dir(dir string, force bool) (fs.Done, error) {
done := make(fs.Done)
if err = mutex.MainWorker.Start(); err != nil {
if err := mutex.MainWorker.Start(); err != nil {
return done, err
}
@ -123,18 +132,18 @@ func (w *Thumbs) Dir(dir string, force bool) (done fs.Done, err error) {
return nil
}
log.Infof("thumbs: processing files in %s folder", clean.Log(filepath.Base(dir)))
log.Infof("thumbs: processing %s", clean.Log(dir))
if err := ignore.Dir(dir); err != nil {
log.Infof("thumbs: %s", err)
}
err = godirwalk.Walk(dir, &godirwalk.Options{
err := godirwalk.Walk(dir, &godirwalk.Options{
ErrorCallback: func(fileName string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Callback: handler,
Unsorted: true,
Unsorted: false,
FollowSymbolicLinks: true,
})

View file

@ -44,7 +44,7 @@ func TestResample_Start(t *testing.T) {
rs := NewThumbs(conf)
err := rs.Start(true, false)
err := rs.Start("", true, false)
if err != nil {
t.Fatal(err)

View file

@ -9,7 +9,7 @@ import (
"github.com/photoprism/photoprism/pkg/media"
)
// FoldersByPath returns a slice of folders in a given directory incl sub-folders in recursive mode.
// FoldersByPath returns a slice of folders in a given directory incl subfolders in recursive mode.
func FoldersByPath(rootName, rootPath, path string, recursive bool) (folders entity.Folders, err error) {
dirs, err := fs.Dirs(filepath.Join(rootPath, path), recursive, true)

View file

@ -30,6 +30,10 @@ const (
// MimeType returns the mime type of a file, or an empty string if it could not be detected.
func MimeType(filename string) (mimeType string) {
if filename == "" {
return MimeTypeUnknown
}
// Workaround for types that cannot be reliably detected.
switch Extensions[strings.ToLower(filepath.Ext(filename))] {
case ImageDNG: