luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

CLI: Generate thumbs for files in the sidecar folder #2669

Browse source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2022-08-31 18:53:04 +02:00
parent ce86e5b6b4
commit fb921a4932
15 changed files with 232 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
internal/commands/thumbs.go
View file

@ -19,6 +19,10 @@ var ThumbsCommand = cli.Command{
Name: "force, f",
Usage: "replace existing thumbnails",
},
cli.BoolFlag{
Name: "originals, o",
Usage: "originals only, skip sidecar files",
},
},
Action: thumbsAction,
}
@ -34,16 +38,16 @@ func thumbsAction(ctx *cli.Context) error {
return err
}
log.Infof("creating thumbnails in %s", clean.Log(conf.ThumbCachePath()))
log.Infof("creating thumbs in %s", clean.Log(conf.ThumbCachePath()))
rs := service.Resample()
rs := service.Thumbs()
if err := rs.Start(ctx.Bool("force")); err != nil {
if err := rs.Start(ctx.Bool("force"), ctx.Bool("originals")); err != nil {
log.Error(err)
return err
}
log.Infof("thumbnails created in %s", time.Since(start))
log.Infof("thumbs created in %s", time.Since(start))
return nil
}

13
internal/config/config.go
View file

@ -445,9 +445,16 @@ func (c *Config) ImprintUrl() string {
return c.options.ImprintUrl
}
// Prod checks if production mode is enabled, hides non-essential log messages.
func (c *Config) Prod() bool {
return c.options.Prod
}
// Debug checks if debug mode is enabled, shows non-essential log messages.
func (c *Config) Debug() bool {
if c.Trace() {
if c.Prod() {
return false
} else if c.Trace() {
return true
}
@ -456,6 +463,10 @@ func (c *Config) Debug() bool {
// Trace checks if trace mode is enabled, shows all log messages.
func (c *Config) Trace() bool {
if c.Prod() {
return false
}
return c.options.Trace || c.options.LogLevel == logrus.TraceLevel.String()
}

21
internal/config/config_test.go
View file

@ -36,10 +36,31 @@ func TestNewConfig(t *testing.T) {
assert.IsType(t, new(Config), c)
assert.Equal(t, fs.Abs("../../assets"), c.AssetsPath())
assert.False(t, c.Prod())
assert.False(t, c.Debug())
assert.False(t, c.ReadOnly())
}
func TestConfig_Prod(t *testing.T) {
c := NewConfig(CliTestContext())
assert.False(t, c.Prod())
assert.False(t, c.Debug())
assert.False(t, c.Trace())
c.options.Prod = true
c.options.Debug = true
assert.True(t, c.Prod())
assert.False(t, c.Debug())
assert.False(t, c.Trace())
c.options.Prod = false
assert.True(t, c.Debug())
assert.False(t, c.Trace())
c.options.Debug = false
assert.False(t, c.Debug())
assert.False(t, c.Debug())
assert.False(t, c.Trace())
}
func TestConfig_Name(t *testing.T) {
c := NewConfig(CliTestContext())

1
internal/config/options.go
View file

@ -28,6 +28,7 @@ type Options struct {
Public bool `yaml:"Public" json:"-" flag:"public"`
AdminPassword string `yaml:"AdminPassword" json:"-" flag:"admin-password"`
LogLevel string `yaml:"LogLevel" json:"-" flag:"log-level"`
Prod bool `yaml:"Prod" json:"Prod" flag:"prod"`
Debug bool `yaml:"Debug" json:"Debug" flag:"debug"`
Trace bool `yaml:"Trace" json:"Trace" flag:"Trace"`
Test bool `yaml:"-" json:"Test,omitempty" flag:"test"`

7
internal/config/options_cli.go
View file

@ -38,6 +38,13 @@ var Flags = CliFlags{
Value: "info",
EnvVar: "PHOTOPRISM_LOG_LEVEL",
}},
CliFlag{
Flag: cli.BoolFlag{
Name: "prod",
Hidden: true,
Usage: "enable production mode, hide non-essential log messages",
EnvVar: "PHOTOPRISM_PROD",
}},
CliFlag{
Flag: cli.BoolFlag{
Name: "debug",

123
internal/photoprism/resample.go
View file

@ -1,123 +0,0 @@
package photoprism
import (
"errors"
"fmt"
"path/filepath"
"runtime/debug"
"sync"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/pkg/fs"
)
// Resample represents a thumbnail generator worker.
type Resample struct {
conf *config.Config
}
// NewResample returns a new thumbnail generator and expects the config as argument.
func NewResample(conf *config.Config) *Resample {
return &Resample{conf: conf}
}
// Start creates default thumbnails for all files in originalsPath.
func (w *Resample) Start(force bool) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("resample: %s (panic)\nstack: %s", r, debug.Stack())
log.Error(err)
}
}()
if err := mutex.MainWorker.Start(); err != nil {
return err
}
defer mutex.MainWorker.Stop()
originalsPath := w.conf.OriginalsPath()
thumbnailsPath := w.conf.ThumbCachePath()
jobs := make(chan ResampleJob)
// Start a fixed number of goroutines to read and digest files.
var wg sync.WaitGroup
var numWorkers = w.conf.Workers()
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
ResampleWorker(jobs)
wg.Done()
}()
}
done := make(fs.Done)
ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
if err := ignore.Dir(originalsPath); err != nil {
log.Infof("resample: %s", err)
}
ignore.Log = func(fileName string) {
log.Infof(`resample: ignored "%s"`, fs.RelName(fileName, originalsPath))
}
err = godirwalk.Walk(originalsPath, &godirwalk.Options{
ErrorCallback: func(fileName string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Callback: func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if r := recover(); r != nil {
log.Errorf("resample: %s (panic)\nstack: %s", r, debug.Stack())
}
}()
if mutex.MainWorker.Canceled() {
return errors.New("canceled")
}
isDir, _ := info.IsDirOrSymlinkToDir()
isSymlink := info.IsSymlink()
if skip, result := fs.SkipWalk(fileName, isDir, isSymlink, done, ignore); skip {
return result
}
mf, err := NewMediaFile(fileName)
if err != nil || mf.Empty() || !mf.IsJpeg() {
return nil
}
done[fileName] = fs.Processed
relativeName := mf.RelName(originalsPath)
event.Publish("index.thumbnails", event.Data{
"fileName": relativeName,
"baseName": filepath.Base(relativeName),
"force": force,
})
jobs <- ResampleJob{
mediaFile: mf,
path: thumbnailsPath,
force: force,
}
return nil
},
Unsorted: true,
FollowSymbolicLinks: true,
})
close(jobs)
wg.Wait()
return err
}

146
internal/photoprism/thumbs.go Normal file
View file

@ -0,0 +1,146 @@
package photoprism
import (
"errors"
"fmt"
"path/filepath"
"runtime/debug"
"sync"
"github.com/karrick/godirwalk"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/pkg/clean"
"github.com/photoprism/photoprism/pkg/fs"
)
// Thumbs represents a thumbnail image generator.
type Thumbs struct {
conf *config.Config
}
// NewThumbs returns a new thumbnails generator and expects the config as argument.
func NewThumbs(conf *config.Config) *Thumbs {
return &Thumbs{conf: conf}
}
// Start creates thumbnail images for all files found in the originals and sidecar folders.
func (w *Thumbs) Start(force, originalsOnly bool) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("thumbs: %s (panic)\nstack: %s", r, debug.Stack())
log.Error(err)
}
}()
originalsPath := w.conf.OriginalsPath()
sidecarPath := w.conf.SidecarPath()
originalsOnly = originalsOnly || sidecarPath == "" || sidecarPath == originalsPath
if _, err = w.Dir(originalsPath, force); err != nil || originalsOnly {
return err
} else if _, err = w.Dir(sidecarPath, force); err != nil {
return err
}
return nil
}
// Dir creates thumbnail images for files found in a given path.
func (w *Thumbs) Dir(dir string, force bool) (done fs.Done, err error) {
done = make(fs.Done)
if err = mutex.MainWorker.Start(); err != nil {
return done, err
}
defer mutex.MainWorker.Stop()
jobs := make(chan ThumbsJob)
thumbnailsPath := w.conf.ThumbCachePath()
// Start a fixed number of goroutines to read and digest files.
var wg sync.WaitGroup
var numWorkers = w.conf.Workers()
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
ThumbsWorker(jobs)
wg.Done()
}()
}
ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
ignore.Log = func(fileName string) {
log.Infof(`thumbs: ignored "%s"`, fs.RelName(fileName, dir))
}
handler := func(fileName string, info *godirwalk.Dirent) error {
defer func() {
if r := recover(); r != nil {
log.Errorf("thumbs: %s (panic)\nstack: %s", r, debug.Stack())
}
}()
if mutex.MainWorker.Canceled() {
return errors.New("canceled")
}
isDir, _ := info.IsDirOrSymlinkToDir()
isSymlink := info.IsSymlink()
if skip, result := fs.SkipWalk(fileName, isDir, isSymlink, done, ignore); skip {
return result
}
mf, err := NewMediaFile(fileName)
if err != nil || mf.Empty() || !mf.IsJpeg() {
return nil
}
done[fileName] = fs.Processed
relativeName := mf.RelName(dir)
event.Publish("index.thumbnails", event.Data{
"fileName": relativeName,
"baseName": filepath.Base(relativeName),
"force": force,
})
jobs <- ThumbsJob{
mediaFile: mf,
path: thumbnailsPath,
force: force,
}
return nil
}
log.Infof("thumbs: processing files in %s folder", clean.Log(filepath.Base(dir)))
if err := ignore.Dir(dir); err != nil {
log.Infof("thumbs: %s", err)
}
err = godirwalk.Walk(dir, &godirwalk.Options{
ErrorCallback: func(fileName string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Callback: handler,
Unsorted: true,
FollowSymbolicLinks: true,
})
close(jobs)
wg.Wait()
return done, err
}

4
internal/photoprism/resample_test.go → internal/photoprism/thumbs_test.go
View file

@ -42,9 +42,9 @@ func TestResample_Start(t *testing.T) {
imp.Start(opt)
rs := NewResample(conf)
rs := NewThumbs(conf)
err := rs.Start(true)
err := rs.Start(true, false)
if err != nil {
t.Fatal(err)

8
internal/photoprism/resample_worker.go → internal/photoprism/thumbs_worker.go
View file

@ -1,22 +1,22 @@
package photoprism
type ResampleJob struct {
type ThumbsJob struct {
mediaFile *MediaFile
path string
force bool
}
func ResampleWorker(jobs <-chan ResampleJob) {
func ThumbsWorker(jobs <-chan ThumbsJob) {
for job := range jobs {
mf := job.mediaFile
if mf == nil {
log.Error("resample: media file is nil - might be a bug")
log.Error("thumbs: media file is nil - might be a bug")
continue
}
if err := mf.CreateThumbnails(job.path, job.force); err != nil {
log.Errorf("resample: %s", err)
log.Errorf("thumbs: %s", err)
}
}
}

19
internal/service/resample.go
View file

@ -1,19 +0,0 @@
package service
import (
"sync"
"github.com/photoprism/photoprism/internal/photoprism"
)
var onceResample sync.Once
func initResample() {
services.Resample = photoprism.NewResample(Config())
}
func Resample() *photoprism.Resample {
onceResample.Do(initResample)
return services.Resample
}

2
internal/service/service.go
View file

@ -56,7 +56,7 @@ var services struct {
Nsfw *nsfw.Detector
FaceNet *face.Net
Query *query.Query
Resample *photoprism.Resample
Thumbs *photoprism.Thumbs
Session *session.Session
}

2
internal/service/service_test.go
View file

@ -81,7 +81,7 @@ func TestQuery(t *testing.T) {
}
func TestResample(t *testing.T) {
assert.IsType(t, &photoprism.Resample{}, Resample())
assert.IsType(t, &photoprism.Thumbs{}, Thumbs())
}
func TestSession(t *testing.T) {

19
internal/service/thumbs.go Normal file
View file

@ -0,0 +1,19 @@
package service
import (
"sync"
"github.com/photoprism/photoprism/internal/photoprism"
)
var onceThumbs sync.Once
func initThumbs() {
services.Thumbs = photoprism.NewThumbs(Config())
}
func Thumbs() *photoprism.Thumbs {
onceThumbs.Do(initThumbs)
return services.Thumbs
}

8
pkg/fs/ignore.go
View file

@ -163,3 +163,11 @@ func (l *IgnoreList) Ignore(fileName string) bool {
return false
}
// Reset resets ignored and hidden files.
func (l *IgnoreList) Reset() {
l.items = []IgnoreItem{}
l.hiddenFiles = []string{}
l.ignoredFiles = []string{}
l.configFiles = make(map[string][]string)
}

4
pkg/fs/ignore_test.go
View file

@ -139,12 +139,12 @@ func TestNewIgnoreItem(t *testing.T) {
}
func TestIgnoreList_AppendItems(t *testing.T) {
t.Run("error", func(t *testing.T) {
t.Run("Error", func(t *testing.T) {
ignoreList := NewIgnoreList(".xyz", false, false)
assert.Error(t, ignoreList.AppendItems("", []string{"__test_"}))
})
t.Run("success", func(t *testing.T) {
t.Run("Success", func(t *testing.T) {
ignoreList := NewIgnoreList(".xyz", false, false)
assert.Nil(t, ignoreList.AppendItems("testdata/directory", []string{"__test_"}))
})