Backend: Fix import command and improve logging #233 #287

Signed-off-by: Michael Mayer <michael@liquidbytes.net>

cmd/photoprism/photoprism.go

@@ -28,7 +28,7 @@ func main() {
 		commands.ImportCommand,
 		commands.CopyCommand,
 		commands.ConvertCommand,
-		commands.ThumbsCommand,
+		commands.ResampleCommand,
 		commands.MigrateCommand,
 		commands.ConfigCommand,
 		commands.VersionCommand,

internal/commands/resample.go

@@ -8,21 +8,22 @@ import (
 	"github.com/urfave/cli"
 )
 
-// ThumbsCommand is used to register the thumbs cli command
+// ResampleCommand is used to register the thumbs cli command
-var ThumbsCommand = cli.Command{
+var ResampleCommand = cli.Command{
-	Name:  "thumbs",
+	Name:  "resample",
-	Usage: "Pre-renders thumbnails to boost performance",
+	Aliases: []string{"thumbs"},
+	Usage: "Pre-renders thumbnails (significantly reduces memory and cpu usage)",
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "force, f",
 			Usage: "re-create existing thumbnails",
 		},
 	},
-	Action: thumbsAction,
+	Action: resampleAction,
 }
 
-// thumbsAction pre-render the thumbnails
+// resampleAction pre-render the thumbnails
-func thumbsAction(ctx *cli.Context) error {
+func resampleAction(ctx *cli.Context) error {
 	start := time.Now()
 
 	conf := config.NewConfig(ctx)

internal/photoprism/convert.go

@@ -85,6 +85,8 @@ func (c *Convert) Start(path string) error {
 				return nil
 			}
 
+			done[fileName] = true
+
 			jobs <- ConvertJob{
 				image:   mf,
 				convert: c,
@@ -131,7 +133,7 @@ func (c *Convert) ConvertCommand(image *MediaFile, jpegName string, xmpName stri
 // ToJpeg converts a single image file to JPEG if possible.
 func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) {
 	if !image.Exists() {
-		return nil, fmt.Errorf("convert: can not convert to jpeg, file does not exist (%s)", image.FileName())
+		return nil, fmt.Errorf("convert: can not convert to jpeg, file does not exist (%s)", image.RelativeName(c.conf.OriginalsPath()))
 	}
 
 	if image.IsJpeg() {
@@ -149,12 +151,12 @@ func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) {
 	jpegName = image.AbsBase(c.conf.Settings().Library.GroupRelated) + ".jpg"
 
 	if c.conf.ReadOnly() {
-		return nil, fmt.Errorf("convert: disabled in read only mode (%s)", image.FileName())
+		return nil, fmt.Errorf("convert: disabled in read only mode (%s)", image.RelativeName(c.conf.OriginalsPath()))
 	}
 
 	fileName := image.RelativeName(c.conf.OriginalsPath())
 
-	log.Infof("convert: %s -> %s", fileName, jpegName)
+	log.Infof("convert: %s -> %s", fileName, fs.RelativeName(jpegName, c.conf.OriginalsPath()))
 
 	xmpName := fs.TypeXMP.Find(image.FileName(), c.conf.Settings().Library.GroupRelated)
 

internal/photoprism/import.go

@@ -99,10 +99,6 @@ func (imp *Import) Start(opt ImportOptions) {
 				return errors.New("import canceled")
 			}
 
-			if done[fileName] {
-				return nil
-			}
-
 			isDir := info.IsDir()
 			isSymlink := info.IsSymlink()
 
@@ -141,7 +137,7 @@ func (imp *Import) Start(opt ImportOptions) {
 				done[f.FileName()] = true
 			}
 
-			done[mf.FileName()] = true
+			done[fileName] = true
 
 			related.Files = files
 

internal/photoprism/import_worker.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/pkg/fs"
 )
 
 type ImportJob struct {
@@ -47,23 +48,23 @@ func ImportWorker(jobs <-chan ImportJob) {
 
 				if related.Main.HasSameName(f) {
 					destinationMainFilename = destinationFilename
-					log.Infof("import: moving main %s file \"%s\" to \"%s\"", f.FileType(), relativeFilename, destinationFilename)
+					log.Infof("import: moving main %s file \"%s\" to \"%s\"", f.FileType(), relativeFilename, fs.RelativeName(destinationFilename, imp.originalsPath()))
 				} else {
-					log.Infof("import: moving related %s file \"%s\" to \"%s\"", f.FileType(), relativeFilename, destinationFilename)
+					log.Infof("import: moving related %s file \"%s\" to \"%s\"", f.FileType(), relativeFilename, fs.RelativeName(destinationFilename, imp.originalsPath()))
 				}
 
 				if opt.Move {
 					if err := f.Move(destinationFilename); err != nil {
-						log.Errorf("import: could not move file to %s (%s)", destinationMainFilename, err.Error())
+						log.Errorf("import: could not move file to %s (%s)", fs.RelativeName(destinationMainFilename, imp.originalsPath()), err.Error())
 					}
 				} else {
 					if err := f.Copy(destinationFilename); err != nil {
-						log.Errorf("import: could not copy file to %s (%s)", destinationMainFilename, err.Error())
+						log.Errorf("import: could not copy file to %s (%s)", fs.RelativeName(destinationMainFilename, imp.originalsPath()), err.Error())
 					}
 				}
 			} else if opt.RemoveExistingFiles {
 				if err := f.Remove(); err != nil {
-					log.Errorf("import: could not delete %s (%s)", f.FileName(), err.Error())
+					log.Errorf("import: could not delete %s (%s)",fs.RelativeName(f.FileName(), importPath), err.Error())
 				} else {
 					log.Infof("import: deleted %s (already exists)", relativeFilename)
 				}
@@ -74,7 +75,7 @@ func ImportWorker(jobs <-chan ImportJob) {
 			importedMainFile, err := NewMediaFile(destinationMainFilename)
 
 			if err != nil {
-				log.Errorf("import: could not index \"%s\" (%s)", destinationMainFilename, err.Error())
+				log.Errorf("import: could not index \"%s\" (%s)", fs.RelativeName(destinationMainFilename, imp.originalsPath()), err.Error())
 
 				continue
 			}
@@ -96,7 +97,7 @@ func ImportWorker(jobs <-chan ImportJob) {
 			related, err := importedMainFile.RelatedFiles(imp.conf.Settings().Library.GroupRelated)
 
 			if err != nil {
-				log.Errorf("import: could not index \"%s\" (%s)", destinationMainFilename, err.Error())
+				log.Errorf("import: could not index \"%s\" (%s)", fs.RelativeName(destinationMainFilename, imp.originalsPath()), err.Error())
 
 				continue
 			}
@@ -109,7 +110,7 @@ func ImportWorker(jobs <-chan ImportJob) {
 				log.Infof("import: %s main %s file \"%s\"", res, related.Main.FileType(), related.Main.RelativeName(ind.originalsPath()))
 				done[related.Main.FileName()] = true
 			} else {
-				log.Warnf("import: no main file for %s (conversion to jpeg failed?)", destinationMainFilename)
+				log.Warnf("import: no main file for %s (conversion to jpeg failed?)", fs.RelativeName(destinationMainFilename, imp.originalsPath()))
 			}
 
 			for _, f := range related.Files {

internal/photoprism/index.go

@@ -141,7 +141,7 @@ func (ind *Index) Start(options IndexOptions) map[string]bool {
 				done[f.FileName()] = true
 			}
 
-			done[mf.FileName()] = true
+			done[fileName] = true
 
 			related.Files = files
 

internal/photoprism/index_worker.go

@@ -1,5 +1,7 @@
 package photoprism
 
+import "github.com/photoprism/photoprism/pkg/fs"
+
 type IndexJob struct {
 	FileName string
 	Related  RelatedFiles
@@ -20,7 +22,7 @@ func IndexWorker(jobs <-chan IndexJob) {
 
 			log.Infof("index: %s main %s file \"%s\"", res, related.Main.FileType(), related.Main.RelativeName(ind.originalsPath()))
 		} else {
-			log.Warnf("index: no main file for %s (conversion to jpeg failed?)", job.FileName)
+			log.Warnf("index: no main file for %s (conversion to jpeg failed?)", fs.RelativeName(job.FileName, ind.originalsPath()))
 		}
 
 		for _, f := range related.Files {

internal/photoprism/resample.go

@@ -82,6 +82,8 @@ func (rs *Resample) Start(force bool) error {
 				return nil
 			}
 
+			done[fileName] = true
+
 			relativeName := mf.RelativeName(originalsPath)
 
 			event.Publish("index.thumbnails", event.Data{

pkg/fs/ignore.go

@@ -112,17 +112,17 @@ func (l *IgnoreList) Dir(dir string) error {
 	}
 
 	if l.configFile == "" {
-		return errors.New("ignore file name not set")
+		return errors.New("empty ignore file name")
 	}
 
 	fileName := filepath.Join(dir, l.configFile)
 
 	if _, ok := l.configFiles[fileName]; ok {
-		return fmt.Errorf("%s already parsed", l.configFile)
+		return nil
 	}
 
 	if !FileExists(fileName) {
-		return fmt.Errorf("%s does not exist", l.configFile)
+		return fmt.Errorf("no %s file found", l.configFile)
 	}
 
 	return l.ConfigFile(fileName)

pkg/fs/walk.go

@@ -10,8 +10,6 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
 	isDone := done[fileName]
 	isIgnored := ignore.Ignore(fileName)
 
-	done[fileName] = true
-
 	if isSymlink {
 		// Symlinks are skipped by default unless they are links to directories
 		skip = true
@@ -39,5 +37,9 @@ func SkipWalk(fileName string, isDir, isSymlink bool, done map[string]bool, igno
 		skip = true
 	}
 
+	if skip {
+		done[fileName] = true
+	}
+
 	return skip, result
 }