luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Index: Skip duplicates and handle files with wrong extension #391

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-07-20 19:48:31 +02:00
parent 714edfebe5
commit a01e54070d
19 changed files with 318 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
internal/entity/album_test.go
View file

@ -227,12 +227,20 @@ func TestNewMonthAlbum(t *testing.T) {
func TestFindAlbumBySlug(t *testing.T) {
t.Run("1 result", func(t *testing.T) {
album := FindAlbumBySlug("holiday-2030", AlbumDefault)
if album == nil {
t.Fatal("expected to find an album")
}
assert.Equal(t, "Holiday2030", album.AlbumTitle)
assert.Equal(t, "holiday-2030", album.AlbumSlug)
})
t.Run("no result", func(t *testing.T) {
album := FindAlbumBySlug("holiday-2030", AlbumMonth)
assert.Nil(t, album)
if album != nil {
t.Fatal("album should be nil")
}
})
}

3
internal/entity/const.go
View file

@ -38,7 +38,8 @@ const (
TypeText = "text"
// Root directories.
RootOriginals = ""
RootUnknown = ""
RootOriginals = "/"
RootExamples = "examples"
RootSidecar = "sidecar"
RootImport = "import"

87
internal/entity/duplicate.go Normal file
View file

@ -0,0 +1,87 @@
package entity
import (
"fmt"
"github.com/photoprism/photoprism/pkg/txt"
)
type Duplicates []Duplicate
type DuplicatesMap map[string]Duplicate
// Duplicate represents an exact file duplicate.
type Duplicate struct {
FileName string `gorm:"type:varbinary(768);primary_key;" json:"Name" yaml:"Name"`
FileRoot string `gorm:"type:varbinary(16);primary_key;default:'/';" json:"Root" yaml:"Root,omitempty"`
FileHash string `gorm:"type:varbinary(128);default:'';index" json:"Hash" yaml:"Hash,omitempty"`
FileSize int64 `json:"Size" yaml:"Size,omitempty"`
ModTime int64 `json:"ModTime" yaml:"-"`
}
func AddDuplicate(fileName, fileRoot, fileHash string, fileSize, modTime int64) error {
if fileName == "" {
return fmt.Errorf("duplicate: file name must not be empty (add)")
} else if fileHash == "" {
return fmt.Errorf("duplicate: file hash must not be empty (add)")
} else if modTime == 0 {
return fmt.Errorf("duplicate: mod time must not be empty (add)")
} else if fileRoot == "" {
return fmt.Errorf("duplicate: file root must not be empty (add)")
}
duplicate := &Duplicate{
FileName: fileName,
FileRoot: fileRoot,
FileHash: fileHash,
FileSize: fileSize,
ModTime: modTime,
}
if err := duplicate.Create(); err == nil {
return nil
} else if err := duplicate.Save(); err != nil {
return err
}
return nil
}
// Find returns a photo from the database.
func (m *Duplicate) Find() error {
return UnscopedDb().First(m, "file_name = ?", m.FileName).Error
}
// Create inserts a new row to the database.
func (m *Duplicate) Create() error {
if m.FileName == "" {
return fmt.Errorf("duplicate: file name must not be empty (create)")
} else if m.FileHash == "" {
return fmt.Errorf("duplicate: file hash must not be empty (create)")
} else if m.ModTime == 0 {
return fmt.Errorf("duplicate: mod time must not be empty (create)")
} else if m.FileRoot == "" {
return fmt.Errorf("duplicate: file root must not be empty (create)")
}
return UnscopedDb().Create(m).Error
}
// Saves the duplicates in the database.
func (m *Duplicate) Save() error {
if m.FileName == "" {
return fmt.Errorf("duplicate: file name must not be empty (save)")
} else if m.FileHash == "" {
return fmt.Errorf("duplicate: file hash must not be empty (save)")
} else if m.ModTime == 0 {
return fmt.Errorf("duplicate: mod time must not be empty (save)")
} else if m.FileRoot == "" {
return fmt.Errorf("duplicate: file root must not be empty (save)")
}
if err := UnscopedDb().Save(m).Error; err != nil {
log.Errorf("duplicate: %s (save %s)", err, txt.Quote(m.FileName))
return err
}
return nil
}

53
internal/entity/duplicate_test.go Normal file
View file

@ -0,0 +1,53 @@
package entity
import (
"testing"
"time"
)
func TestAddDuplicate(t *testing.T) {
t.Run("success", func(t *testing.T) {
if err := AddDuplicate(
"foobar.jpg",
RootOriginals,
"3cad9168fa6acc5c5c2965ddf6ec465ca42fd811",
661851,
time.Date(2019, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
t.Fatal(err)
}
if err := AddDuplicate(
"foobar.jpg",
RootOriginals,
"3cad9168fa6acc5c5c2965ddf6ec465ca42fd818",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
t.Fatal(err)
}
duplicate := &Duplicate{FileName: "foobar.jpg", FileRoot: RootOriginals}
if err := duplicate.Find(); err != nil {
t.Fatal(err)
} else if duplicate.FileHash != "3cad9168fa6acc5c5c2965ddf6ec465ca42fd818"{
t.Fatal("file hash should be 3cad9168fa6acc5c5c2965ddf6ec465ca42fd818")
} else if duplicate.ModTime != time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix() {
t.Fatalf("mod time should be %d", time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix())
}
})
t.Run("error", func(t *testing.T) {
err := AddDuplicate(
"",
"",
"3cad9168fa6acc5c5c2965ddf6ec465ca42fd818",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
)
if err == nil {
t.Fatal("error expected")
}
})
}

6
internal/entity/entity.go
View file

@ -33,6 +33,7 @@ var Entities = Types{
"people": &Person{},
"accounts": &Account{},
"folders": &Folder{},
"duplicates": &Duplicate{},
"files": &File{},
"files_share": &FileShare{},
"files_sync": &FileSync{},
@ -83,12 +84,11 @@ func (list Types) WaitForMigration() {
// Truncate removes all data from tables without dropping them.
func (list Types) Truncate() {
for name := range list {
row := RowCount{}
if err := Db().Raw(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Scan(&row).Error; err == nil {
if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
log.Debugf("entity: removed all data from %s", name)
break
} else if err.Error() != "record not found" {
log.Debugf("entity: truncate %s (%s)", err.Error(), name)
log.Debugf("entity: %s in %s", err, name)
}
}
}

2
internal/entity/file.go
View file

@ -23,7 +23,7 @@ type File struct {
PhotoUID string `gorm:"type:varbinary(42);index;" json:"PhotoUID" yaml:"PhotoUID"`
FileUID string `gorm:"type:varbinary(42);unique_index;" json:"UID" yaml:"UID"`
FileName string `gorm:"type:varbinary(768);unique_index:idx_files_name_root;" json:"Name" yaml:"Name"`
FileRoot string `gorm:"type:varbinary(16);default:'';unique_index:idx_files_name_root;" json:"Root" yaml:"Root,omitempty"`
FileRoot string `gorm:"type:varbinary(16);default:'/';unique_index:idx_files_name_root;" json:"Root" yaml:"Root,omitempty"`
OriginalName string `gorm:"type:varbinary(768);" json:"OriginalName" yaml:"OriginalName,omitempty"`
FileHash string `gorm:"type:varbinary(128);index" json:"Hash" yaml:"Hash,omitempty"`
FileSize int64 `json:"Size" yaml:"Size,omitempty"`

14
internal/entity/file_fixtures.go
View file

@ -12,6 +12,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("19800101_000002_D640C559").PhotoUID,
FileUID: "ft8es39w45bnlqdw",
FileName: "exampleFileName.jpg",
FileRoot: RootOriginals,
OriginalName: "exampleFileNameOriginal.jpg",
FileHash: "2cad9168fa6acc5c5c2965ddf6ec465ca42fd818",
ModTime: time.Date(2020, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -51,6 +52,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo01").PhotoUID,
FileUID: "ft9es39w45bnlqdw",
FileName: "exampleDNGFile.dng",
FileRoot: RootOriginals,
OriginalName: "exampleDNGFile.dng",
FileHash: "3cad9168fa6acc5c5c2965ddf6ec465ca42fd818",
ModTime: time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -87,6 +89,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo01").PhotoUID,
FileUID: "ft1es39w45bnlqdw",
FileName: "exampleXmpFile.xmp",
FileRoot: RootOriginals,
OriginalName: "exampleXmpFile.xmp",
FileHash: "ocad9168fa6acc5c5c2965ddf6ec465ca42fd818",
ModTime: time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -123,6 +126,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo04").PhotoUID,
FileUID: "ft2es39w45bnlqdw",
FileName: "bridge.jpg",
FileRoot: RootOriginals,
OriginalName: "bridgeOriginal.jpg",
FileHash: "pcad9168fa6acc5c5c2965ddf6ec465ca42fd818",
ModTime: time.Date(2017, 2, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -159,6 +163,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo05").PhotoUID,
FileUID: "ft3es39w45bnlqdw",
FileName: "reunion.jpg",
FileRoot: RootOriginals,
OriginalName: "reunionOriginal.jpg",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd818",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -195,6 +200,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo17").PhotoUID,
FileUID: "ft4es39w45bnlqdw",
FileName: "Quality1FavoriteTrue.jpg",
FileRoot: RootOriginals,
OriginalName: "Quality1FavoriteTrue.jpg",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd819",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -231,6 +237,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo15").PhotoUID,
FileUID: "ft5es39w45bnlqdw",
FileName: "missing.jpg",
FileRoot: RootOriginals,
OriginalName: "missing.jpg",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd819",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -267,6 +274,7 @@ var FileFixtures = map[string]File{
PhotoUID: "pt9jtdre2lvl0y25",
FileUID: "ft6es39w45bnlqdw",
FileName: "Photo18.jpg",
FileRoot: RootOriginals,
OriginalName: "Photo18.jpg",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd820",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -303,6 +311,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo10").PhotoUID,
FileUID: "ft71s39w45bnlqdw",
FileName: "Video.mp4",
FileRoot: RootOriginals,
OriginalName: "Video.mp4",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd831",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -339,6 +348,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo10").PhotoUID,
FileUID: "ft72s39w45bnlqdw",
FileName: "VideoError.mp4",
FileRoot: RootOriginals,
OriginalName: "VideoError.mp4",
FileHash: "acad9168fa6acc5c5c2965ddf6ec465ca42fd832",
ModTime: time.Date(2017, 1, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -375,6 +385,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo02").PhotoUID,
FileUID: "ft2es39q45bnlqd0",
FileName: "bridge1.jpg",
FileRoot: RootOriginals,
OriginalName: "bridgeOriginal1.jpg",
FileHash: "pcad9168fa6acc5c5c2965ddf6ec465ca42fd828",
ModTime: time.Date(2017, 2, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -411,6 +422,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo03").PhotoUID,
FileUID: "ft2es49w15bnlqdw",
FileName: "bridge2.jpg",
FileRoot: RootOriginals,
OriginalName: "bridgeOriginal2.jpg",
FileHash: "pcad9168fa6acc5c5c2965adf6ec465ca42fd818",
ModTime: time.Date(2017, 2, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -447,6 +459,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo03").PhotoUID,
FileUID: "ft2es49whhbnlqdn",
FileName: "bridge3.jpg",
FileRoot: RootOriginals,
OriginalName: "bridgeOriginal.jpg",
FileHash: "pcad9168fa6acc5c5ba965adf6ec465ca42fd818",
ModTime: time.Date(2017, 2, 6, 2, 6, 51, 0, time.UTC).Unix(),
@ -483,6 +496,7 @@ var FileFixtures = map[string]File{
PhotoUID: PhotoFixtures.Pointer("Photo03").PhotoUID,
FileUID: "ft2es49whhbnlqdy",
FileName: "bridge.mp4",
FileRoot: RootOriginals,
OriginalName: "bridgeOriginal.mp4",
FileHash: "pcad9168fa6acc5c5ba965adf6ec465ca42fd819",
ModTime: time.Date(2017, 2, 6, 2, 6, 51, 0, time.UTC).Unix(),

2
internal/meta/data.go
View file

@ -66,7 +66,7 @@ func (data Data) AspectRatio() float32 {
return 0
}
aspectRatio := float32(width / height)
aspectRatio := float32(math.Round((width / height)*100)/100)
return aspectRatio
}

2
internal/meta/data_test.go
View file

@ -24,7 +24,7 @@ func TestData_AspectRatio(t *testing.T) {
All: nil,
}
assert.Equal(t, float32(0.8333333), data.AspectRatio())
assert.Equal(t, float32(0.83), data.AspectRatio())
})
t.Run("invalid", func(t *testing.T) {

2
internal/meta/json_test.go
View file

@ -77,7 +77,7 @@ func TestJSON(t *testing.T) {
assert.Equal(t, 1080, data.Height)
assert.Equal(t, 1080, data.ActualWidth())
assert.Equal(t, 1920, data.ActualHeight())
assert.Equal(t, float32(0.5625), data.AspectRatio())
assert.Equal(t, float32(0.56), data.AspectRatio())
assert.Equal(t, 6, data.Orientation)
assert.Equal(t, float32(52.4596), data.Lat)
assert.Equal(t, float32(13.3218), data.Lng)

2
internal/photoprism/convert.go
View file

@ -216,7 +216,7 @@ func (c *Convert) ToJpeg(image *MediaFile) (*MediaFile, error) {
mediaFile, err := NewMediaFile(jpegName)
if err == nil {
if err == nil && mediaFile.IsJpeg() {
return mediaFile, nil
}

25
internal/photoprism/files.go
View file

@ -2,6 +2,7 @@ package photoprism
import (
"fmt"
"path"
"sync"
"time"
@ -44,39 +45,41 @@ func (m *Files) Init() error {
}
// Ignore tests of a file requires indexing, file name must be relative to the originals path.
func (m *Files) Ignore(fileName string, modTime time.Time, rescan bool) bool {
func (m *Files) Ignore(fileName, fileRoot string, modTime time.Time, rescan bool) bool {
timestamp := modTime.Unix()
key := path.Join(fileRoot, fileName)
m.mutex.Lock()
defer m.mutex.Unlock()
if rescan {
m.files[fileName] = timestamp
m.files[key] = timestamp
return false
}
mod, ok := m.files[fileName]
mod, ok := m.files[key]
if ok && mod == timestamp {
return true
} else {
m.files[fileName] = timestamp
m.files[key] = timestamp
return false
}
}
// Indexed tests of a file was already indexed without modifying the files map.
func (m *Files) Indexed(fileName string, modTime time.Time, rescan bool) bool {
func (m *Files) Indexed(fileName, fileRoot string, modTime time.Time, rescan bool) bool {
if rescan {
return false
}
timestamp := modTime.Unix()
key := path.Join(fileRoot, fileName)
m.mutex.RLock()
defer m.mutex.RUnlock()
mod, ok := m.files[fileName]
mod, ok := m.files[key]
if ok && mod == timestamp {
return true
@ -84,13 +87,3 @@ func (m *Files) Indexed(fileName string, modTime time.Time, rescan bool) bool {
return false
}
}
// Add adds or updates a file on the list.
func (m *Files) Add(fileName string, modTime time.Time) {
timestamp := modTime.Unix()
m.mutex.Lock()
defer m.mutex.Unlock()
m.files[fileName] = timestamp
}

31
internal/photoprism/files_test.go
View file

@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/stretchr/testify/assert"
)
@ -14,16 +15,22 @@ func TestFiles_Ignore(t *testing.T) {
t.Fatal(err)
}
assert.True(t, files.Ignore("exampleFileName.jpg", time.Unix(1583460411, 0), false))
assert.False(t, files.Ignore("exampleFileName.jpg", time.Unix(1583460412, 0), false))
assert.True(t, files.Ignore("exampleFileName.jpg", time.Unix(1583460412, 0), false))
assert.False(t, files.Ignore("exampleFileName.jpg", time.Unix(1583460412, 0), true))
assert.False(t, files.Ignore("exampleFileName.jpg", time.Unix(500, 0), false))
assert.True(t, files.Ignore("exampleFileName.jpg", time.Unix(500, 0), false))
assert.False(t, files.Ignore("new-file.jpg", time.Unix(1583460000, 1), false))
assert.True(t, files.Ignore("new-file.jpg", time.Unix(1583460000, 2), false))
assert.False(t, files.Ignore("new-file.jpg", time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", time.Unix(1583460001, 2), true))
assert.True(t, files.Ignore("new-file.jpg", time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", time.Unix(501, 0), false))
assert.True(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals, time.Unix(1583460411, 0), false))
assert.False(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals,time.Unix(1583460412, 0), false))
assert.True(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals,time.Unix(1583460412, 0), false))
assert.False(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals,time.Unix(1583460412, 0), true))
assert.False(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals,time.Unix(500, 0), false))
assert.True(t, files.Ignore("exampleFileName.jpg", entity.RootOriginals,time.Unix(500, 0), false))
assert.False(t, files.Ignore("new-file.jpg",entity.RootOriginals, time.Unix(1583460000, 1), false))
assert.True(t, files.Ignore("new-file.jpg",entity.RootOriginals,time.Unix(1583460000, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootOriginals,time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootOriginals,time.Unix(1583460001, 2), true))
assert.True(t, files.Ignore("new-file.jpg",entity.RootOriginals, time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootOriginals,time.Unix(501, 0), false))
assert.False(t, files.Ignore("new-file.jpg",entity.RootSidecar, time.Unix(1583460000, 1), false))
assert.True(t, files.Ignore("new-file.jpg",entity.RootSidecar,time.Unix(1583460000, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootSidecar,time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootSidecar,time.Unix(1583460001, 2), true))
assert.True(t, files.Ignore("new-file.jpg",entity.RootSidecar, time.Unix(1583460001, 2), false))
assert.False(t, files.Ignore("new-file.jpg", entity.RootSidecar,time.Unix(501, 0), false))
}

4
internal/photoprism/index.go
View file

@ -151,7 +151,7 @@ func (ind *Index) Start(opt IndexOptions) fs.Done {
return nil
}
if ind.files.Indexed(relName, mf.modTime, opt.Rescan) {
if ind.files.Indexed(relName, entity.RootOriginals, mf.modTime, opt.Rescan) {
return nil
}
@ -170,7 +170,7 @@ func (ind *Index) Start(opt IndexOptions) fs.Done {
continue
}
if ind.files.Indexed(f.RootRelName(), f.ModTime(), opt.Rescan) {
if ind.files.Indexed(f.RootRelName(), f.Root(), f.ModTime(), opt.Rescan) {
done[f.FileName()] = fs.Found
continue
}

33
internal/photoprism/index_mediafile.go
View file

@ -3,7 +3,6 @@ package photoprism
import (
"errors"
"fmt"
"math"
"path/filepath"
"sort"
"strings"
@ -70,7 +69,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
return result
}
if ind.files.Ignore(m.RootRelName(), m.ModTime(), o.Rescan) {
if ind.files.Ignore(m.RootRelName(), m.Root(), m.ModTime(), o.Rescan) {
result.Status = IndexSkipped
return result
}
@ -116,7 +115,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
"baseName": filepath.Base(fileName),
})
fileQuery = entity.UnscopedDb().First(&file, "file_name = ?", fileName)
fileQuery = entity.UnscopedDb().First(&file, "file_name = ? AND (file_root = ? OR file_root = '')", fileName, fileRoot)
fileExists = fileQuery.Error == nil
if !fileExists && !m.IsSidecar() {
@ -125,6 +124,10 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
fileExists = fileQuery.Error == nil
if fileExists && fs.FileExists(FileName(file.FileRoot, file.FileName)) {
if err := entity.AddDuplicate(m.RootRelName(), m.Root(), m.Hash(), m.FileSize(), m.ModTime().Unix()); err != nil {
log.Error(err)
}
result.Status = IndexDuplicate
return result
}
@ -242,9 +245,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
file.FileWidth = m.Width()
file.FileHeight = m.Height()
file.FileAspectRatio = m.AspectRatio()
file.FilePortrait = m.Width() < m.Height()
file.FilePortrait = m.Portrait()
megapixels := int(math.Round(float64(file.FileWidth*file.FileHeight) / 1000000))
megapixels := m.Megapixels()
if megapixels > photo.PhotoResolution {
photo.PhotoResolution = megapixels
@ -318,13 +321,13 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
}
file.FileCodec = metaData.Codec
file.FileWidth = metaData.ActualWidth()
file.FileHeight = metaData.ActualHeight()
file.FileAspectRatio = metaData.AspectRatio()
file.FilePortrait = metaData.Portrait()
file.FileWidth = m.Width()
file.FileHeight = m.Height()
file.FileAspectRatio = m.AspectRatio()
file.FilePortrait = m.Portrait()
file.FileProjection = metaData.Projection
if res := metaData.Megapixels(); res > photo.PhotoResolution {
if res := m.Megapixels(); res > photo.PhotoResolution {
photo.PhotoResolution = res
}
}
@ -379,14 +382,14 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
}
file.FileCodec = metaData.Codec
file.FileWidth = metaData.ActualWidth()
file.FileHeight = metaData.ActualHeight()
file.FileWidth = m.Width()
file.FileHeight = m.Height()
file.FileAspectRatio = m.AspectRatio()
file.FilePortrait = m.Portrait()
file.FileDuration = metaData.Duration
file.FileAspectRatio = metaData.AspectRatio()
file.FilePortrait = metaData.Portrait()
file.FileProjection = metaData.Projection
if res := metaData.Megapixels(); res > photo.PhotoResolution {
if res := m.Megapixels(); res > photo.PhotoResolution {
photo.PhotoResolution = res
}
}

98
internal/photoprism/mediafile.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"image"
"io"
"math"
"os"
"path"
"path/filepath"
@ -26,6 +27,7 @@ import (
// MediaFile represents a single photo, video or sidecar file.
type MediaFile struct {
fileName string
fileRoot string
statErr error
modTime time.Time
fileSize int64
@ -35,6 +37,7 @@ type MediaFile struct {
takenAtSrc string
hash string
checksum string
hasJpeg bool
width int
height int
metaData meta.Data
@ -46,8 +49,11 @@ type MediaFile struct {
func NewMediaFile(fileName string) (*MediaFile, error) {
m := &MediaFile{
fileName: fileName,
fileRoot: entity.RootUnknown,
fileType: fs.TypeOther,
metaData: meta.NewData(),
width: -1,
height: -1,
}
if _, _, err := m.Stat(); err != nil {
@ -333,6 +339,7 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e
// PathNameInfo returns file name infos for indexing.
func (m *MediaFile) PathNameInfo() (fileRoot, fileBase, relativePath, relativeName string) {
fileRoot = m.Root()
var rootPath string
switch fileRoot {
@ -342,6 +349,8 @@ func (m *MediaFile) PathNameInfo() (fileRoot, fileBase, relativePath, relativeNa
rootPath = Config().ImportPath()
case entity.RootExamples:
rootPath = Config().ExamplesPath()
case entity.RootOriginals:
rootPath = Config().OriginalsPath()
default:
rootPath = Config().OriginalsPath()
}
@ -366,6 +375,7 @@ func (m *MediaFile) BaseName() string {
// SetFileName sets the filename to the given string.
func (m *MediaFile) SetFileName(fileName string) {
m.fileName = fileName
m.fileRoot = entity.RootUnknown
}
// RootRelName returns the relative filename and automatically detects the root path.
@ -448,29 +458,37 @@ func (m *MediaFile) BasePrefix(stripSequence bool) string {
// Root returns the file root directory.
func (m *MediaFile) Root() string {
if m.fileRoot != entity.RootUnknown {
return m.fileRoot
}
if strings.HasPrefix(m.FileName(), Config().OriginalsPath()) {
return entity.RootOriginals
m.fileRoot = entity.RootOriginals
return m.fileRoot
}
importPath := Config().ImportPath()
if importPath != "" && strings.HasPrefix(m.FileName(), importPath) {
return entity.RootImport
m.fileRoot = entity.RootImport
return m.fileRoot
}
sidecarPath := Config().SidecarPath()
if sidecarPath != "" && strings.HasPrefix(m.FileName(), sidecarPath) {
return entity.RootSidecar
m.fileRoot = entity.RootSidecar
return m.fileRoot
}
examplesPath := Config().ExamplesPath()
if examplesPath != "" && strings.HasPrefix(m.FileName(), examplesPath) {
return entity.RootExamples
m.fileRoot = entity.RootExamples
return m.fileRoot
}
return ""
return m.fileRoot
}
// AbsPrefix returns the directory and base filename without any extensions.
@ -527,7 +545,7 @@ func (m *MediaFile) Move(dest string) error {
if err := os.Rename(m.fileName, dest); err != nil {
log.Debugf("failed renaming file, fallback to copy and delete: %s", err.Error())
} else {
m.fileName = dest
m.SetFileName(dest)
return nil
}
@ -540,7 +558,7 @@ func (m *MediaFile) Move(dest string) error {
return err
}
m.fileName = dest
m.SetFileName(dest)
return nil
}
@ -725,11 +743,24 @@ func (m *MediaFile) Jpeg() (*MediaFile, error) {
// ContainsJpeg returns true if this file has or is a jpeg media file.
func (m *MediaFile) HasJpeg() bool {
if m.IsJpeg() {
if m.hasJpeg {
return true
}
return fs.TypeJpeg.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false) != ""
if m.IsJpeg() {
m.hasJpeg = true
return true
}
jpegName := fs.TypeJpeg.FindFirst(m.FileName(), []string{Config().SidecarPath(), fs.HiddenPath}, Config().OriginalsPath(), false)
if jpegName == "" {
m.hasJpeg = false
} else {
m.hasJpeg = fs.MimeType(jpegName) == fs.MimeTypeJpeg
}
return m.hasJpeg
}
// HasJson returns true if this file has or is a json sidecar file.
@ -746,15 +777,6 @@ func (m *MediaFile) decodeDimensions() error {
return fmt.Errorf("failed decoding dimensions for %s", txt.Quote(m.BaseName()))
}
var width, height int
data := m.MetaData()
if data.Error == nil {
width = data.Width
height = data.Height
}
if m.IsJpeg() || m.IsPng() || m.IsGif() {
file, err := os.Open(m.FileName())
@ -770,16 +792,18 @@ func (m *MediaFile) decodeDimensions() error {
return err
}
width = size.Width
height = size.Height
}
if m.Orientation() > 4 {
m.width = height
m.height = width
if m.Orientation() > 4 {
m.width = size.Height
m.height = size.Width
} else {
m.width = size.Width
m.height = size.Height
}
} else if data := m.MetaData(); data.Error == nil {
m.width = data.ActualWidth()
m.height = data.ActualHeight()
} else {
m.width = width
m.height = height
return data.Error
}
return nil
@ -791,9 +815,9 @@ func (m *MediaFile) Width() int {
return 0
}
if m.width <= 0 {
if m.width < 0 {
if err := m.decodeDimensions(); err != nil {
log.Error(err)
log.Debugf("mediafile: %s", err)
}
}
@ -806,9 +830,9 @@ func (m *MediaFile) Height() int {
return 0
}
if m.height <= 0 {
if m.height < 0 {
if err := m.decodeDimensions(); err != nil {
log.Error(err)
log.Debugf("mediafile: %s", err)
}
}
@ -824,11 +848,21 @@ func (m *MediaFile) AspectRatio() float32 {
return 0
}
aspectRatio := float32(width / height)
aspectRatio := float32(math.Round((width / height)*100)/100)
return aspectRatio
}
// Portrait tests if the image is a portrait.
func (m *MediaFile) Portrait() bool {
return m.Width() < m.Height()
}
// Megapixels returns the resolution in megapixels.
func (m *MediaFile) Megapixels() int {
return int(math.Round(float64(m.Width()*m.Height()) / 1000000))
}
// Orientation returns the orientation of a MediaFile.
func (m *MediaFile) Orientation() int {
if data := m.MetaData(); data.Error == nil {

6
internal/photoprism/mediafile_test.go
View file

@ -1618,7 +1618,7 @@ func TestMediaFile_AspectRatio(t *testing.T) {
}
ratio := mediaFile.AspectRatio()
assert.Equal(t, float32(1.5015106), ratio)
assert.Equal(t, float32(1.5), ratio)
})
}
@ -1891,7 +1891,7 @@ func TestMediaFile_PathNameInfo(t *testing.T) {
t.Log(Config().SidecarPath())
t.Log(Config().ImportPath())
mediaFile, err := NewMediaFile(conf.ExamplesPath() + "/beach_sand.jpg")
mediaFile, err := NewMediaFile(filepath.Join(conf.ExamplesPath(), "beach_sand.jpg"))
if err != nil {
t.Fatal(err)
@ -1899,7 +1899,7 @@ func TestMediaFile_PathNameInfo(t *testing.T) {
initialName := mediaFile.FileName()
t.Log(initialName)
mediaFile.SetFileName("/go/src/github.com/photoprism/photoprism/storage/testdata/import/beach_sand.jpg")
mediaFile.SetFileName(filepath.Join(conf.ImportPath(), "beach_sand.jpg"))
root, base, path, name := mediaFile.PathNameInfo()
assert.Equal(t, "import", root)

22
internal/query/files.go
View file

@ -1,6 +1,7 @@
package query
import (
"path"
"strings"
"github.com/photoprism/photoprism/internal/entity"
@ -109,21 +110,32 @@ type FileMap map[string]int64
func IndexedFiles() (result FileMap, err error) {
result = make(FileMap)
type Files struct {
type File struct {
FileRoot string
FileName string
ModTime int64
}
var files []Files
// Query known duplicates.
var duplicates []File
sql := "SELECT file_name, mod_time FROM files"
if err := UnscopedDb().Raw("SELECT file_root, file_name, mod_time FROM duplicates").Scan(&duplicates).Error; err != nil {
return result, err
}
if err := UnscopedDb().Raw(sql).Scan(&files).Error; err != nil {
for _, row := range duplicates {
result[path.Join(row.FileRoot, row.FileName)] = row.ModTime
}
// Query indexed files.
var files []File
if err := UnscopedDb().Raw("SELECT file_root, file_name, mod_time FROM files").Scan(&files).Error; err != nil {
return result, err
}
for _, row := range files {
result[row.FileName] = row.ModTime
result[path.Join(row.FileRoot, row.FileName)] = row.ModTime
}
return result, err

11
internal/query/files_test.go
View file

@ -2,6 +2,7 @@ package query
import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/stretchr/testify/assert"
@ -185,6 +186,16 @@ func TestSetFileError(t *testing.T) {
}
func TestIndexedFiles(t *testing.T) {
if err := entity.AddDuplicate(
"Photo18.jpg",
entity.RootSidecar,
"3cad9168fa6acc5c5c2965ddf6ec465ca42fd818",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
t.Fatal(err)
}
result, err := IndexedFiles()
if err != nil {