luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Detect and report photos without primary file, e.g. after purging #234

Browse source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer 2020-05-08 12:01:22 +02:00
parent 3288ede6e6
commit 1894b4440f
10 changed files with 81 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
assets/resources/examples/Screenshot 2019-05-21 at 10.45.52.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 91 KiB

13
frontend/src/pages/library/originals.vue
View file

@ -66,6 +66,18 @@
<translate>Index</translate>
<v-icon right dark>update</v-icon>
</v-btn>
<v-alert
:value="true"
color="error"
icon="priority_high"
class="mt-3"
outline
v-if="config.count.hidden > 0"
>
The index currently contains {{ config.count.hidden }} hidden files. Their format may not be supported,
they haven't been converted to JPEG yet or there are duplicates.
</v-alert>
</v-container>
</v-form>
</div>
@ -87,6 +99,7 @@
return {
settings: new Settings(this.$config.settings()),
readonly: this.$config.get("readonly"),
config: this.$config.values,
started: false,
busy: false,
loading: false,

4
internal/config/client.go
View file

@ -59,6 +59,7 @@ func (c *Config) PublicClientConfig() ClientConfig {
var count = struct {
Photos uint `json:"photos"`
Hidden uint `json:"hidden"`
Favorites uint `json:"favorites"`
Private uint `json:"private"`
Stories uint `json:"stories"`
@ -131,6 +132,7 @@ func (c *Config) ClientConfig() ClientConfig {
var count = struct {
Photos uint `json:"photos"`
Hidden uint `json:"hidden"`
Favorites uint `json:"favorites"`
Private uint `json:"private"`
Stories uint `json:"stories"`
@ -141,7 +143,7 @@ func (c *Config) ClientConfig() ClientConfig {
}{}
db.Table("photos").
Select("COUNT(*) AS photos, SUM(photo_favorite) AS favorites, SUM(photo_private) AS private, SUM(photo_story) AS stories").
Select("SUM(photo_quality = -1) AS hidden, SUM(photo_quality >= 0) AS photos, SUM(photo_favorite) AS favorites, SUM(photo_private) AS private, SUM(photo_story) AS stories").
Where("deleted_at IS NULL").
Take(&count)

4
internal/entity/file.go
View file

@ -35,8 +35,8 @@ type File struct {
FileOrientation int
FileAspectRatio float32 `gorm:"type:FLOAT;"`
FileMainColor string `gorm:"type:varbinary(16);index;"`
FileColors string `gorm:"type:binary(9);"`
FileLuminance string `gorm:"type:binary(9);"`
FileColors string `gorm:"type:varbinary(9);"`
FileLuminance string `gorm:"type:varbinary(9);"`
FileDiff uint32
FileChroma uint8
FileNotes string `gorm:"type:text"`

13
internal/photoprism/colors_test.go
View file

@ -52,6 +52,12 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
Luminance: colors.LightMap{0x9, 0x5, 0xb, 0x6, 0x1, 0x6, 0xa, 0x1, 0x8},
Chroma: 20,
},
"Screenshot 2019-05-21 at 10.45.52.png": {
Colors: colors.Colors{},
MainColor: 0,
Luminance: colors.LightMap{},
Chroma: 0,
},
}
if err := fastwalk.Walk(conf.ExamplesPath(), func(filename string, info os.FileMode) error {
@ -61,7 +67,12 @@ func TestMediaFile_Colors_Testdata(t *testing.T) {
mediaFile, err := NewMediaFile(filename)
if err != nil || !mediaFile.IsJpeg() {
if err != nil {
t.Fatal(err)
}
if !mediaFile.IsJpeg() {
t.Logf("not a jpeg: %s", filepath.Base(mediaFile.FileName()))
return nil
}

16
internal/photoprism/index_mediafile.go
View file

@ -56,14 +56,16 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
start := time.Now()
var photo entity.Photo
var description entity.Description
var file, primaryFile entity.File
var metaData meta.Data
var photoQuery, fileQuery *gorm.DB
var locKeywords []string
file, primaryFile := entity.File{}, entity.File{}
photo := entity.Photo{}
metaData := meta.Data{}
description := entity.Description{}
labels := classify.Labels{}
fileBase := m.Base(ind.conf.Settings().Index.Group)
filePath := m.RelativePath(ind.originalsPath())
fileName := m.RelativeName(ind.originalsPath())
@ -120,6 +122,8 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
if photoExists {
ind.db.Model(&photo).Related(&description)
} else {
photo.PhotoQuality = -1
}
if fileHash == "" {
@ -377,7 +381,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
log.Warnf("%s (%s)", err.Error(), photo.PhotoUUID)
}
} else {
photo.PhotoQuality = photo.QualityScore()
if photo.PhotoQuality >= 0 {
photo.PhotoQuality = photo.QualityScore()
}
if err := ind.db.Unscoped().Save(&photo).Error; err != nil {
log.Errorf("index: %s", err)

2
internal/photoprism/purge.go
View file

@ -163,6 +163,8 @@ func (prg *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPh
offset += limit
}
err = q.ResetPhotosQuality()
return purgedFiles, purgedPhotos, err
}

7
internal/query/photo.go
View file

@ -469,3 +469,10 @@ func (q *Query) MissingPhotos(limit int, offset int) (entities []entity.Photo, e
return entities, err
}
// ResetPhotosQuality resets the quality of photos without primary file to -1.
func (q *Query) ResetPhotosQuality() error {
return q.db.Table("photos").
Where("id IN (SELECT photos.id FROM photos LEFT JOIN files ON photos.id = files.photo_id AND files.file_primary = 1 WHERE files.id IS NULL GROUP BY photos.id)").
Update("photo_quality", -1).Error
}

14
pkg/fs/base.go
View file

@ -2,6 +2,7 @@ package fs
import (
"path/filepath"
"strconv"
"strings"
)
@ -9,8 +10,8 @@ import (
func Base(fileName string, stripSequence bool) string {
basename := filepath.Base(fileName)
if end := strings.Index(basename, "."); end != -1 {
// ignore everything behind the first dot in the file name
// strip file type extension
if end := strings.LastIndex(basename, "."); end != -1 {
basename = basename[:end]
}
@ -18,7 +19,14 @@ func Base(fileName string, stripSequence bool) string {
return basename
}
// common sequential naming schemes
// strip numeric extensions like .0000, .0001, .4542353245,....
if dot := strings.LastIndex(basename, "."); dot != -1 {
if i, err := strconv.Atoi(basename[dot+1:]); err == nil && i >= 0 {
basename = basename[:dot]
}
}
// other common sequential naming schemes
if end := strings.Index(basename, " ("); end != -1 {
// copies created by Chrome & Windows, example: IMG_1234 (2)
basename = basename[:end]

27
pkg/fs/base_test.go
View file

@ -7,13 +7,15 @@ import (
)
func TestBase(t *testing.T) {
t.Run("Test.jpg", func(t *testing.T) {
result := Base("/testdata/Test.jpg", true)
assert.Equal(t, "Test", result)
t.Run("Screenshot 2019-05-21 at 10.45.52.png", func(t *testing.T) {
regular := Base("Screenshot 2019-05-21 at 10.45.52.png", false)
assert.Equal(t, "Screenshot 2019-05-21 at 10.45.52", regular)
stripped := Base("Screenshot 2019-05-21 at 10.45.52.png", true)
assert.Equal(t, "Screenshot 2019-05-21 at 10.45", stripped)
})
t.Run("Test.3453453.jpg", func(t *testing.T) {
result := Base("/testdata/Test.3453453.jpg", true)
t.Run("Test.jpg", func(t *testing.T) {
result := Base("/testdata/Test.jpg", true)
assert.Equal(t, "Test", result)
})
@ -33,8 +35,19 @@ func TestBase(t *testing.T) {
})
t.Run("Test.3453453.jpg", func(t *testing.T) {
result := Base("/testdata/Test.3453453.jpg", false)
assert.Equal(t, "Test", result)
regular := Base("/testdata/Test.3453453.jpg", false)
assert.Equal(t, "Test.3453453", regular)
stripped := Base("/testdata/Test.3453453.jpg", true)
assert.Equal(t, "Test", stripped)
})
t.Run("/foo/bar.0000.ZIP", func(t *testing.T) {
regular := Base("/foo/bar.0000.ZIP", false)
assert.Equal(t, "bar.0000", regular)
stripped := Base("/foo/bar.0000.ZIP", true)
assert.Equal(t, "bar", stripped)
})
t.Run("Test copy 3.jpg", func(t *testing.T) {