luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

People: Refactor face cropping #22

Browse source
This commit is contained in:
Michael Mayer 2021-09-05 17:10:52 +02:00
parent b9d1c7afb3
commit 6d1179dc03
25 changed files with 286 additions and 225 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
frontend/src/model/marker.js
View file

@ -41,7 +41,7 @@ export class Marker extends RestModel {
UID: "",
FileUID: "",
FileHash: "",
FileArea: "",
CropArea: "",
Type: "",
Src: "",
Name: "",
@ -84,12 +84,12 @@ export class Marker extends RestModel {
thumbnailUrl(size) {
if (!size) {
size = "crop_160";
size = "tile_160";
}
if (this.FileHash && this.FileArea) {
if (this.FileHash && this.CropArea) {
return `${config.contentUri}/t/${this.FileHash}/${config.previewToken()}/${size}/${
this.FileArea
this.CropArea
}`;
} else {
return `${config.contentUri}/svg/portrait`;

79
internal/api/photo_thumb.go
View file

@ -7,7 +7,6 @@ import (
"github.com/gin-gonic/gin"
"github.com/photoprism/photoprism/internal/crop"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/internal/service"
@ -31,6 +30,8 @@ func GetThumb(router *gin.RouterGroup) {
return
}
logPrefix := "thumb"
start := time.Now()
conf := service.Config()
fileHash := c.Param("hash")
@ -40,7 +41,7 @@ func GetThumb(router *gin.RouterGroup) {
size, ok := thumb.Sizes[thumbName]
if !ok {
log.Errorf("thumbs: invalid size %s", thumbName)
log.Errorf("%s: invalid size %s", logPrefix, thumbName)
c.Data(http.StatusOK, "image/svg+xml", photoIconSvg)
return
}
@ -49,7 +50,7 @@ func GetThumb(router *gin.RouterGroup) {
thumbName, size = thumb.Find(conf.ThumbSizePrecached())
if thumbName == "" {
log.Errorf("thumbs: invalid size %d", conf.ThumbSizePrecached())
log.Errorf("%s: invalid size %d", logPrefix, conf.ThumbSizePrecached())
c.Data(http.StatusOK, "image/svg+xml", photoIconSvg)
return
}
@ -64,7 +65,7 @@ func GetThumb(router *gin.RouterGroup) {
cached := cacheData.(ThumbCache)
if !fs.FileExists(cached.FileName) {
log.Errorf("thumbs: %s not found", fileHash)
log.Errorf("%s: %s not found", logPrefix, fileHash)
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
}
@ -115,16 +116,16 @@ func GetThumb(router *gin.RouterGroup) {
fileName := photoprism.FileName(f.FileRoot, f.FileName)
if !fs.FileExists(fileName) {
log.Errorf("thumbs: file %s is missing", txt.Quote(f.FileName))
log.Errorf("%s: file %s is missing", logPrefix, txt.Quote(f.FileName))
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
// Set missing flag so that the file doesn't show up in search results anymore.
logError("thumbs", f.Update("FileMissing", true))
logError(logPrefix, f.Update("FileMissing", true))
if f.AllFilesMissing() {
log.Infof("thumbs: deleting photo, all files missing for %s", txt.Quote(f.FileName))
log.Infof("%s: deleting photo, all files missing for %s", logPrefix, txt.Quote(f.FileName))
logError("thumbs", f.RelatedPhoto().Delete(false))
logError(logPrefix, f.RelatedPhoto().Delete(false))
}
return
@ -132,7 +133,7 @@ func GetThumb(router *gin.RouterGroup) {
// Use original file if thumb size exceeds limit, see https://github.com/photoprism/photoprism/issues/157
if size.ExceedsLimit() && c.Query("download") == "" {
log.Debugf("thumbs: using original, size exceeds limit (width %d, height %d)", size.Width, size.Height)
log.Debugf("%s: using original, size exceeds limit (width %d, height %d)", logPrefix, size.Width, size.Height)
AddThumbCacheHeader(c)
c.File(fileName)
@ -149,11 +150,11 @@ func GetThumb(router *gin.RouterGroup) {
}
if err != nil {
log.Errorf("thumbs: %s", err)
log.Errorf("%s: %s", logPrefix, err)
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
} else if thumbnail == "" {
log.Errorf("thumbs: %s has empty thumb name - bug?", filepath.Base(fileName))
log.Errorf("%s: %s has empty thumb name - bug?", logPrefix, filepath.Base(fileName))
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
}
@ -170,59 +171,3 @@ func GetThumb(router *gin.RouterGroup) {
}
})
}
// GetThumbCrop returns a cropped thumbnail image matching the hash and type.
//
// GET /api/v1/t/:hash/:token/:size/:area
//
// Parameters:
// hash: string sha1 file hash
// token: string url security token, see config
// size: string thumb type, see thumb.Sizes
// area: string image area identifier, e.g. 022004010015
func GetThumbCrop(router *gin.RouterGroup) {
router.GET("/t/:hash/:token/:size/:area", func(c *gin.Context) {
if InvalidPreviewToken(c) {
c.Data(http.StatusForbidden, "image/svg+xml", brokenIconSvg)
return
}
conf := service.Config()
fileHash := c.Param("hash")
thumbName := thumb.Name(c.Param("size"))
cropArea := c.Param("area")
download := c.Query("download") != ""
size, ok := thumb.Sizes[thumbName]
if !ok || len(size.Options) < 1 {
log.Errorf("thumbs: invalid size %s", thumbName)
c.Data(http.StatusOK, "image/svg+xml", photoIconSvg)
return
} else if size.Options[0] != thumb.ResampleCrop {
log.Errorf("thumbs: invalid size %s", thumbName)
c.Data(http.StatusOK, "image/svg+xml", photoIconSvg)
return
}
fileName, err := crop.FromCache(fileHash, conf.ThumbPath(), size.Width, size.Height, cropArea)
if err != nil {
log.Errorf("thumbs: %s", err)
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
} else if fileName == "" {
log.Errorf("thumbs: empty file name, potential bug")
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
}
AddThumbCacheHeader(c)
if download {
c.FileAttachment(fileName, thumbName.Jpeg())
} else {
c.File(fileName)
}
})
}

64
internal/api/photo_thumb_crop.go Normal file
View file

@ -0,0 +1,64 @@
package api
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/photoprism/photoprism/internal/crop"
"github.com/photoprism/photoprism/internal/service"
)
// GetThumbCrop returns a cropped thumbnail image matching the hash and type.
//
// GET /api/v1/t/:hash/:token/:size/:crop
//
// Parameters:
// hash: string sha1 file hash
// token: string url security token, see config
// size: string crop size, see crop.Sizes
// area: string image area identifier, e.g. 1690960ff17f
func GetThumbCrop(router *gin.RouterGroup) {
router.GET("/t/:hash/:token/:size/:area", func(c *gin.Context) {
if InvalidPreviewToken(c) {
c.Data(http.StatusForbidden, "image/svg+xml", brokenIconSvg)
return
}
logPrefix := "thumb-crop"
conf := service.Config()
fileHash := c.Param("hash")
cropName := crop.Name(c.Param("size"))
cropArea := c.Param("area")
download := c.Query("download") != ""
cropSize, ok := crop.Sizes[cropName]
if !ok {
log.Errorf("%s: invalid size %s", logPrefix, cropName)
c.Data(http.StatusOK, "image/svg+xml", photoIconSvg)
return
}
fileName, err := crop.FromCache(fileHash, cropArea, cropSize, conf.ThumbPath())
if err != nil {
log.Warnf("%s: %s", logPrefix, err)
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
} else if fileName == "" {
log.Errorf("%s: empty file name, potential bug", logPrefix)
c.Data(http.StatusOK, "image/svg+xml", brokenIconSvg)
return
}
AddThumbCacheHeader(c)
if download {
c.FileAttachment(fileName, cropName.Jpeg())
} else {
c.File(fileName)
}
})
}

2
internal/api/photo_thumb_test.go
View file

@ -52,7 +52,7 @@ func TestGetThumbCrop(t *testing.T) {
t.Run("NotFound", func(t *testing.T) {
app, router, conf := NewApiTest()
GetThumbCrop(router)
r := PerformRequest(app, "GET", "/api/v1/t/46f5b5c0c027f0c1b15136644f404c57210bf20c/"+conf.PreviewToken()+"/crop_160/016014058037")
r := PerformRequest(app, "GET", "/api/v1/t/46f5b5c0c027f0c1b15136644f404c57210bf20c/"+conf.PreviewToken()+"/tile_160/016014058037")
assert.Equal(t, http.StatusOK, r.Code)
})

18
internal/crop/area.go
View file

@ -2,6 +2,7 @@ package crop
import (
"fmt"
"image"
)
// Areas represents a list of relative crop areas.
@ -16,9 +17,20 @@ type Area struct {
H float32 `json:"h,omitempty"`
}
// String returns a string identifying the approximate marker area.
func (m Area) String() string {
return fmt.Sprintf("%03d%03d%03d%03d", int(m.X*100), int(m.Y*100), int(m.W*100), int(m.H*100))
// String returns a string identifying the crop area.
func (a Area) String() string {
return fmt.Sprintf("%03x%03x%03x%03x", int(a.X*1000), int(a.Y*1000), int(a.W*1000), int(a.H*1000))
}
// Bounds returns absolute coordinates and dimension.
func (a Area) Bounds(img image.Image) (min, max image.Point, dim int) {
size := img.Bounds().Max
min = image.Point{X: int(float32(size.X) * a.X), Y: int(float32(size.Y) * a.Y)}
max = image.Point{X: int(float32(size.X) * (a.X + a.W)), Y: int(float32(size.Y) * (a.Y + a.H))}
dim = int(float32(size.X) * a.W)
return min, max, dim
}
// clipVal ensures the relative size is within a valid range.

26
internal/crop/area_test.go
View file

@ -1,30 +1,36 @@
package crop
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestArea_String(t *testing.T) {
t.Run("082016010006_face", func(t *testing.T) {
t.Run("3e814d3e81f4", func(t *testing.T) {
expected := fmt.Sprintf("%x%x00%x%x", 1000, 333, 1, 500)
m := NewArea("face", 1.000, 0.33333, 0.001, 0.5)
assert.Equal(t, expected, m.String())
})
t.Run("3360a7064042_face", func(t *testing.T) {
m := NewArea("face", 0.822059, 0.167969, 0.1, 0.0664062)
assert.Equal(t, "082016010006", m.String())
assert.Equal(t, "3360a7064042", m.String())
})
t.Run("082016010006_back", func(t *testing.T) {
t.Run("3360a7064042_back", func(t *testing.T) {
m := NewArea("back", 0.822059, 0.167969, 0.1, 0.0664062)
assert.Equal(t, "082016010006", m.String())
assert.Equal(t, "3360a7064042", m.String())
})
t.Run("020100003000", func(t *testing.T) {
t.Run("0c93e801e000", func(t *testing.T) {
m := NewArea("face", 0.201, 1.000, 0.03, 0.00000001)
assert.Equal(t, "020100003000", m.String())
assert.Equal(t, "0c93e801e000", m.String())
})
t.Run("000100000000", func(t *testing.T) {
t.Run("0003e8000000", func(t *testing.T) {
m := NewArea("face", 0.0001, 1.000, 0, 0.00000001)
assert.Equal(t, "000100000000", m.String())
assert.Equal(t, "0003e8000000", m.String())
})
t.Run("000012000100", func(t *testing.T) {
t.Run("00007b0003e8", func(t *testing.T) {
m := NewArea("", -2.0001, 0.123, -0.1, 4.00000001)
assert.Equal(t, "000012000100", m.String())
assert.Equal(t, "00007b0003e8", m.String())
})
}

9
internal/crop/cache.go
View file

@ -2,6 +2,7 @@ package crop
import (
"fmt"
"path/filepath"
"path"
@ -10,8 +11,8 @@ import (
)
// FromCache returns the crop file name if cached.
func FromCache(hash, thumbPath string, width, height int, area string) (fileName string, err error) {
fileName, err = FileName(hash, thumbPath, width, height, area)
func FromCache(hash, area string, size Size, thumbPath string) (fileName string, err error) {
fileName, err = FileName(hash, area, size.Width, size.Height, thumbPath)
if err != nil {
return "", err
@ -21,11 +22,11 @@ func FromCache(hash, thumbPath string, width, height int, area string) (fileName
return fileName, nil
}
return "", ErrNotFound
return "", fmt.Errorf("%s not found", filepath.Base(fileName))
}
// FileName returns the crop file name based on cache path, size, and area.
func FileName(hash string, thumbPath string, width, height int, area string) (fileName string, err error) {
func FileName(hash, area string, width, height int, thumbPath string) (fileName string, err error) {
if len(hash) < 4 {
return "", fmt.Errorf("crop: invalid file hash %s", txt.Quote(hash))
}

8
internal/crop/cache_test.go
View file

@ -8,7 +8,7 @@ import (
func TestFileName(t *testing.T) {
t.Run("Crop160", func(t *testing.T) {
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "/example", 160, 160, "042008007010")
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "042008007010", 160, 160, "/example")
if err != nil {
t.Fatal(err)
@ -17,7 +17,7 @@ func TestFileName(t *testing.T) {
assert.Equal(t, "/example/1/4/7/147da9f0261e2d81e9a52b266f1945556588bb78_160x160_crop_042008007010.jpg", result)
})
t.Run("InvalidSize", func(t *testing.T) {
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "/example", 15000, 160, "042008007010")
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "042008007010", 15000, 160, "/example")
if err == nil {
t.Fatal("error expected")
@ -27,7 +27,7 @@ func TestFileName(t *testing.T) {
assert.Empty(t, result)
})
t.Run("InvalidHash", func(t *testing.T) {
result, err := FileName("147", "/example", 160, 160, "042008007010")
result, err := FileName("147", "042008007010", 160, 160, "/example")
if err == nil {
t.Fatal("error expected")
@ -37,7 +37,7 @@ func TestFileName(t *testing.T) {
assert.Empty(t, result)
})
t.Run("InvalidPath", func(t *testing.T) {
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "", 160, 160, "042008007010")
result, err := FileName("147da9f0261e2d81e9a52b266f1945556588bb78", "042008007010", 160, 160, "")
if err == nil {
t.Fatal("error expected")

4
internal/crop/crop.go
View file

@ -30,3 +30,7 @@ https://docs.photoprism.org/developer-guide/
*/
package crop
import "github.com/photoprism/photoprism/internal/event"
var log = event.Log

2
internal/crop/errors.go
View file

@ -5,5 +5,5 @@ import (
)
var (
ErrNotFound = errors.New("crop not found")
ErrNotFound = errors.New("not found")
)

17
internal/crop/names.go Normal file
View file

@ -0,0 +1,17 @@
package crop
import "github.com/photoprism/photoprism/pkg/fs"
// Name represents a crop size name.
type Name string
// Jpeg returns the crop name with a jpeg file extension suffix as string.
func (n Name) Jpeg() string {
return string(n) + fs.JpegExt
}
// Names of standard crop sizes.
const (
Tile160 Name = "tile_160"
Tile320 Name = "tile_320"
)

24
internal/crop/sizes.go Normal file
View file

@ -0,0 +1,24 @@
package crop
import "github.com/photoprism/photoprism/internal/thumb"
var (
DefaultOptions = []thumb.ResampleOption{thumb.ResampleFillCenter, thumb.ResampleDefault}
)
type Size struct {
Name Name `json:"name"`
Source Name `json:"-"`
Use string `json:"use"`
Width int `json:"w"`
Height int `json:"h"`
Options []thumb.ResampleOption `json:"-"`
}
type SizeMap map[Name]Size
// Sizes contains the properties of all thumbnail sizes.
var Sizes = SizeMap{
Tile160: {Tile160, Tile320, "FaceNet", 160, 160, DefaultOptions},
Tile320: {Tile320, "", "UI", 320, 320, DefaultOptions},
}

71
internal/crop/thumb.go Normal file
View file

@ -0,0 +1,71 @@
package crop
import (
"bytes"
"fmt"
"image"
"io/ioutil"
"path/filepath"
"strings"
"github.com/disintegration/imaging"
"github.com/photoprism/photoprism/internal/thumb"
"github.com/photoprism/photoprism/pkg/fs"
)
// FromThumb returns a cropped area from an existing thumbnail image.
func FromThumb(thumbName string, area Area, size Size, cache bool) (img image.Image, err error) {
// Use same folder for caching if "cache" is true.
cacheFolder := filepath.Dir(thumbName)
// Use existing thumbnail name as cached crop filename prefix.
thumbBase := filepath.Base(thumbName)
if i := strings.Index(thumbBase, "_"); i > 0 {
thumbBase = thumbBase[:i]
}
// Compose cached crop image file name.
cacheBase := fmt.Sprintf("%s_%dx%d_crop_%s", thumbBase, size.Width, size.Height, area.String())
cropFile := filepath.Join(cacheFolder, cacheBase+fs.JpegExt)
// Cached?
if !fs.FileExists(cropFile) {
// Do nothing.
} else if img, err := imaging.Open(cropFile); err != nil {
log.Errorf("crop: failed loading %s", filepath.Base(cropFile))
} else {
return img, nil
}
// Open image.
imageBuffer, err := ioutil.ReadFile(thumbName)
img, err = imaging.Decode(bytes.NewReader(imageBuffer), imaging.AutoOrientation(true))
if err != nil {
return img, err
}
// Get absolute crop coordinates and dimension.
min, max, dim := area.Bounds(img)
if dim < size.Width {
log.Debugf("crop: %s too small, crop size %dpx, actual size %dpx", filepath.Base(thumbName), size.Width, dim)
}
// Crop area from image.
img = imaging.Crop(img, image.Rect(min.X, min.Y, max.X, max.Y))
// Resample crop area.
img = thumb.Resample(img, size.Width, size.Height, size.Options...)
// Cache crop image?
if cache {
if err := imaging.Save(img, cropFile); err != nil {
log.Errorf("crop: failed caching %s", filepath.Base(cropFile))
} else {
log.Debugf("crop: saved %s", filepath.Base(cropFile))
}
}
return img, nil
}

6
internal/entity/marker.go
View file

@ -28,7 +28,7 @@ type Marker struct {
MarkerUID string `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"UID" yaml:"UID"`
FileUID string `gorm:"type:VARBINARY(42);index;" json:"FileUID" yaml:"FileUID"`
FileHash string `gorm:"type:VARBINARY(128);index" json:"FileHash" yaml:"FileHash,omitempty"`
FileArea string `gorm:"type:VARBINARY(16);default:''" json:"FileArea" yaml:"FileArea,omitempty"`
CropArea string `gorm:"type:VARBINARY(16);default:''" json:"CropArea" yaml:"CropArea,omitempty"`
MarkerType string `gorm:"type:VARBINARY(8);default:'';" json:"Type" yaml:"Type"`
MarkerSrc string `gorm:"type:VARBINARY(8);default:'';" json:"Src" yaml:"Src,omitempty"`
MarkerName string `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name,omitempty"`
@ -73,7 +73,7 @@ func NewMarker(file File, area crop.Area, subjectUID, markerSrc, markerType stri
m := &Marker{
FileUID: file.FileUID,
FileHash: file.FileHash,
FileArea: area.String(),
CropArea: area.String(),
MarkerSrc: markerSrc,
MarkerType: markerType,
SubjectUID: subjectUID,
@ -492,7 +492,7 @@ func UpdateOrCreateMarker(m *Marker) (*Marker, error) {
err := result.Updates(map[string]interface{}{
"MarkerType": m.MarkerType,
"MarkerSrc": m.MarkerSrc,
"FileArea": m.FileArea,
"CropArea": m.CropArea,
"X": m.X,
"Y": m.Y,
"W": m.W,

6
internal/entity/marker_fixtures.go
View file

@ -199,7 +199,7 @@ var MarkerFixtures = MarkerMap{
MarkerUID: "mt9k3pw1wowuy999",
FileUID: "ft2es49qhhinlple",
FaceID: FaceFixtures.Get("actress-1").ID,
FileArea: "045038063041",
CropArea: "045038063041",
FaceDist: 0.26852392873736236,
SubjectSrc: SrcManual,
SubjectUID: SubjectFixtures.Get("actress-1").SubjectUID,
@ -219,7 +219,7 @@ var MarkerFixtures = MarkerMap{
MarkerUID: "mt9k3pw1wowu1000",
FileUID: "ft2es49whhbnlqdn",
FaceID: FaceFixtures.Get("actress-1").ID,
FileArea: "046045043065",
CropArea: "046045043065",
FaceDist: 0.4507357278575355,
SubjectSrc: "",
SubjectUID: SubjectFixtures.Get("actress-1").SubjectUID,
@ -239,7 +239,7 @@ var MarkerFixtures = MarkerMap{
MarkerUID: "mt9k3pw1wowu1001",
FileUID: "ft8es39w45bnlqdw",
FaceID: FaceFixtures.Get("actress-1").ID,
FileArea: "05403304060446",
CropArea: "05403304060446",
FaceDist: 0.5099754448545762,
SubjectSrc: "",
SubjectUID: SubjectFixtures.Get("actress-1").SubjectUID,

4
internal/entity/marker_json.go
View file

@ -20,7 +20,7 @@ func (m *Marker) MarshalJSON() ([]byte, error) {
UID string
FileUID string
FileHash string
FileArea string
CropArea string
Type string
Src string
Name string
@ -40,7 +40,7 @@ func (m *Marker) MarshalJSON() ([]byte, error) {
UID: m.MarkerUID,
FileUID: m.FileUID,
FileHash: m.FileHash,
FileArea: m.FileArea,
CropArea: m.CropArea,
Type: m.MarkerType,
Src: m.MarkerSrc,
Name: name,

2
internal/entity/marker_test.go
View file

@ -25,6 +25,8 @@ func TestNewMarker(t *testing.T) {
m := NewMarker(FileFixtures.Get("exampleFileName.jpg"), testArea, "lt9k3pw1wowuy3c3", SrcImage, MarkerLabel)
assert.IsType(t, &Marker{}, m)
assert.Equal(t, "ft8es39w45bnlqdw", m.FileUID)
assert.Equal(t, "2cad9168fa6acc5c5c2965ddf6ec465ca42fd818", m.FileHash)
assert.Equal(t, "1340ce163163", m.CropArea)
assert.Equal(t, "lt9k3pw1wowuy3c3", m.SubjectUID)
assert.Equal(t, SrcImage, m.MarkerSrc)
assert.Equal(t, MarkerLabel, m.MarkerType)

5
internal/face/face.go
View file

@ -39,11 +39,11 @@ import (
"github.com/photoprism/photoprism/internal/event"
)
var CropSize = 160
var CropSize = crop.Sizes[crop.Tile160]
var ClusterCore = 4
var ClusterRadius = 0.6
var ClusterMinScore = 30
var ClusterMinSize = CropSize
var ClusterMinSize = CropSize.Width
var SampleThreshold = 2 * ClusterCore
var log = event.Log
@ -105,7 +105,6 @@ type Face struct {
Eyes Areas `json:"eyes,omitempty"`
Landmarks Areas `json:"landmarks,omitempty"`
Embeddings [][]float32 `json:"embeddings,omitempty"`
Thumb string `json:"-"`
}
// Size returns the absolute face size in pixels.

17
internal/face/face_test.go
View file

@ -7,7 +7,7 @@ import (
"strings"
"testing"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/internal/crop"
"github.com/photoprism/photoprism/pkg/fastwalk"
"github.com/stretchr/testify/assert"
@ -57,9 +57,9 @@ func TestDetect(t *testing.T) {
var embeddings [11][]float32
tfInstance := NewNet(modelPath, "testdata/cache", false)
faceNet := NewNet(modelPath, "testdata/cache", false)
if err := tfInstance.loadModel(); err != nil {
if err := faceNet.loadModel(); err != nil {
t.Fatal(err)
}
@ -69,10 +69,10 @@ func TestDetect(t *testing.T) {
}
t.Run(fileName, func(t *testing.T) {
fileHash := fs.Hash(fileName)
baseName := filepath.Base(fileName)
faces, err := Detect(fileName, true, 20)
if err != nil {
t.Fatal(err)
}
@ -80,24 +80,25 @@ func TestDetect(t *testing.T) {
t.Logf("found %d faces in '%s'", len(faces), baseName)
if len(faces) > 0 {
t.Logf("results: %#v", faces)
// t.Logf("results: %#v", faces)
for i, f := range faces {
t.Logf("marker[%d]: %#v %#v", i, f.CropArea(), f.Area)
t.Logf("landmarks[%d]: %s", i, f.RelativeLandmarksJSON())
img, err := tfInstance.getFaceCrop(fileName, fileHash, &faces[i])
img, err := crop.FromThumb(fileName, f.CropArea(), CropSize, false)
if err != nil {
t.Fatal(err)
}
embedding := tfInstance.getEmbeddings(img)
embedding := faceNet.getEmbeddings(img)
if b, err := json.Marshal(embedding[0]); err != nil {
t.Fatal(err)
} else {
t.Logf("embedding: %#v", string(b))
assert.NotEmpty(t, b)
// t.Logf("embedding: %#v", string(b))
}
t.Logf("faces: %d %v", i, faceindices[baseName])

92
internal/face/net.go
View file

@ -1,21 +1,17 @@
package face
import (
"bytes"
"fmt"
"image"
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime/debug"
"strings"
"sync"
"github.com/photoprism/photoprism/pkg/fs"
"github.com/photoprism/photoprism/internal/crop"
"github.com/disintegration/imaging"
"github.com/photoprism/photoprism/pkg/txt"
tf "github.com/tensorflow/tensorflow/tensorflow/go"
)
@ -36,7 +32,7 @@ func NewNet(modelPath, cachePath string, disabled bool) *Net {
}
// Detect runs the detection and facenet algorithms over the provided source image.
func (t *Net) Detect(fileName string, minSize int) (faces Faces, err error) {
func (t *Net) Detect(fileName string, minSize int, cacheCrop bool) (faces Faces, err error) {
faces, err = Detect(fileName, false, minSize)
if err != nil {
@ -53,18 +49,12 @@ func (t *Net) Detect(fileName string, minSize int) (faces Faces, err error) {
return faces, err
}
var cacheHash string
if t.cachePath != "" {
cacheHash = fs.Hash(fileName)
}
for i, f := range faces {
if f.Area.Col == 0 && f.Area.Row == 0 {
continue
}
if img, err := t.getFaceCrop(fileName, cacheHash, &faces[i]); err != nil {
if img, err := crop.FromThumb(fileName, f.CropArea(), CropSize, cacheCrop); err != nil {
log.Errorf("faces: failed to decode image: %v", err)
} else if embeddings := t.getEmbeddings(img); len(embeddings) > 0 {
faces[i].Embeddings = embeddings
@ -103,80 +93,8 @@ func (t *Net) loadModel() error {
return nil
}
func (t *Net) getCacheFolder(fileName, cacheHash string) string {
if t.cachePath == "" || cacheHash == "" {
return filepath.Dir(fileName)
}
if cacheHash == "" {
log.Debugf("faces: no hash provided for caching %s crops", filepath.Base(fileName))
cacheHash = fs.Hash(fileName)
}
result := filepath.Join(t.cachePath, "faces", string(cacheHash[0]), string(cacheHash[1]), string(cacheHash[2]))
if err := os.MkdirAll(result, os.ModePerm); err != nil {
log.Errorf("faces: failed creating cache folder")
}
return result
}
func (t *Net) getFaceCrop(fileName, cacheHash string, f *Face) (img image.Image, err error) {
if f == nil {
return img, fmt.Errorf("face is nil")
}
area := f.Area
cacheFolder := t.getCacheFolder(fileName, cacheHash)
if cacheHash != "" {
f.Thumb = fmt.Sprintf("%s_%dx%d_crop_%s", cacheHash, CropSize, CropSize, f.CropArea().String())
} else {
base := filepath.Base(fileName)
i := strings.Index(base, "_")
if i > 32 {
base = base[:i]
}
f.Thumb = fmt.Sprintf("%s_%dx%d_crop_%s", base, CropSize, CropSize, f.CropArea().String())
}
cacheFile := filepath.Join(cacheFolder, f.Thumb+fs.JpegExt)
if !fs.FileExists(cacheFile) {
// Do nothing.
} else if img, err := imaging.Open(cacheFile); err != nil {
log.Errorf("faces: failed loading %s", filepath.Base(cacheFile))
} else {
log.Debugf("faces: extracting from %s", filepath.Base(cacheFile))
return img, nil
}
x, y := area.TopLeft()
imageBuffer, err := ioutil.ReadFile(fileName)
img, err = imaging.Decode(bytes.NewReader(imageBuffer), imaging.AutoOrientation(true))
if err != nil {
return img, err
}
img = imaging.Crop(img, image.Rect(y, x, y+area.Scale, x+area.Scale))
img = imaging.Fill(img, CropSize, CropSize, imaging.Center, imaging.Lanczos)
if err := imaging.Save(img, cacheFile); err != nil {
log.Errorf("faces: failed caching %s", filepath.Base(cacheFile))
} else {
log.Debugf("faces: saved %s", filepath.Base(cacheFile))
}
return img, nil
}
func (t *Net) getEmbeddings(img image.Image) [][]float32 {
tensor, err := imageToTensor(img, CropSize, CropSize)
tensor, err := imageToTensor(img, CropSize.Width, CropSize.Height)
if err != nil {
log.Errorf("faces: failed to convert image to tensor: %v", err)

4
internal/face/net_test.go
View file

@ -62,7 +62,7 @@ func TestNet(t *testing.T) {
t.Run(fileName, func(t *testing.T) {
baseName := filepath.Base(fileName)
faces, err := faceNet.Detect(fileName, 20)
faces, err := faceNet.Detect(fileName, 20, false)
if err != nil {
t.Fatal(err)
@ -71,7 +71,7 @@ func TestNet(t *testing.T) {
t.Logf("found %d faces in '%s'", len(faces), baseName)
if len(faces) > 0 {
t.Logf("results: %#v", faces)
// t.Logf("results: %#v", faces)
for i, f := range faces {
if len(f.Embeddings) > 0 {

6
internal/photoprism/index_faces.go
View file

@ -20,10 +20,10 @@ func (ind *Index) detectFaces(jpeg *MediaFile) face.Faces {
// Select best thumbnail depending on configured size.
if Config().ThumbSizePrecached() < 1280 {
minSize = 30
minSize = 20
thumbSize = thumb.Fit720
} else {
minSize = 40
minSize = 30
thumbSize = thumb.Fit1280
}
@ -41,7 +41,7 @@ func (ind *Index) detectFaces(jpeg *MediaFile) face.Faces {
start := time.Now()
faces, err := ind.faceNet.Detect(thumbName, minSize)
faces, err := ind.faceNet.Detect(thumbName, minSize, true)
if err != nil {
log.Debugf("%s in %s", err, txt.Quote(jpeg.BaseName()))

1
internal/thumb/names.go
View file

@ -14,7 +14,6 @@ func (n Name) Jpeg() string {
const (
Tile50 Name = "tile_50"
Tile100 Name = "tile_100"
Crop160 Name = "crop_160"
Tile224 Name = "tile_224"
Tile500 Name = "tile_500"
Colors Name = "colors"

2
internal/thumb/resample_options.go
View file

@ -12,7 +12,6 @@ const (
ResampleFillTopLeft
ResampleFillBottomRight
ResampleFit
ResampleCrop
ResampleResize
ResampleNearestNeighbor
ResampleDefault
@ -24,7 +23,6 @@ var ResampleMethods = map[ResampleOption]string{
ResampleFillTopLeft: "left",
ResampleFillBottomRight: "right",
ResampleFit: "fit",
ResampleCrop: "crop",
ResampleResize: "resize",
}

34
internal/thumb/sizes.go
View file

@ -21,8 +21,9 @@ func InvalidSize(size int) bool {
}
type Size struct {
Use string `json:"use"`
Name Name `json:"name"`
Source Name `json:"-"`
Use string `json:"use"`
Width int `json:"w"`
Height int `json:"h"`
Public bool `json:"-"`
@ -33,22 +34,21 @@ type SizeMap map[Name]Size
// Sizes contains the properties of all thumbnail sizes.
var Sizes = SizeMap{
Tile50: {"Lists", Tile500, 50, 50, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Tile100: {"Maps", Tile500, 100, 100, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Crop160: {"FaceNet", "", 160, 160, false, []ResampleOption{ResampleCrop, ResampleDefault}},
Tile224: {"TensorFlow, Mosaic", Tile500, 224, 224, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Tile500: {"Tiles", "", 500, 500, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Colors: {"Color Detection", Fit720, 3, 3, false, []ResampleOption{ResampleResize, ResampleNearestNeighbor, ResamplePng}},
Left224: {"TensorFlow", Fit720, 224, 224, false, []ResampleOption{ResampleFillTopLeft, ResampleDefault}},
Right224: {"TensorFlow", Fit720, 224, 224, false, []ResampleOption{ResampleFillBottomRight, ResampleDefault}},
Fit720: {"Mobile, TV", "", 720, 720, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit1280: {"Mobile, HD Ready TV", Fit2048, 1280, 1024, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit1920: {"Mobile, Full HD TV", Fit2048, 1920, 1200, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit2048: {"Tablets, Cinema 2K", "", 2048, 2048, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit2560: {"Quad HD, Retina Display", "", 2560, 1600, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit3840: {"Ultra HD", "", 3840, 2400, false, []ResampleOption{ResampleFit, ResampleDefault}}, // Deprecated in favor of fit_4096
Fit4096: {"Ultra HD, Retina 4K", "", 4096, 4096, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit7680: {"8K Ultra HD 2, Retina 6K", "", 7680, 4320, true, []ResampleOption{ResampleFit, ResampleDefault}},
Tile50: {Tile50, Tile500, "Lists", 50, 50, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Tile100: {Tile100, Tile500, "Maps", 100, 100, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Tile224: {Tile224, Tile500, "TensorFlow, Mosaic", 224, 224, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Tile500: {Tile500, "", "Tiles", 500, 500, false, []ResampleOption{ResampleFillCenter, ResampleDefault}},
Colors: {Colors, Fit720, "Color Detection", 3, 3, false, []ResampleOption{ResampleResize, ResampleNearestNeighbor, ResamplePng}},
Left224: {Left224, Fit720, "TensorFlow", 224, 224, false, []ResampleOption{ResampleFillTopLeft, ResampleDefault}},
Right224: {Right224, Fit720, "TensorFlow", 224, 224, false, []ResampleOption{ResampleFillBottomRight, ResampleDefault}},
Fit720: {Fit720, "", "Mobile, TV", 720, 720, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit1280: {Fit1280, Fit2048, "Mobile, HD Ready TV", 1280, 1024, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit1920: {Fit1920, Fit2048, "Mobile, Full HD TV", 1920, 1200, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit2048: {Fit2048, "", "Tablets, Cinema 2K", 2048, 2048, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit2560: {Fit2560, "", "Quad HD, Retina Display", 2560, 1600, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit3840: {Fit3840, "", "Ultra HD", 3840, 2400, false, []ResampleOption{ResampleFit, ResampleDefault}}, // Deprecated in favor of fit_4096
Fit4096: {Fit4096, "", "Ultra HD, Retina 4K", 4096, 4096, true, []ResampleOption{ResampleFit, ResampleDefault}},
Fit7680: {Fit7680, "", "8K Ultra HD 2, Retina 6K", 7680, 4320, true, []ResampleOption{ResampleFit, ResampleDefault}},
}
// DefaultSizes contains all default size names.