luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Metadata: Ignore unknown values when parsing timestamps #2510

Browse source 
Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer 2022-07-22 12:38:25 +02:00
parent d2086d5622
commit c7ad17b60c
10 changed files with 244 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/meta/json_exiftool.go
View file

@ -66,6 +66,8 @@ func (data *Data) Exiftool(jsonData []byte, originalName string) (err error) {
for _, tagValue = range tagValues {
if r, ok := jsonValues[tagValue]; !ok {
continue
} else if txt.Empty(r.String()) {
continue
} else {
jsonValue = r
break

25
internal/meta/json_exiftool_test.go → internal/meta/json_test.go
View file

@ -757,6 +757,31 @@ func TestJSON(t *testing.T) {
assert.Equal(t, "", data.LensModel)
})
t.Run("datetime-zero.json", func(t *testing.T) {
data, err := JSON("testdata/datetime-zero.json", "")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, CodecJpeg, data.Codec)
assert.Equal(t, "0s", data.Duration.String())
assert.Equal(t, "2015-03-20 12:07:53 +0000 UTC", data.TakenAtLocal.String())
assert.Equal(t, "2015-03-20 12:07:53 +0000 UTC", data.TakenAt.String())
assert.Equal(t, 0, data.TakenNs)
assert.Equal(t, "", data.TimeZone)
assert.Equal(t, 4608, data.Width)
assert.Equal(t, 3072, data.Height)
assert.Equal(t, 4608, data.ActualWidth())
assert.Equal(t, 3072, data.ActualHeight())
assert.Equal(t, 1, data.Orientation)
assert.Equal(t, float32(0), data.Lat)
assert.Equal(t, float32(0), data.Lng)
assert.Equal(t, "OLYMPUS IMAGING CORP.", data.CameraMake)
assert.Equal(t, "TG-830", data.CameraModel)
assert.Equal(t, "", data.LensModel)
})
t.Run("subject-1.json", func(t *testing.T) {
data, err := JSON("testdata/subject-1.json", "")

131
internal/meta/testdata/datetime-zero.json vendored Normal file
View file

@ -0,0 +1,131 @@
[{
"SourceFile": "snow.jpeg",
"ExifToolVersion": 12.00,
"FileName": "snow.jpeg",
"Directory": ".",
"FileSize": "3.4 MB",
"FileModifyDate": "2020:12:23 18:18:39+01:00",
"FileAccessDate": "2020:12:23 18:21:05+01:00",
"FileInodeChangeDate": "2020:12:23 18:19:03+01:00",
"FilePermissions": "rw-r--r--",
"FileType": "JPEG",
"FileTypeExtension": "jpg",
"MIMEType": "image/jpeg",
"ExifByteOrder": "Little-endian (Intel, II)",
"ImageDescription": "OLYMPUS DIGITAL CAMERA ",
"Make": "OLYMPUS IMAGING CORP.",
"Model": "TG-830",
"Orientation": "Horizontal (normal)",
"XResolution": 72,
"YResolution": 72,
"ResolutionUnit": "inches",
"Software": "Version 1.0",
"ModifyDate": "2015:03:20 12:07:53",
"Artist": "",
"YCbCrPositioning": "Co-sited",
"Copyright": "",
"ExposureTime": "1/1600",
"FNumber": 3.9,
"ExposureProgram": "Creative (Slow speed)",
"ISO": 125,
"SensitivityType": "Standard Output Sensitivity",
"ExifVersion": "0230",
"DateTimeOriginal": "0000:00:00 00:00:00",
"CreateDate": "2015:03:20 12:07:53",
"ComponentsConfiguration": "Y, Cb, Cr, -",
"ExposureCompensation": 0,
"MaxApertureValue": 3.9,
"LightSource": "Unknown",
"Flash": "Auto, Did not fire",
"FocalLength": "5.0 mm",
"SpecialMode": "Normal, Sequence: 0, Panorama: (none)",
"CameraID": "OLYMPUS DIGITAL CAMERA",
"EquipmentVersion": "0100",
"CameraType2": "TG-830",
"FocalPlaneDiagonal": "7.71 mm",
"CameraSettingsVersion": "0100",
"PreviewImageValid": "Yes",
"PreviewImageStart": 3141874,
"PreviewImageLength": 373253,
"AELock": "Off",
"MeteringMode": "ESP",
"MacroMode": "Off",
"FocusMode": "Single AF; S-AF, Imager AF",
"FlashMode": "Off",
"WhiteBalance2": "Auto",
"WhiteBalanceBracket": "0 0",
"NoiseReduction": "(none)",
"MagicFilter": "Off; 0; 0; 0",
"DriveMode": "Single Shot",
"PanoramaMode": "Off",
"ExtendedWBDetect": "Off",
"ImageProcessingVersion": "0112",
"WB_RBLevels": "459 470 256 256",
"DistortionCorrection2": "On",
"AspectRatio": "3:2",
"AspectFrame": "0 0 4607 3071",
"FacesDetected": "0 0 0",
"FaceDetectArea": "(Binary data 191 bytes, use -b option to extract)",
"MaxFaces": "8 8 8",
"FaceDetectFrameSize": "0 0 0 0 0 0",
"BodyFirmwareVersion": 0,
"Quality": "SQ (Low)",
"Macro": "Off",
"Resolution": 1,
"CameraType": "TG-830",
"SceneMode": "Sport",
"SerialNumber": "123JLW205383",
"Warning": "Bad PrintIM data",
"DataDump": "(Binary data 4916 bytes, use -b option to extract)",
"UserComment": "",
"FlashpixVersion": "0100",
"ColorSpace": "sRGB",
"ExifImageWidth": 4608,
"ExifImageHeight": 3072,
"InteropIndex": "R98 - DCF basic file (sRGB)",
"InteropVersion": "0100",
"FileSource": "Digital Camera",
"CustomRendered": "Normal",
"ExposureMode": "Auto",
"WhiteBalance": "Auto",
"DigitalZoomRatio": 0,
"FocalLengthIn35mmFormat": "28 mm",
"SceneCaptureType": "Standard",
"GainControl": "None",
"Contrast": "Normal",
"Saturation": "Normal",
"Sharpness": "Normal",
"GPSVersionID": "2.3.0.0",
"GPSLatitudeRef": "Unknown ()",
"GPSLongitudeRef": "Unknown ()",
"GPSStatus": "Measurement Void",
"GPSImgDirectionRef": "Magnetic North",
"GPSImgDirection": 117.44,
"GPSAreaInformation": "",
"PrintIMVersion": "0300",
"Compression": "JPEG (old-style)",
"ThumbnailOffset": 16384,
"ThumbnailLength": 7125,
"ImageWidth": 4608,
"ImageHeight": 3072,
"EncodingProcess": "Baseline DCT, Huffman coding",
"BitsPerSample": 8,
"ColorComponents": 3,
"YCbCrSubSampling": "YCbCr4:2:2 (2 1)",
"Aperture": 3.9,
"BlueBalance": 1.835938,
"ImageSize": "4608x3072",
"Megapixels": 14.2,
"PreviewImage": "(Binary data 373253 bytes, use -b option to extract)",
"RedBalance": 1.792969,
"ScaleFactor35efl": 5.6,
"ShutterSpeed": "1/1600",
"ThumbnailImage": "(Binary data 7125 bytes, use -b option to extract)",
"GPSLatitude": "",
"GPSLongitude": "",
"CircleOfConfusion": "0.005 mm",
"FOV": "65.5 deg",
"FocalLength35efl": "5.0 mm (35 mm equivalent: 28.0 mm)",
"HyperfocalDistance": "1.19 m",
"LightValue": 14.2
}]

4
internal/search/conditions.go
View file

@ -139,7 +139,7 @@ func LikeAllNames(cols Cols, s string) (wheres []string) {
for _, w := range strings.Split(k, txt.Or) {
w = strings.TrimSpace(w)
if w == txt.Empty {
if w == txt.EmptyString {
continue
}
@ -236,7 +236,7 @@ func AnyInt(col, numbers, sep string, min, max int) (where string) {
// OrLike returns a where condition and values for finding multiple terms combined with OR.
func OrLike(col, s string) (where string, values []interface{}) {
if txt.IsEmpty(col) || txt.IsEmpty(s) {
if txt.Empty(col) || txt.Empty(s) {
return "", []interface{}{}
}

6
pkg/txt/datetime.go
View file

@ -51,7 +51,7 @@ const (
SecMax = 59
)
// DateTime parses a timestamp string and returns a valid time.Time if possible.
// DateTime parses a time string and returns a valid time.Time if possible.
func DateTime(s, timeZone string) (t time.Time) {
defer func() {
if r := recover(); r != nil {
@ -60,8 +60,8 @@ func DateTime(s, timeZone string) (t time.Time) {
}
}()
// Empty timestamp? Return unknown time.
if s == "" {
// Empty time string?
if EmptyTime(s) {
return time.Time{}
}

15
pkg/txt/datetime_test.go
View file

@ -7,6 +7,21 @@ import (
)
func TestDateTime(t *testing.T) {
t.Run("EmptyString", func(t *testing.T) {
result := DateTime("", "")
assert.True(t, result.IsZero())
assert.Equal(t, "0001-01-01 00:00:00 +0000 UTC", result.String())
})
t.Run("0000-00-00 00:00:00", func(t *testing.T) {
result := DateTime("0000-00-00 00:00:00", "")
assert.True(t, result.IsZero())
assert.Equal(t, "0001-01-01 00:00:00 +0000 UTC", result.String())
})
t.Run("0001-01-01 00:00:00 +0000 UTC", func(t *testing.T) {
result := DateTime("0001-01-01 00:00:00 +0000 UTC", "")
assert.True(t, result.IsZero())
assert.Equal(t, "0001-01-01 00:00:00 +0000 UTC", result.String())
})
t.Run("2016: : : : ", func(t *testing.T) {
result := DateTime("2016: : : : ", "")
assert.Equal(t, "2016-01-01 00:00:00 +0000 UTC", result.String())

22
pkg/txt/empty.go
View file

@ -4,11 +4,11 @@ import (
"strings"
)
// IsEmpty tests if a string represents an empty/invalid value.
func IsEmpty(s string) bool {
// Empty tests if a string represents an empty/invalid value.
func Empty(s string) bool {
s = strings.Trim(strings.TrimSpace(s), "%*")
if s == "" || s == "0" || s == "-1" {
if s == "" || s == "0" || s == "-1" || EmptyTime(s) {
return true
}
@ -19,5 +19,19 @@ func IsEmpty(s string) bool {
// NotEmpty tests if a string does not represent an empty/invalid value.
func NotEmpty(s string) bool {
return !IsEmpty(s)
return !Empty(s)
}
// EmptyTime tests if the string is empty or matches an unknown time pattern.
func EmptyTime(s string) bool {
switch s {
case "":
return true
case "0000:00:00 00:00:00", "0000-00-00 00-00-00", "0000-00-00 00:00:00":
return true
case "0001-01-01 00:00:00", "0001-01-01 00:00:00 +0000 UTC":
return true
default:
return false
}
}

52
pkg/txt/empty_test.go
View file

@ -8,46 +8,49 @@ import (
func TestIsEmpty(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
assert.Equal(t, true, IsEmpty(""))
assert.Equal(t, true, Empty(""))
})
t.Run("EnNew", func(t *testing.T) {
assert.Equal(t, false, IsEmpty(EnNew))
assert.Equal(t, false, Empty(EnNew))
})
t.Run("Spaces", func(t *testing.T) {
assert.Equal(t, false, IsEmpty(" new "))
assert.Equal(t, false, Empty(" new "))
})
t.Run("Uppercase", func(t *testing.T) {
assert.Equal(t, false, IsEmpty("NEW"))
assert.Equal(t, false, Empty("NEW"))
})
t.Run("Lowercase", func(t *testing.T) {
assert.Equal(t, false, IsEmpty("new"))
assert.Equal(t, false, Empty("new"))
})
t.Run("True", func(t *testing.T) {
assert.Equal(t, false, IsEmpty("New"))
assert.Equal(t, false, Empty("New"))
})
t.Run("False", func(t *testing.T) {
assert.Equal(t, false, IsEmpty("non"))
assert.Equal(t, false, Empty("non"))
})
t.Run("0", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("0"))
assert.Equal(t, true, Empty("0"))
})
t.Run("-1", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("-1"))
assert.Equal(t, true, Empty("-1"))
})
t.Run("Date", func(t *testing.T) {
assert.Equal(t, true, Empty("0000:00:00 00:00:00"))
})
t.Run("nil", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("nil"))
assert.Equal(t, true, Empty("nil"))
})
t.Run("NaN", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("NaN"))
assert.Equal(t, true, Empty("NaN"))
})
t.Run("NULL", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("NULL"))
assert.Equal(t, true, Empty("NULL"))
})
t.Run("*", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("*"))
assert.Equal(t, true, Empty("*"))
})
t.Run("%", func(t *testing.T) {
assert.Equal(t, true, IsEmpty("%"))
assert.Equal(t, true, Empty("%"))
})
}
@ -79,6 +82,9 @@ func TestNotEmpty(t *testing.T) {
t.Run("-1", func(t *testing.T) {
assert.Equal(t, false, NotEmpty("-1"))
})
t.Run("Date", func(t *testing.T) {
assert.Equal(t, false, NotEmpty("0000:00:00 00:00:00"))
})
t.Run("nil", func(t *testing.T) {
assert.Equal(t, false, NotEmpty("nil"))
})
@ -95,3 +101,21 @@ func TestNotEmpty(t *testing.T) {
assert.Equal(t, false, NotEmpty("%"))
})
}
func TestEmptyTime(t *testing.T) {
t.Run("EmptyString", func(t *testing.T) {
assert.True(t, EmptyTime(""))
})
t.Run("0000-00-00 00-00-00", func(t *testing.T) {
assert.True(t, EmptyTime("0000-00-00 00-00-00"))
})
t.Run("0000:00:00 00:00:00", func(t *testing.T) {
assert.True(t, EmptyTime("0000:00:00 00:00:00"))
})
t.Run("0000-00-00 00:00:00", func(t *testing.T) {
assert.True(t, EmptyTime("0000-00-00 00:00:00"))
})
t.Run("0001-01-01 00:00:00 +0000 UTC", func(t *testing.T) {
assert.True(t, EmptyTime("0001-01-01 00:00:00 +0000 UTC"))
})
}

10
pkg/txt/int.go
View file

@ -21,9 +21,9 @@ func Int(s string) int {
}
// IntVal converts a string to a validated integer or a default if invalid.
func IntVal(s string, min, max, d int) (i int) {
func IntVal(s string, min, max, def int) (i int) {
if s == "" {
return d
return def
} else if s[0] == ' ' {
s = strings.TrimSpace(s)
}
@ -31,15 +31,15 @@ func IntVal(s string, min, max, d int) (i int) {
result, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return d
return def
}
i = int(result)
if i < min {
return d
return def
} else if max != 0 && i > max {
return d
return def
}
return i

12
pkg/txt/query.go
View file

@ -5,12 +5,10 @@ import (
)
const (
Empty = ""
Space = " "
Or = "|"
And = "&"
Plus = "+"
SpacedPlus = Space + Plus + Space
EmptyString = ""
Space = " "
Or = "|"
And = "&"
)
// Spaced returns the string padded with a space left and right.
@ -28,5 +26,5 @@ func StripOr(s string) string {
func QueryTooShort(q string) bool {
q = strings.Trim(q, "- '")
return q != Empty && len(q) < 3 && IsLatin(q)
return q != EmptyString && len(q) < 3 && IsLatin(q)
}