Backend: Refactor SQL batch size limit #3124 #3742

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/query/faces.go

@@ -152,7 +152,7 @@ func CountNewFaceMarkers(size, score int) (n int) {
 // PurgeOrphanFaces removes unused faces from the index.
 func PurgeOrphanFaces(faceIds []string, ignored bool) (removed int64, err error) {
 	// Remove invalid face IDs in batches to be compatible with SQLite.
-	batchSize := 500
+	batchSize := BatchSize()
 
 	for i := 0; i < len(faceIds); i += batchSize {
 		j := i + batchSize

internal/query/photo.go

@@ -196,7 +196,7 @@ func FlagHiddenPhotos() (err error) {
 		return nil
 	} else {
 		// Update photos in batches to be compatible with SQLite.
-		batchSize := 500
+		batchSize := BatchSize()
 
 		for i := 0; i < len(hidden); i += batchSize {
 			j := i + batchSize
@@ -209,7 +209,9 @@ func FlagHiddenPhotos() (err error) {
 			ids := hidden[i:j]
 
 			// Set photos.photo_quality = -1.
-			if err = Db().Table(entity.Photo{}.TableName()).Where("id IN (?)", ids).UpdateColumn("photo_quality", -1).Error; err != nil {
+			if err = UnscopedDb().Table(entity.Photo{}.TableName()).
+				Where("id IN (?) AND photo_quality > -1", ids).
+				UpdateColumn("photo_quality", -1).Error; err != nil {
 				// Failed.
 				log.Warnf("index: failed to flag %d pictures as hidden", len(ids))
 				return err

internal/query/query.go

@@ -69,11 +69,21 @@ func UnscopedDb() *gorm.DB {
 	return entity.Db().Unscoped()
 }
 
-// DbDialect returns the sql dialect name.
+// DbDialect returns the sql database dialect name.
 func DbDialect() string {
 	return Db().Dialect().GetName()
 }
 
+// BatchSize returns the maximum query parameter number based on the current sql database dialect.
+func BatchSize() int {
+	switch DbDialect() {
+	case SQLite3:
+		return 333
+	default:
+		return 1000
+	}
+}
+
 // logErr logs an error and keeps quiet otherwise.
 func logErr(prefix, action string, err error) {
 	if err != nil {

internal/query/query_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/photoprism/photoprism/internal/entity"
 )
@@ -23,3 +24,15 @@ func TestMain(m *testing.M) {
 
 	os.Exit(code)
 }
+
+func TestDbDialect(t *testing.T) {
+	t.Run("SQLite", func(t *testing.T) {
+		assert.Equal(t, "sqlite3", DbDialect())
+	})
+}
+
+func TestBatchSize(t *testing.T) {
+	t.Run("SQLite", func(t *testing.T) {
+		assert.Equal(t, 333, BatchSize())
+	})
+}