diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 53539bd97..982918d60 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,8 +1,7 @@ version: '3.5' -## Continuous Integration (CI) Test Environment services: - ## App Dev Container + ## Continuous Integration (CI) Environment ## Docs: https://docs.photoprism.app/developer-guide/ photoprism: build: . @@ -43,7 +42,7 @@ services: PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals" PHOTOPRISM_IMPORT_PATH: "/go/src/github.com/photoprism/photoprism/storage/import" PHOTOPRISM_DISABLE_CHOWN: "true" # disables storage permission updates on startup - PHOTOPRISM_DISABLE_BACKUPS: "false" # disables creating YAML metadata files + PHOTOPRISM_DISABLE_BACKUPS: "false" # disables backing up albums and photo metadata to YAML files PHOTOPRISM_DISABLE_WEBDAV: "false" # disables built-in WebDAV server PHOTOPRISM_DISABLE_SETTINGS: "false" # disables settings UI and API PHOTOPRISM_DISABLE_PLACES: "false" # disables reverse geocoding and maps @@ -51,7 +50,7 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "false" # allow uploads that may be offensive - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_THUMB_FILTER: "lanczos" # resample filter, best to worst: blackman, lanczos, cubic, linear PHOTOPRISM_THUMB_UNCACHED: "true" # enables on-demand thumbnail rendering (high memory and cpu usage) PHOTOPRISM_THUMB_SIZE: 2048 # pre-rendered thumbnail size limit (default 2048, min 720, max 7680) @@ -158,6 +157,10 @@ services: MARIADB_PASSWORD: "photoprism" MARIADB_ROOT_PASSWORD: "photoprism" + ## Dummy OpenID Connect Provider + dummy-oidc: + image: photoprism/dummy-oidc:220405 + ## Dummy WebDAV Server dummy-webdav: image: photoprism/dummy-webdav:20211109 diff --git a/docker-compose.latest.yml b/docker-compose.latest.yml index 4cb414d73..f6cf5736f 100644 --- a/docker-compose.latest.yml +++ b/docker-compose.latest.yml @@ -1,8 +1,7 @@ version: '3.5' -## Latest Stable Release for QA services: - ## App Server + ## Stable Release ## Docs: https://docs.photoprism.org/ photoprism-latest: image: photoprism/photoprism:latest @@ -43,7 +42,7 @@ services: PHOTOPRISM_DATABASE_USER: "photoprism_latest" PHOTOPRISM_DATABASE_PASSWORD: "photoprism_latest" PHOTOPRISM_DISABLE_CHOWN: "false" # disables storage permission updates on startup - PHOTOPRISM_DISABLE_BACKUPS: "false" # disables creating YAML metadata files + PHOTOPRISM_DISABLE_BACKUPS: "false" # disables backing up albums and photo metadata to YAML files PHOTOPRISM_DISABLE_WEBDAV: "false" # disables built-in WebDAV server PHOTOPRISM_DISABLE_SETTINGS: "false" # disables settings UI and API PHOTOPRISM_DISABLE_PLACES: "false" # disables reverse geocoding and maps @@ -51,7 +50,7 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "false" # allows uploads that may be offensive - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_THUMB_FILTER: "lanczos" # resample filter, best to worst: blackman, lanczos, cubic, linear PHOTOPRISM_THUMB_UNCACHED: "true" # enables on-demand thumbnail rendering (high memory and cpu usage) PHOTOPRISM_THUMB_SIZE: 2048 # pre-rendered thumbnail size limit (default 2048, min 720, max 7680) diff --git a/docker-compose.mariadb.yml b/docker-compose.mariadb.yml index 09be9455c..9dda48ffa 100644 --- a/docker-compose.mariadb.yml +++ b/docker-compose.mariadb.yml @@ -1,6 +1,7 @@ version: '3.5' -## MariaDB Server Versions for Development & Testing +# Additional MariaDB and MySQL versions for testing compatibility + services: ## MariaDB 10.8 Database Server ## Docs: https://mariadb.com/kb/en/release-notes-mariadb-108-series/ diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 69d17c917..f1fa36d69 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -4,12 +4,8 @@ version: '3.5' # The current Gorm version does NOT support compatible general data types: # https://github.com/photoprism/photoprism/issues/47 -## Development Environment with -## - App Dev Container -## - PostgreSQL Database Server -## - and Dummy Services services: - ## App Dev Container + ## PhotoPrism Development Environment (PostgresSQL) ## Docs: https://docs.photoprism.app/developer-guide/ photoprism: build: . @@ -54,7 +50,7 @@ services: PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals" PHOTOPRISM_IMPORT_PATH: "/go/src/github.com/photoprism/photoprism/storage/import" PHOTOPRISM_DISABLE_CHOWN: "false" # disables storage permission updates on startup - PHOTOPRISM_DISABLE_BACKUPS: "false" # disables creating YAML metadata files + PHOTOPRISM_DISABLE_BACKUPS: "false" # disables backing up albums and photo metadata to YAML files PHOTOPRISM_DISABLE_WEBDAV: "false" # disables built-in WebDAV server PHOTOPRISM_DISABLE_SETTINGS: "false" # disables settings UI and API PHOTOPRISM_DISABLE_PLACES: "false" # disables reverse geocoding and maps @@ -62,7 +58,7 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "false" # allows uploads that may be offensive - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_THUMB_FILTER: "lanczos" # resample filter, best to worst: blackman, lanczos, cubic, linear PHOTOPRISM_THUMB_UNCACHED: "true" # enables on-demand thumbnail rendering (high memory and cpu usage) PHOTOPRISM_THUMB_SIZE: 2048 # pre-rendered thumbnail size limit (default 2048, min 720, max 7680) diff --git a/docker-compose.yml b/docker-compose.yml index 39aab4a5e..fc809d01d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,7 @@ version: '3.5' -## Development Environment with -## - HTTPS Reverse Proxy -## - App Dev Container -## - MariaDB Database Server -## - Keycloak OpenID Connect Provider -## - and Dummy Services services: - ## Traefik HTTPS Reverse Proxy - ## Includes Let's Encrypt certs for local dev domain "localssl.dev" (all records point to 127.0.0.1) - ## Docs: https://doc.traefik.io/traefik/ - traefik: - image: photoprism/traefik:220405 - ports: - # - "80:80" # HTTP (redirects to HTTPS) - - "443:443" # HTTPS (required) - labels: - - "traefik.enable=true" - - "traefik.http.routers.traefik.rule=Host(`traefik.localssl.dev`)" - - "traefik.http.routers.traefik.tls.domains[0].main=localssl.dev" - - "traefik.http.routers.traefik.tls.domains[0].sans=*.localssl.dev" - - "traefik.http.routers.traefik.tls=true" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.service=api@internal" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" # enables Traefik to watch services - - ## App Build Environment + ## PhotoPrism Development Environment (MariaDB) ## Docs: https://docs.photoprism.org/developer-guide/ photoprism: build: . @@ -85,15 +60,16 @@ services: PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals" PHOTOPRISM_IMPORT_PATH: "/go/src/github.com/photoprism/photoprism/storage/import" PHOTOPRISM_DISABLE_CHOWN: "false" # disables storage permission updates on startup - PHOTOPRISM_DISABLE_BACKUPS: "false" # disables creating YAML metadata files + PHOTOPRISM_DISABLE_BACKUPS: "false" # disables backing up albums and photo metadata to YAML files PHOTOPRISM_DISABLE_WEBDAV: "false" # disables built-in WebDAV server PHOTOPRISM_DISABLE_SETTINGS: "false" # disables settings UI and API PHOTOPRISM_DISABLE_PLACES: "false" # disables reverse geocoding and maps PHOTOPRISM_DISABLE_EXIFTOOL: "false" # disables creating JSON metadata sidecar files with ExifTool PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "false" # allows uploads that may be offensive - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_THUMB_FILTER: "lanczos" # resample filter, best to worst: blackman, lanczos, cubic, linear PHOTOPRISM_THUMB_UNCACHED: "true" # enables on-demand thumbnail rendering (high memory and cpu usage) PHOTOPRISM_THUMB_SIZE: 2048 # pre-rendered thumbnail size limit (default 2048, min 720, max 7680) @@ -146,9 +122,28 @@ services: MARIADB_PASSWORD: "photoprism" MARIADB_ROOT_PASSWORD: "photoprism" + ## HTTPS Reverse Proxy + ## includes "*.localssl.dev" SSL certificate for local development + ## Docs: https://doc.traefik.io/traefik/ + traefik: + image: photoprism/traefik:220405 + ports: + # - "80:80" # HTTP (redirects to HTTPS) + - "443:443" # HTTPS (required) + labels: + - "traefik.enable=true" + - "traefik.http.routers.traefik.rule=Host(`traefik.localssl.dev`)" + - "traefik.http.routers.traefik.tls.domains[0].main=localssl.dev" + - "traefik.http.routers.traefik.tls.domains[0].sans=*.localssl.dev" + - "traefik.http.routers.traefik.tls=true" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" + - "traefik.http.routers.traefik.service=api@internal" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" # enables Traefik to watch services + ## Keycloak OpenID Connect Provider - ## Admin Account: admin / photoprism - ## User Account: user / photoprism + ## Login: user / photoprism + ## Admin: admin / photoprism keycloak: image: quay.io/keycloak/keycloak:17.0.1 command: "start-dev" # development mode, do not use this in production! @@ -174,6 +169,18 @@ services: KC_DB_USERNAME: "keycloak" KC_DB_PASSWORD: "keycloak" + ## Dummy OpenID Connect Provider + dummy-oidc: + image: photoprism/dummy-oidc:220405 + labels: + - "traefik.enable=true" + - "traefik.http.services.dummy-oidc.loadbalancer.server.port=9998" + - "traefik.http.routers.dummy-oidc.entrypoints=websecure" + - "traefik.http.routers.dummy-oidc.rule=Host(`dummy-oidc.localssl.dev`)" + - "traefik.http.routers.dummy-oidc.tls.domains[0].main=localssl.dev" + - "traefik.http.routers.dummy-oidc.tls.domains[0].sans=*.localssl.dev" + - "traefik.http.routers.dummy-oidc.tls=true" + ## Dummy WebDAV Server dummy-webdav: image: photoprism/dummy-webdav:220405 @@ -189,18 +196,6 @@ services: - "traefik.http.routers.dummy-webdav.tls.domains[0].sans=*.localssl.dev" - "traefik.http.routers.dummy-webdav.tls=true" - ## Dummy OpenID Connect Server - dummy-oidc: - image: photoprism/dummy-oidc:220405 - labels: - - "traefik.enable=true" - - "traefik.http.services.dummy-oidc.loadbalancer.server.port=9998" - - "traefik.http.routers.dummy-oidc.entrypoints=websecure" - - "traefik.http.routers.dummy-oidc.rule=Host(`dummy-oidc.localssl.dev`)" - - "traefik.http.routers.dummy-oidc.tls.domains[0].main=localssl.dev" - - "traefik.http.routers.dummy-oidc.tls.domains[0].sans=*.localssl.dev" - - "traefik.http.routers.dummy-oidc.tls=true" - ## Create named volume for Go module cache volumes: go-mod: diff --git a/docker/examples/arm64/docker-compose.yml b/docker/examples/arm64/docker-compose.yml index b8ae71b82..9c15d6030 100644 --- a/docker/examples/arm64/docker-compose.yml +++ b/docker/examples/arm64/docker-compose.yml @@ -75,8 +75,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive PHOTOPRISM_UPLOAD_NSFW: "true" # allow uploads that MAY be offensive # PHOTOPRISM_DATABASE_DRIVER: "sqlite" # SQLite is an embedded database that doesn't require a server diff --git a/docker/examples/armv7/docker-compose.yml b/docker/examples/armv7/docker-compose.yml index 1765b4f53..2625cfe0a 100644 --- a/docker/examples/armv7/docker-compose.yml +++ b/docker/examples/armv7/docker-compose.yml @@ -68,8 +68,11 @@ services: PHOTOPRISM_DISABLE_WEBDAV: "false" # disables built-in WebDAV server PHOTOPRISM_DISABLE_SETTINGS: "false" # disables Settings in Web UI PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow - PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition + PHOTOPRISM_DISABLE_FACES: "true" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "true" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) + PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive # PHOTOPRISM_DATABASE_DRIVER: "sqlite" # SQLite is an embedded database that doesn't require a server diff --git a/docker/examples/cloud/docker-compose.yml b/docker/examples/cloud/docker-compose.yml index 024c5ba45..64c6d55c3 100644 --- a/docker/examples/cloud/docker-compose.yml +++ b/docker/examples/cloud/docker-compose.yml @@ -146,8 +146,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive PHOTOPRISM_DATABASE_DRIVER: "mysql" # use MariaDB 10.5+ or MySQL 8+ instead of SQLite for improved performance diff --git a/docker/examples/docker-compose.yml b/docker/examples/docker-compose.yml index 2dda4675a..e6c11f883 100644 --- a/docker/examples/docker-compose.yml +++ b/docker/examples/docker-compose.yml @@ -66,8 +66,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive # PHOTOPRISM_DATABASE_DRIVER: "sqlite" # SQLite is an embedded database that doesn't require a server diff --git a/docker/examples/macos/docker-compose.yml b/docker/examples/macos/docker-compose.yml index 845166f6c..46088397f 100644 --- a/docker/examples/macos/docker-compose.yml +++ b/docker/examples/macos/docker-compose.yml @@ -63,8 +63,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive PHOTOPRISM_DATABASE_DRIVER: "mysql" # use MariaDB 10.5+ or MySQL 8+ instead of SQLite for improved performance diff --git a/docker/examples/scheduler/docker-compose.yml b/docker/examples/scheduler/docker-compose.yml index 39efb4fb2..489b9788a 100644 --- a/docker/examples/scheduler/docker-compose.yml +++ b/docker/examples/scheduler/docker-compose.yml @@ -68,8 +68,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive # PHOTOPRISM_DATABASE_DRIVER: "sqlite" # SQLite is an embedded database that doesn't require a server diff --git a/docker/examples/sqlite/docker-compose.yml b/docker/examples/sqlite/docker-compose.yml index b0d0643f8..29fd446e4 100644 --- a/docker/examples/sqlite/docker-compose.yml +++ b/docker/examples/sqlite/docker-compose.yml @@ -66,8 +66,9 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive PHOTOPRISM_DATABASE_DRIVER: "sqlite" # SQLite is an embedded database that doesn't require a server diff --git a/docker/examples/windows/docker-compose.yml b/docker/examples/windows/docker-compose.yml index e74549b2f..a1c6268b9 100644 --- a/docker/examples/windows/docker-compose.yml +++ b/docker/examples/windows/docker-compose.yml @@ -68,7 +68,8 @@ services: PHOTOPRISM_DISABLE_TENSORFLOW: "false" # disables all features depending on TensorFlow PHOTOPRISM_DISABLE_FACES: "false" # disables facial recognition PHOTOPRISM_DISABLE_CLASSIFICATION: "false" # disables image classification - PHOTOPRISM_RAW_PRESETS: "false" # enables RAW file converter presets (may reduce performance) + PHOTOPRISM_DISABLE_RAW: "false" # disables indexing and conversion of RAW files + PHOTOPRISM_RAW_PRESETS: "false" # enables applying user presets when converting RAW files (reduces performance) PHOTOPRISM_JPEG_QUALITY: 85 # image quality, a higher value reduces compression (25-100) PHOTOPRISM_DETECT_NSFW: "false" # flag photos as private that MAY be offensive (requires TensorFlow) PHOTOPRISM_UPLOAD_NSFW: "true" # allows uploads that MAY be offensive diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c18f3b52c..678328595 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -4348,9 +4348,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.103", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.103.tgz", - "integrity": "sha512-c/uKWR1Z/W30Wy/sx3dkZoj4BijbXX85QKWu9jJfjho3LBAXNEGAEW3oWiGb+dotA6C6BzCTxL2/aLes7jlUeg==" + "version": "1.4.104", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.104.tgz", + "integrity": "sha512-2kjoAyiG7uMyGRM9mx25s3HAzmQG2ayuYXxsFmYugHSDcwxREgLtscZvbL1JcW9S/OemeQ3f/SG6JhDwpnCclQ==" }, "node_modules/emoji-regex": { "version": "8.0.0", @@ -4937,23 +4937,23 @@ } }, "node_modules/eslint-plugin-import": { - "version": "2.25.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", - "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", + "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "dependencies": { "array-includes": "^3.1.4", "array.prototype.flat": "^1.2.5", "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.2", + "eslint-module-utils": "^2.7.3", "has": "^1.0.3", - "is-core-module": "^2.8.0", + "is-core-module": "^2.8.1", "is-glob": "^4.0.3", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "object.values": "^1.1.5", - "resolve": "^1.20.0", - "tsconfig-paths": "^3.12.0" + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" }, "engines": { "node": ">=4" @@ -5156,9 +5156,9 @@ } }, "node_modules/eslint-plugin-vue": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-8.5.0.tgz", - "integrity": "sha512-i1uHCTAKOoEj12RDvdtONWrGzjFm/djkzqfhmQ0d6M/W8KM81mhswd/z+iTZ0jCpdUedW3YRgcVfQ37/J4zoYQ==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-8.6.0.tgz", + "integrity": "sha512-abXiF2J18n/7ZPy9foSlJyouKf54IqpKlNvNmzhM93N0zs3QUxZG/oBd3tVPOJTKg7SlhBUtPxugpqzNbgGpQQ==", "dependencies": { "eslint-utils": "^3.0.0", "natural-compare": "^1.4.0", @@ -6037,19 +6037,19 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==" }, "node_modules/flow-parser": { - "version": "0.175.0", - "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.175.0.tgz", - "integrity": "sha512-9XG5JGOjhODF+OQF5ufCw8XiGi+8B46scjr3Q49JxN7IDRdT2W+1AOuvKKd6j766/5E7qSuCn/dsq1y3hihntg==", + "version": "0.175.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.175.1.tgz", + "integrity": "sha512-gYes5/nxeLYiu02MMb+WH4KaOIYrVcTVIuV9M4aP/4hqJ+zULxxS/In+WEj/tEBsQ+8/wSHo9IDWKQL1FhrLmA==", "engines": { "node": ">=0.4.0" } }, "node_modules/flow-remove-types": { - "version": "2.175.0", - "resolved": "https://registry.npmjs.org/flow-remove-types/-/flow-remove-types-2.175.0.tgz", - "integrity": "sha512-5DKqnBquIdg6KwwH4VtHw+eHw+uCrnhVCTi32q9wMWGjbe2FnpJTDbODKIQyMYowCGTb8jTs0Kqk4Hb66kY1Mg==", + "version": "2.175.1", + "resolved": "https://registry.npmjs.org/flow-remove-types/-/flow-remove-types-2.175.1.tgz", + "integrity": "sha512-malB3a9t7zEi5TMBbQlSZ47vqH673aiN+a34Xlv6/q3fJwyaXYlIz2wa9tMa3h5kM26aH7dFRS0GeBbfV1C5IQ==", "dependencies": { - "flow-parser": "^0.175.0", + "flow-parser": "^0.175.1", "pirates": "^3.0.2", "vlq": "^0.2.1" }, @@ -6320,9 +6320,9 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "node_modules/growl": { "version": "1.10.5", @@ -9028,15 +9028,19 @@ } }, "node_modules/postcss-custom-properties": { - "version": "12.1.5", - "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-12.1.5.tgz", - "integrity": "sha512-FHbbB/hRo/7cxLGkc2NS7cDRIDN1oFqQnUKBiyh4b/gwk8DD8udvmRDpUhEK836kB8ggUCieHVOvZDnF9XhI3g==", + "version": "12.1.6", + "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-12.1.6.tgz", + "integrity": "sha512-QEnQkDkb+J+j2bfJisJJpTAFL+lUFl66rUNvnjPBIvRbZACLG4Eu5bmBCIY4FJCqhwsfbBpmJUyb3FcR/31lAg==", "dependencies": { "postcss-value-parser": "^4.2.0" }, "engines": { "node": "^12 || ^14 || >=16" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + }, "peerDependencies": { "postcss": "^8.4" } @@ -11260,9 +11264,9 @@ } }, "node_modules/supercluster": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-7.1.4.tgz", - "integrity": "sha512-GhKkRM1jMR6WUwGPw05fs66pOFWhf59lXq+Q3J3SxPvhNcmgOtLRV6aVQPMRsmXdpaeFJGivt+t7QXUPL3ff4g==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-7.1.5.tgz", + "integrity": "sha512-EulshI3pGUM66o6ZdH3ReiFcvHpM3vAigyK+vcxdjpJyEbIIrtbmBdY23mGgnI24uXiGFvrGq9Gkum/8U7vJWg==", "dependencies": { "kdbush": "^3.0.0" } @@ -15911,9 +15915,9 @@ } }, "electron-to-chromium": { - "version": "1.4.103", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.103.tgz", - "integrity": "sha512-c/uKWR1Z/W30Wy/sx3dkZoj4BijbXX85QKWu9jJfjho3LBAXNEGAEW3oWiGb+dotA6C6BzCTxL2/aLes7jlUeg==" + "version": "1.4.104", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.104.tgz", + "integrity": "sha512-2kjoAyiG7uMyGRM9mx25s3HAzmQG2ayuYXxsFmYugHSDcwxREgLtscZvbL1JcW9S/OemeQ3f/SG6JhDwpnCclQ==" }, "emoji-regex": { "version": "8.0.0", @@ -16447,23 +16451,23 @@ } }, "eslint-plugin-import": { - "version": "2.25.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", - "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", + "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "requires": { "array-includes": "^3.1.4", "array.prototype.flat": "^1.2.5", "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.7.2", + "eslint-module-utils": "^2.7.3", "has": "^1.0.3", - "is-core-module": "^2.8.0", + "is-core-module": "^2.8.1", "is-glob": "^4.0.3", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "object.values": "^1.1.5", - "resolve": "^1.20.0", - "tsconfig-paths": "^3.12.0" + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" }, "dependencies": { "debug": { @@ -16604,9 +16608,9 @@ "requires": {} }, "eslint-plugin-vue": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-8.5.0.tgz", - "integrity": "sha512-i1uHCTAKOoEj12RDvdtONWrGzjFm/djkzqfhmQ0d6M/W8KM81mhswd/z+iTZ0jCpdUedW3YRgcVfQ37/J4zoYQ==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-8.6.0.tgz", + "integrity": "sha512-abXiF2J18n/7ZPy9foSlJyouKf54IqpKlNvNmzhM93N0zs3QUxZG/oBd3tVPOJTKg7SlhBUtPxugpqzNbgGpQQ==", "requires": { "eslint-utils": "^3.0.0", "natural-compare": "^1.4.0", @@ -17140,16 +17144,16 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==" }, "flow-parser": { - "version": "0.175.0", - "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.175.0.tgz", - "integrity": "sha512-9XG5JGOjhODF+OQF5ufCw8XiGi+8B46scjr3Q49JxN7IDRdT2W+1AOuvKKd6j766/5E7qSuCn/dsq1y3hihntg==" + "version": "0.175.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.175.1.tgz", + "integrity": "sha512-gYes5/nxeLYiu02MMb+WH4KaOIYrVcTVIuV9M4aP/4hqJ+zULxxS/In+WEj/tEBsQ+8/wSHo9IDWKQL1FhrLmA==" }, "flow-remove-types": { - "version": "2.175.0", - "resolved": "https://registry.npmjs.org/flow-remove-types/-/flow-remove-types-2.175.0.tgz", - "integrity": "sha512-5DKqnBquIdg6KwwH4VtHw+eHw+uCrnhVCTi32q9wMWGjbe2FnpJTDbODKIQyMYowCGTb8jTs0Kqk4Hb66kY1Mg==", + "version": "2.175.1", + "resolved": "https://registry.npmjs.org/flow-remove-types/-/flow-remove-types-2.175.1.tgz", + "integrity": "sha512-malB3a9t7zEi5TMBbQlSZ47vqH673aiN+a34Xlv6/q3fJwyaXYlIz2wa9tMa3h5kM26aH7dFRS0GeBbfV1C5IQ==", "requires": { - "flow-parser": "^0.175.0", + "flow-parser": "^0.175.1", "pirates": "^3.0.2", "vlq": "^0.2.1" }, @@ -17329,9 +17333,9 @@ "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" }, "graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "growl": { "version": "1.10.5", @@ -19263,9 +19267,9 @@ "requires": {} }, "postcss-custom-properties": { - "version": "12.1.5", - "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-12.1.5.tgz", - "integrity": "sha512-FHbbB/hRo/7cxLGkc2NS7cDRIDN1oFqQnUKBiyh4b/gwk8DD8udvmRDpUhEK836kB8ggUCieHVOvZDnF9XhI3g==", + "version": "12.1.6", + "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-12.1.6.tgz", + "integrity": "sha512-QEnQkDkb+J+j2bfJisJJpTAFL+lUFl66rUNvnjPBIvRbZACLG4Eu5bmBCIY4FJCqhwsfbBpmJUyb3FcR/31lAg==", "requires": { "postcss-value-parser": "^4.2.0" } @@ -20855,9 +20859,9 @@ } }, "supercluster": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-7.1.4.tgz", - "integrity": "sha512-GhKkRM1jMR6WUwGPw05fs66pOFWhf59lXq+Q3J3SxPvhNcmgOtLRV6aVQPMRsmXdpaeFJGivt+t7QXUPL3ff4g==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-7.1.5.tgz", + "integrity": "sha512-EulshI3pGUM66o6ZdH3ReiFcvHpM3vAigyK+vcxdjpJyEbIIrtbmBdY23mGgnI24uXiGFvrGq9Gkum/8U7vJWg==", "requires": { "kdbush": "^3.0.0" } diff --git a/frontend/src/model/config-options.js b/frontend/src/model/config-options.js index 444be6014..59bd32d60 100644 --- a/frontend/src/model/config-options.js +++ b/frontend/src/model/config-options.js @@ -44,6 +44,7 @@ export class ConfigOptions extends Model { DisableDarktable: config.values.disable.darktable, DisableRawtherapee: config.values.disable.rawtherapee, DisableSips: config.values.disable.sips, + DisableRaw: config.values.disable.raw, DisableHeifConvert: config.values.disable.heifconvert, DisableFFmpeg: config.values.disable.ffmpeg, DisableTensorFlow: config.values.disable.tensorflow, diff --git a/frontend/src/pages/settings/advanced.vue b/frontend/src/pages/settings/advanced.vue index 8faf395f6..2b2268fec 100644 --- a/frontend/src/pages/settings/advanced.vue +++ b/frontend/src/pages/settings/advanced.vue @@ -247,7 +247,7 @@ 1 { - log.Debugf("entity: updated statement affected more than one record - bug?") + log.Debugf("entity: updated statement affected more than one record - possible bug") return nil } else if res.RowsAffected == 1 { return nil diff --git a/internal/entity/marker.go b/internal/entity/marker.go index d50fb9f5a..4e0fa48d2 100644 --- a/internal/entity/marker.go +++ b/internal/entity/marker.go @@ -72,7 +72,7 @@ func (m *Marker) BeforeCreate(scope *gorm.Scope) error { // NewMarker creates a new entity. func NewMarker(file File, area crop.Area, subjUID, markerSrc, markerType string, size, score int) *Marker { if file.FileHash == "" { - log.Errorf("markers: file hash is empty - you might have found a bug") + log.Errorf("markers: file hash is empty - possible bug") return nil } diff --git a/internal/entity/photo.go b/internal/entity/photo.go index d689eb3fa..9a374de3e 100644 --- a/internal/entity/photo.go +++ b/internal/entity/photo.go @@ -442,7 +442,7 @@ func (m *Photo) IndexKeywords() error { kw := FirstOrCreateKeyword(NewKeyword(w)) if kw == nil { - log.Errorf("index keyword should not be nil - bug?") + log.Errorf("index keyword should not be nil - possible bug") continue } @@ -562,7 +562,7 @@ func (m *Photo) AddLabels(labels classify.Labels) { labelEntity := FirstOrCreateLabel(NewLabel(classifyLabel.Title(), classifyLabel.Priority)) if labelEntity == nil { - log.Errorf("index: label %s should not be nil - bug? (%s)", sanitize.Log(classifyLabel.Title()), m) + log.Errorf("index: label %s should not be nil - possible bug (%s)", sanitize.Log(classifyLabel.Title()), m) continue } @@ -578,7 +578,7 @@ func (m *Photo) AddLabels(labels classify.Labels) { photoLabel := FirstOrCreatePhotoLabel(NewPhotoLabel(m.ID, labelEntity.ID, classifyLabel.Uncertainty, classifyLabel.Source)) if photoLabel == nil { - log.Errorf("index: photo-label %d should not be nil - bug? (%s)", labelEntity.ID, m) + log.Errorf("index: photo-label %d should not be nil - possible bug (%s)", labelEntity.ID, m) continue } diff --git a/internal/entity/photo_label.go b/internal/entity/photo_label.go index 39ea61371..5df1f5020 100644 --- a/internal/entity/photo_label.go +++ b/internal/entity/photo_label.go @@ -87,7 +87,7 @@ func FirstOrCreatePhotoLabel(m *PhotoLabel) *PhotoLabel { // ClassifyLabel returns the label as classify.Label func (m *PhotoLabel) ClassifyLabel() classify.Label { if m.Label == nil { - log.Errorf("photo-label: classify label is nil (photo id %d, label id %d) - bug?", m.PhotoID, m.LabelID) + log.Errorf("photo-label: classify label is nil (photo id %d, label id %d) - possible bug", m.PhotoID, m.LabelID) return classify.Label{} } diff --git a/internal/ffmpeg/convert.go b/internal/ffmpeg/convert.go new file mode 100644 index 000000000..dcb1f0f5f --- /dev/null +++ b/internal/ffmpeg/convert.go @@ -0,0 +1,123 @@ +package ffmpeg + +import ( + "os/exec" +) + +// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC. +func AvcConvertCommand(fileName, avcName, ffmpegBin, bitrate string, encoder AvcEncoder) (result *exec.Cmd, useMutex bool, err error) { + // Don't transcode more than one video at the same time. + useMutex = true + + encoderName := string(encoder) + + // Display encoder info. + if encoder != SoftwareEncoder { + log.Infof("convert: ffmpeg encoder %s selected", encoderName) + } + + if encoder == IntelEncoder { + format := "format=rgb32" + + // Options: ffmpeg -hide_banner -h encoder=h264_qsv + result = exec.Command( + ffmpegBin, + "-qsv_device", "/dev/dri/renderD128", + "-i", fileName, + "-c:a", "aac", + "-vf", format, + "-c:v", string(encoder), + "-vsync", "vfr", + "-r", "30", + "-b:v", bitrate, + "-itrate", bitrate, + "-f", "mp4", + "-y", + avcName, + ) + } else if encoder == AppleEncoder { + format := "format=yuv420p" + + // Options: ffmpeg -hide_banner -h encoder=h264_videotoolbox + result = exec.Command( + ffmpegBin, + "-i", fileName, + "-c:v", string(encoder), + "-c:a", "aac", + "-vf", format, + "-profile", "high", + "-level", "51", + "-vsync", "vfr", + "-r", "30", + "-b:v", bitrate, + "-f", "mp4", + "-y", + avcName, + ) + } else if encoder == NvidiaEncoder { + // Options: ffmpeg -hide_banner -h encoder=h264_nvenc + result = exec.Command( + ffmpegBin, + "-r", "30", + "-i", fileName, + "-pix_fmt", "yuv420p", + "-c:v", string(encoder), + "-c:a", "aac", + "-preset", "15", + "-pixel_format", "yuv420p", + "-gpu", "any", + "-vf", "format=yuv420p", + "-rc:v", "constqp", + "-cq", "0", + "-tune", "2", + "-b:v", bitrate, + "-profile:v", "1", + "-level:v", "41", + "-coder:v", "1", + "-f", "mp4", + "-y", + avcName, + ) + } else if encoder == Video4LinuxEncoder { + format := "format=yuv420p" + + // Options: ffmpeg -hide_banner -h encoder=h264_v4l2m2m + result = exec.Command( + ffmpegBin, + "-i", fileName, + "-c:v", string(encoder), + "-c:a", "aac", + "-vf", format, + "-num_output_buffers", "72", + "-num_capture_buffers", "64", + "-max_muxing_queue_size", "1024", + "-crf", "23", + "-vsync", "vfr", + "-r", "30", + "-b:v", bitrate, + "-f", "mp4", + "-y", + avcName, + ) + } else { + format := "format=yuv420p" + + result = exec.Command( + ffmpegBin, + "-i", fileName, + "-c:v", string(encoder), + "-c:a", "aac", + "-vf", format, + "-max_muxing_queue_size", "1024", + "-crf", "23", + "-vsync", "vfr", + "-r", "30", + "-b:v", bitrate, + "-f", "mp4", + "-y", + avcName, + ) + } + + return result, useMutex, nil +} diff --git a/internal/ffmpeg/encoders.go b/internal/ffmpeg/encoders.go new file mode 100644 index 000000000..9b8e7eef0 --- /dev/null +++ b/internal/ffmpeg/encoders.go @@ -0,0 +1,63 @@ +package ffmpeg + +import "github.com/photoprism/photoprism/pkg/sanitize" + +// AvcEncoder represents a supported FFmpeg AVC encoder name. +type AvcEncoder string + +// String returns the FFmpeg AVC encoder name as string. +func (name AvcEncoder) String() string { + return string(name) +} + +// Supported FFmpeg AVC encoders. +const ( + SoftwareEncoder AvcEncoder = "libx264" // SoftwareEncoder see https://trac.ffmpeg.org/wiki/HWAccelIntro. + IntelEncoder AvcEncoder = "h264_qsv" // IntelEncoder is the Intel Quick Sync H.264 encoder. + AppleEncoder AvcEncoder = "h264_videotoolbox" // AppleEncoder is the Apple Video Toolbox H.264 encoder. + VAAPIEncoder AvcEncoder = "h264_vaapi" // VAAPIEncoder is the Video Acceleration API H.264 encoder. + NvidiaEncoder AvcEncoder = "h264_nvenc" // NvidiaEncoder is the NVIDIA H.264 encoder. + Video4LinuxEncoder AvcEncoder = "h264_v4l2m2m" // Video4LinuxEncoder is the Video4Linux H.264 encoder. +) + +// AvcEncoders is the list of supported H.264 encoders with aliases. +var AvcEncoders = map[string]AvcEncoder{ + "": SoftwareEncoder, + "default": SoftwareEncoder, + "software": SoftwareEncoder, + string(SoftwareEncoder): SoftwareEncoder, + "intel": IntelEncoder, + "qsv": IntelEncoder, + string(IntelEncoder): IntelEncoder, + "apple": AppleEncoder, + "osx": AppleEncoder, + "mac": AppleEncoder, + "macos": AppleEncoder, + "darwin": AppleEncoder, + string(AppleEncoder): AppleEncoder, + "vaapi": VAAPIEncoder, + "libva": VAAPIEncoder, + string(VAAPIEncoder): VAAPIEncoder, + "nvidia": NvidiaEncoder, + "nvenc": NvidiaEncoder, + "cuda": NvidiaEncoder, + string(NvidiaEncoder): NvidiaEncoder, + "v4l2": Video4LinuxEncoder, + "v4l": Video4LinuxEncoder, + "video4linux": Video4LinuxEncoder, + "rp4": Video4LinuxEncoder, + "raspberry": Video4LinuxEncoder, + "raspberrypi": Video4LinuxEncoder, + string(Video4LinuxEncoder): Video4LinuxEncoder, +} + +// FindEncoder finds an FFmpeg encoder by name. +func FindEncoder(s string) AvcEncoder { + if encoder, ok := AvcEncoders[s]; ok { + return encoder + } else { + log.Warnf("ffmpeg: unsupported encoder %s", sanitize.Log(s)) + } + + return SoftwareEncoder +} diff --git a/internal/ffmpeg/ffmpeg.go b/internal/ffmpeg/ffmpeg.go new file mode 100644 index 000000000..9b87e591f --- /dev/null +++ b/internal/ffmpeg/ffmpeg.go @@ -0,0 +1,33 @@ +/* + +Package ffmpeg provides FFmpeg video transcoding related types and functions. + +Copyright (c) 2018 - 2022 Michael Mayer + + This program is free software: you can redistribute it and/or modify + it under Version 3 of the GNU Affero General Public License (the "AGPL"): + + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + The AGPL is supplemented by our Trademark and Brand Guidelines, + which describe how our Brand Assets may be used: + + +Feel free to send an e-mail to hello@photoprism.app if you have questions, +want to support our work, or just want to say hello. + +Additional information can be found in our Developer Guide: + + +*/ +package ffmpeg + +import ( + "github.com/photoprism/photoprism/internal/event" +) + +var log = event.Log diff --git a/internal/photoprism/convert.go b/internal/photoprism/convert.go index f9dd30e92..17ca2272d 100644 --- a/internal/photoprism/convert.go +++ b/internal/photoprism/convert.go @@ -1,25 +1,17 @@ package photoprism import ( - "bytes" "errors" "fmt" - "math" - "os" - "os/exec" "path/filepath" "runtime/debug" - "strconv" "strings" "sync" - "time" "github.com/karrick/godirwalk" "github.com/photoprism/photoprism/internal/config" - "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/mutex" - "github.com/photoprism/photoprism/internal/thumb" "github.com/photoprism/photoprism/pkg/fs" "github.com/photoprism/photoprism/pkg/sanitize" ) @@ -130,242 +122,3 @@ func (c *Convert) Start(path string, force bool) (err error) { return err } - -// ToJson uses exiftool to export metadata to a json file. -func (c *Convert) ToJson(f *MediaFile) (jsonName string, err error) { - if f == nil { - return "", fmt.Errorf("exiftool: file is nil - you might have found a bug") - } - - jsonName, err = f.ExifToolJsonName() - - if err != nil { - return "", nil - } - - if fs.FileExists(jsonName) { - return jsonName, nil - } - - log.Debugf("exiftool: extracting metadata from %s", sanitize.Log(f.RootRelName())) - - cmd := exec.Command(c.conf.ExifToolBin(), "-n", "-m", "-api", "LargeFileSupport", "-j", f.FileName()) - - // Fetch command output. - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - // Log exact command for debugging in trace mode. - log.Trace(cmd.String()) - - // Run convert command. - if err := cmd.Run(); err != nil { - if stderr.String() != "" { - return "", errors.New(stderr.String()) - } else { - return "", err - } - } - - // Write output to file. - if err := os.WriteFile(jsonName, []byte(out.String()), os.ModePerm); err != nil { - return "", err - } - - // Check if file exists. - if !fs.FileExists(jsonName) { - return "", fmt.Errorf("exiftool: failed creating %s", filepath.Base(jsonName)) - } - - return jsonName, err -} - -// JpegConvertCommand returns the command for converting files to JPEG, depending on the format. -func (c *Convert) JpegConvertCommand(f *MediaFile, jpegName string, xmpName string) (result *exec.Cmd, useMutex bool, err error) { - if f == nil { - return result, useMutex, fmt.Errorf("file is nil - you might have found a bug") - } - - size := strconv.Itoa(c.conf.JpegSize()) - fileExt := f.Extension() - - if f.IsRaw() { - if c.conf.SipsEnabled() { - result = exec.Command(c.conf.SipsBin(), "-Z", size, "-s", "format", "jpeg", "--out", jpegName, f.FileName()) - } else if c.conf.DarktableEnabled() && c.darktableBlacklist.Ok(fileExt) { - var args []string - - // Set RAW, XMP, and JPEG filenames. - if xmpName != "" { - args = []string{f.FileName(), xmpName, jpegName} - } else { - args = []string{f.FileName(), jpegName} - } - - // Set RAW to JPEG conversion options. - if c.conf.RawPresets() { - useMutex = true // can run one instance only with presets enabled - args = append(args, "--width", size, "--height", size, "--hq", "true", "--upscale", "false") - } else { - useMutex = false // --apply-custom-presets=false disables locking - args = append(args, "--apply-custom-presets", "false", "--width", size, "--height", size, "--hq", "true", "--upscale", "false") - } - - // Set Darktable core storage paths. - args = append(args, "--core", "--configdir", c.conf.DarktableConfigPath(), "--cachedir", c.conf.DarktableCachePath(), "--library", ":memory:") - - result = exec.Command(c.conf.DarktableBin(), args...) - } else if c.conf.RawtherapeeEnabled() && c.rawtherapeeBlacklist.Ok(fileExt) { - jpegQuality := fmt.Sprintf("-j%d", c.conf.JpegQuality()) - profile := filepath.Join(conf.AssetsPath(), "profiles", "raw.pp3") - - args := []string{"-o", jpegName, "-p", profile, "-s", "-d", jpegQuality, "-js3", "-b8", "-c", f.FileName()} - - result = exec.Command(c.conf.RawtherapeeBin(), args...) - } else { - return nil, useMutex, fmt.Errorf("no suitable converter found") - } - } else if f.IsVideo() && c.conf.FFmpegEnabled() { - result = exec.Command(c.conf.FFmpegBin(), "-y", "-i", f.FileName(), "-ss", "00:00:00.001", "-vframes", "1", jpegName) - } else if f.IsHEIF() && c.conf.HeifConvertEnabled() { - result = exec.Command(c.conf.HeifConvertBin(), f.FileName(), jpegName) - } else { - return nil, useMutex, fmt.Errorf("file type %s not supported", f.FileType()) - } - - // Log convert command in trace mode only as it exposes server internals. - if result != nil { - log.Tracef("convert: %s", result.String()) - } - - return result, useMutex, nil -} - -// ToJpeg converts a single image file to JPEG if possible. -func (c *Convert) ToJpeg(f *MediaFile, force bool) (*MediaFile, error) { - if f == nil { - return nil, fmt.Errorf("convert: file is nil - you might have found a bug") - } - - if !f.Exists() { - return nil, fmt.Errorf("convert: %s not found", f.RootRelName()) - } - - if f.IsJpeg() { - return f, nil - } - - jpegName := fs.FormatJpeg.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) - - mediaFile, err := NewMediaFile(jpegName) - - // Replace existing sidecar if "force" is true. - if err == nil && mediaFile.IsJpeg() { - if force && mediaFile.InSidecar() { - if err := mediaFile.Remove(); err != nil { - return mediaFile, fmt.Errorf("convert: failed removing %s (%s)", mediaFile.RootRelName(), err) - } else { - log.Infof("convert: replacing %s", sanitize.Log(mediaFile.RootRelName())) - } - } else { - return mediaFile, nil - } - } else { - jpegName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.JpegExt) - } - - if !c.conf.SidecarWritable() { - return nil, fmt.Errorf("convert: disabled in read only mode (%s)", f.RootRelName()) - } - - fileName := f.RelName(c.conf.OriginalsPath()) - xmpName := fs.FormatXMP.Find(f.FileName(), false) - - event.Publish("index.converting", event.Data{ - "fileType": f.FileType(), - "fileName": fileName, - "baseName": filepath.Base(fileName), - "xmpName": filepath.Base(xmpName), - }) - - start := time.Now() - - if f.IsImageOther() { - log.Infof("convert: converting %s to %s (%s)", sanitize.Log(filepath.Base(fileName)), sanitize.Log(filepath.Base(jpegName)), f.FileType()) - - _, err = thumb.Jpeg(f.FileName(), jpegName, f.Orientation()) - - if err != nil { - return nil, err - } - - log.Infof("convert: %s created in %s (%s)", sanitize.Log(filepath.Base(jpegName)), time.Since(start), f.FileType()) - - return NewMediaFile(jpegName) - } - - cmd, useMutex, err := c.JpegConvertCommand(f, jpegName, xmpName) - - if err != nil { - return nil, err - } - - if useMutex { - // Make sure only one command is executed at a time. - // See https://photo.stackexchange.com/questions/105969/darktable-cli-fails-because-of-locked-database-file - c.cmdMutex.Lock() - defer c.cmdMutex.Unlock() - } - - if fs.FileExists(jpegName) { - return NewMediaFile(jpegName) - } - - // Fetch command output. - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - log.Infof("convert: converting %s to %s (%s)", sanitize.Log(filepath.Base(fileName)), sanitize.Log(filepath.Base(jpegName)), filepath.Base(cmd.Path)) - - // Log exact command for debugging in trace mode. - log.Trace(cmd.String()) - - // Run convert command. - if err := cmd.Run(); err != nil { - if stderr.String() != "" { - return nil, errors.New(stderr.String()) - } else { - return nil, err - } - } - - log.Infof("convert: %s created in %s (%s)", sanitize.Log(filepath.Base(jpegName)), time.Since(start), filepath.Base(cmd.Path)) - - return NewMediaFile(jpegName) -} - -// AvcBitrate returns the ideal AVC encoding bitrate in megabits per second. -func (c *Convert) AvcBitrate(f *MediaFile) string { - const defaultBitrate = "8M" - - if f == nil { - return defaultBitrate - } - - limit := c.conf.FFmpegBitrate() - quality := 12 - - bitrate := int(math.Ceil(float64(f.Width()*f.Height()*quality) / 1000000)) - - if bitrate <= 0 { - return defaultBitrate - } else if bitrate > limit { - bitrate = limit - } - - return fmt.Sprintf("%dM", bitrate) -} diff --git a/internal/photoprism/convert_avc.go b/internal/photoprism/convert_avc.go index b4787a2c4..0359ed979 100644 --- a/internal/photoprism/convert_avc.go +++ b/internal/photoprism/convert_avc.go @@ -4,196 +4,23 @@ import ( "bytes" "errors" "fmt" + "math" "os" "os/exec" "path/filepath" "time" "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/internal/ffmpeg" + "github.com/photoprism/photoprism/pkg/fs" "github.com/photoprism/photoprism/pkg/sanitize" ) -// FFmpegSoftwareEncoder see https://trac.ffmpeg.org/wiki/HWAccelIntro. -const FFmpegSoftwareEncoder = "libx264" - -// FFmpegIntelEncoder is the Intel Quick Sync H.264 encoder. -const FFmpegIntelEncoder = "h264_qsv" - -// FFmpegAppleEncoder is the Apple Video Toolboar H.264 encoder. -const FFmpegAppleEncoder = "h264_videotoolbox" - -// FFmpegVAAPIEncoder is the Video Acceleration API H.264 encoder. -const FFmpegVAAPIEncoder = "h264_vaapi" - -// FFmpegNvidiaEncoder is the NVIDIA H.264 encoder. -const FFmpegNvidiaEncoder = "h264_nvenc" - -// FFmpegV4L2Encoder is the Video4Linux H.264 encoder. -const FFmpegV4L2Encoder = "h264_v4l2m2m" - -// FFmpegAvcEncoders is the list of supported H.264 encoders with aliases. -var FFmpegAvcEncoders = map[string]string{ - "": FFmpegSoftwareEncoder, - "default": FFmpegSoftwareEncoder, - "software": FFmpegSoftwareEncoder, - FFmpegSoftwareEncoder: FFmpegSoftwareEncoder, - "intel": FFmpegIntelEncoder, - "qsv": FFmpegIntelEncoder, - FFmpegIntelEncoder: FFmpegIntelEncoder, - "apple": FFmpegAppleEncoder, - "osx": FFmpegAppleEncoder, - "mac": FFmpegAppleEncoder, - "macos": FFmpegAppleEncoder, - "darwin": FFmpegAppleEncoder, - FFmpegAppleEncoder: FFmpegAppleEncoder, - "vaapi": FFmpegVAAPIEncoder, - "libva": FFmpegVAAPIEncoder, - FFmpegVAAPIEncoder: FFmpegVAAPIEncoder, - "nvidia": FFmpegNvidiaEncoder, - "nvenc": FFmpegNvidiaEncoder, - "cuda": FFmpegNvidiaEncoder, - FFmpegNvidiaEncoder: FFmpegNvidiaEncoder, - "v4l2": FFmpegV4L2Encoder, - "v4l": FFmpegV4L2Encoder, - "video4linux": FFmpegV4L2Encoder, - "rp4": FFmpegV4L2Encoder, - "raspberry": FFmpegV4L2Encoder, - "raspberrypi": FFmpegV4L2Encoder, - FFmpegV4L2Encoder: FFmpegV4L2Encoder, -} - -// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC. -func (c *Convert) AvcConvertCommand(f *MediaFile, avcName, encoderName string) (result *exec.Cmd, useMutex bool, err error) { - if f.IsVideo() { - // Don't transcode more than one video at the same time. - useMutex = true - - // Display encoder info. - if encoderName != FFmpegSoftwareEncoder { - log.Infof("convert: ffmpeg encoder %s selected", encoderName) - } - - if encoderName == FFmpegIntelEncoder { - format := "format=rgb32" - - // Options: ffmpeg -hide_banner -h encoder=h264_qsv - result = exec.Command( - c.conf.FFmpegBin(), - "-qsv_device", "/dev/dri/renderD128", - "-i", f.FileName(), - "-c:a", "aac", - "-vf", format, - "-c:v", encoderName, - "-vsync", "vfr", - "-r", "30", - "-b:v", c.AvcBitrate(f), - "-maxrate", c.AvcBitrate(f), - "-f", "mp4", - "-y", - avcName, - ) - } else if encoderName == FFmpegAppleEncoder { - format := "format=yuv420p" - - // Options: ffmpeg -hide_banner -h encoder=h264_videotoolbox - result = exec.Command( - c.conf.FFmpegBin(), - "-i", f.FileName(), - "-c:v", encoderName, - "-c:a", "aac", - "-vf", format, - "-profile", "high", - "-level", "51", - "-vsync", "vfr", - "-r", "30", - "-b:v", c.AvcBitrate(f), - "-f", "mp4", - "-y", - avcName, - ) - } else if encoderName == FFmpegNvidiaEncoder { - // Options: ffmpeg -hide_banner -h encoder=h264_nvenc - result = exec.Command( - c.conf.FFmpegBin(), - "-r", "30", - "-i", f.FileName(), - "-pix_fmt", "yuv420p", - "-c:v", encoderName, - "-c:a", "aac", - "-preset", "15", - "-pixel_format", "yuv420p", - "-gpu", "any", - "-vf", "format=yuv420p", - "-rc:v", "constqp", - "-cq", "0", - "-tune", "2", - "-b:v", c.AvcBitrate(f), - "-profile:v", "1", - "-level:v", "41", - "-coder:v", "1", - "-f", "mp4", - "-y", - avcName, - ) - } else if encoderName == FFmpegV4L2Encoder { - format := "format=yuv420p" - - // Options: ffmpeg -hide_banner -h encoder=h264_v4l2m2m - result = exec.Command( - c.conf.FFmpegBin(), - "-i", f.FileName(), - "-c:v", encoderName, - "-c:a", "aac", - "-vf", format, - "-num_output_buffers", "72", - "-num_capture_buffers", "64", - "-max_muxing_queue_size", "1024", - "-crf", "23", - "-vsync", "vfr", - "-r", "30", - "-b:v", c.AvcBitrate(f), - "-f", "mp4", - "-y", - avcName, - ) - } else { - format := "format=yuv420p" - - result = exec.Command( - c.conf.FFmpegBin(), - "-i", f.FileName(), - "-c:v", encoderName, - "-c:a", "aac", - "-vf", format, - "-max_muxing_queue_size", "1024", - "-crf", "23", - "-vsync", "vfr", - "-r", "30", - "-b:v", c.AvcBitrate(f), - "-f", "mp4", - "-y", - avcName, - ) - } - } else { - return nil, useMutex, fmt.Errorf("convert: file type %s not supported in %s", f.FileType(), sanitize.Log(f.BaseName())) - } - - return result, useMutex, nil -} - // ToAvc converts a single video file to MPEG-4 AVC. -func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err error) { - if n := FFmpegAvcEncoders[encoderName]; n != "" { - encoderName = n - } else { - log.Warnf("convert: unsupported ffmpeg encoder %s", encoderName) - encoderName = FFmpegSoftwareEncoder - } - +func (c *Convert) ToAvc(f *MediaFile, encoder ffmpeg.AvcEncoder, noMutex, force bool) (file *MediaFile, err error) { if f == nil { - return nil, fmt.Errorf("convert: file is nil - you might have found a bug") + return nil, fmt.Errorf("convert: file is nil - possible bug") } if !f.Exists() { @@ -219,22 +46,30 @@ func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err fileName := f.RelName(c.conf.OriginalsPath()) avcName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.AvcExt) - cmd, useMutex, err := c.AvcConvertCommand(f, avcName, encoderName) + cmd, useMutex, err := c.AvcConvertCommand(f, avcName, encoder) if err != nil { log.Error(err) return nil, err } - if useMutex { - // Make sure only one command is executed at a time. - // See https://photo.stackexchange.com/questions/105969/darktable-cli-fails-because-of-locked-database-file + // Make sure only one convert command runs at a time. + if useMutex && !noMutex { c.cmdMutex.Lock() defer c.cmdMutex.Unlock() } if fs.FileExists(avcName) { - return NewMediaFile(avcName) + avcFile, avcErr := NewMediaFile(avcName) + if avcErr != nil { + return avcFile, avcErr + } else if !force || !avcFile.InSidecar() { + return avcFile, nil + } else if err = avcFile.Remove(); err != nil { + return avcFile, fmt.Errorf("convert: failed removing %s (%s)", sanitize.Log(avcFile.RootRelName()), err) + } else { + log.Infof("convert: replacing %s", sanitize.Log(avcFile.RootRelName())) + } } // Fetch command output. @@ -250,7 +85,7 @@ func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err "xmpName": "", }) - log.Infof("%s: transcoding %s to %s", encoderName, fileName, fs.FormatAvc) + log.Infof("%s: transcoding %s to %s", encoder, fileName, fs.FormatAvc) // Log exact command for debugging in trace mode. log.Trace(cmd.String()) @@ -258,8 +93,6 @@ func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err // Run convert command. start := time.Now() if err = cmd.Run(); err != nil { - _ = os.Remove(avcName) - if stderr.String() != "" { err = errors.New(stderr.String()) } @@ -270,17 +103,67 @@ func (c *Convert) ToAvc(f *MediaFile, encoderName string) (file *MediaFile, err } // Log filename and transcoding time. - log.Warnf("%s: failed transcoding %s [%s]", encoderName, fileName, time.Since(start)) + log.Warnf("%s: failed transcoding %s [%s]", encoder, fileName, time.Since(start)) - if encoderName != FFmpegSoftwareEncoder { - return c.ToAvc(f, FFmpegSoftwareEncoder) + // Remove broken video file. + if !fs.FileExists(avcName) { + // Do nothing. + } else if err = os.Remove(avcName); err != nil { + return nil, fmt.Errorf("convert: failed removing %s (%s)", sanitize.Log(RootRelName(avcName)), err) + } + + // Try again using software encoder. + if encoder != ffmpeg.SoftwareEncoder { + return c.ToAvc(f, ffmpeg.SoftwareEncoder, true, false) } else { return nil, err } } // Log transcoding time. - log.Infof("%s: created %s [%s]", encoderName, filepath.Base(avcName), time.Since(start)) + log.Infof("%s: created %s [%s]", encoder, filepath.Base(avcName), time.Since(start)) return NewMediaFile(avcName) } + +// AvcConvertCommand returns the command for converting video files to MPEG-4 AVC. +func (c *Convert) AvcConvertCommand(f *MediaFile, avcName string, encoder ffmpeg.AvcEncoder) (result *exec.Cmd, useMutex bool, err error) { + fileName := f.FileName() + bitrate := c.AvcBitrate(f) + ffmpegBin := c.conf.FFmpegBin() + + switch { + case fileName == "": + return nil, false, fmt.Errorf("convert: %s video filename is empty - possible bug", f.FileType()) + case bitrate == "": + return nil, false, fmt.Errorf("convert: transcoding bitrate is empty - possible bug") + case ffmpegBin == "": + return nil, false, fmt.Errorf("convert: ffmpeg must be installed to transcode %s to avc", sanitize.Log(f.BaseName())) + case !f.IsVideo(): + return nil, false, fmt.Errorf("convert: file type %s of %s cannot be transcoded to avc", f.FileType(), sanitize.Log(f.BaseName())) + } + + return ffmpeg.AvcConvertCommand(fileName, avcName, ffmpegBin, c.AvcBitrate(f), encoder) +} + +// AvcBitrate returns the ideal AVC encoding bitrate in megabits per second. +func (c *Convert) AvcBitrate(f *MediaFile) string { + const defaultBitrate = "8M" + + if f == nil { + return defaultBitrate + } + + limit := c.conf.FFmpegBitrate() + quality := 12 + + bitrate := int(math.Ceil(float64(f.Width()*f.Height()*quality) / 1000000)) + + if bitrate <= 0 { + return defaultBitrate + } else if bitrate > limit { + bitrate = limit + } + + return fmt.Sprintf("%dM", bitrate) +} diff --git a/internal/photoprism/convert_avc_test.go b/internal/photoprism/convert_avc_test.go index 4f8b63792..a2f390626 100644 --- a/internal/photoprism/convert_avc_test.go +++ b/internal/photoprism/convert_avc_test.go @@ -5,9 +5,10 @@ import ( "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/pkg/fs" - "github.com/stretchr/testify/assert" ) func TestConvert_ToAvc(t *testing.T) { @@ -28,7 +29,7 @@ func TestConvert_ToAvc(t *testing.T) { t.Fatal(err) } - avcFile, err := convert.ToAvc(mf, "") + avcFile, err := convert.ToAvc(mf, "", false, false) if err != nil { t.Fatal(err) @@ -59,7 +60,7 @@ func TestConvert_ToAvc(t *testing.T) { t.Fatal(err) } - avcFile, err := convert.ToAvc(mf, "") + avcFile, err := convert.ToAvc(mf, "", false, false) assert.Error(t, err) assert.Nil(t, avcFile) }) diff --git a/internal/photoprism/convert_jpeg.go b/internal/photoprism/convert_jpeg.go new file mode 100644 index 000000000..a3b77750e --- /dev/null +++ b/internal/photoprism/convert_jpeg.go @@ -0,0 +1,186 @@ +package photoprism + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/internal/thumb" + + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/sanitize" +) + +// ToJpeg converts a single image file to JPEG if possible. +func (c *Convert) ToJpeg(f *MediaFile, force bool) (*MediaFile, error) { + if f == nil { + return nil, fmt.Errorf("convert: file is nil - possible bug") + } + + if !f.Exists() { + return nil, fmt.Errorf("convert: %s not found", sanitize.Log(f.RootRelName())) + } + + if f.IsJpeg() { + return f, nil + } + + jpegName := fs.FormatJpeg.FindFirst(f.FileName(), []string{c.conf.SidecarPath(), fs.HiddenPath}, c.conf.OriginalsPath(), false) + + mediaFile, err := NewMediaFile(jpegName) + + // Replace existing sidecar if "force" is true. + if err == nil && mediaFile.IsJpeg() { + if force && mediaFile.InSidecar() { + if err := mediaFile.Remove(); err != nil { + return mediaFile, fmt.Errorf("convert: failed removing %s (%s)", sanitize.Log(mediaFile.RootRelName()), err) + } else { + log.Infof("convert: replacing %s", sanitize.Log(mediaFile.RootRelName())) + } + } else { + return mediaFile, nil + } + } else { + jpegName = fs.FileName(f.FileName(), c.conf.SidecarPath(), c.conf.OriginalsPath(), fs.JpegExt) + } + + if !c.conf.SidecarWritable() { + return nil, fmt.Errorf("convert: disabled in read only mode (%s)", sanitize.Log(f.RootRelName())) + } + + fileName := f.RelName(c.conf.OriginalsPath()) + xmpName := fs.FormatXMP.Find(f.FileName(), false) + + event.Publish("index.converting", event.Data{ + "fileType": f.FileType(), + "fileName": fileName, + "baseName": filepath.Base(fileName), + "xmpName": filepath.Base(xmpName), + }) + + start := time.Now() + + if f.IsImageOther() { + log.Infof("convert: converting %s to %s (%s)", sanitize.Log(filepath.Base(fileName)), sanitize.Log(filepath.Base(jpegName)), f.FileType()) + + _, err = thumb.Jpeg(f.FileName(), jpegName, f.Orientation()) + + if err != nil { + return nil, err + } + + log.Infof("convert: %s created in %s (%s)", sanitize.Log(filepath.Base(jpegName)), time.Since(start), f.FileType()) + + return NewMediaFile(jpegName) + } + + cmd, useMutex, err := c.JpegConvertCommand(f, jpegName, xmpName) + + if err != nil { + return nil, err + } + + if useMutex { + // Make sure only one command is executed at a time. + // See https://photo.stackexchange.com/questions/105969/darktable-cli-fails-because-of-locked-database-file + c.cmdMutex.Lock() + defer c.cmdMutex.Unlock() + } + + if fs.FileExists(jpegName) { + return NewMediaFile(jpegName) + } + + // Fetch command output. + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + log.Infof("convert: converting %s to %s (%s)", sanitize.Log(filepath.Base(fileName)), sanitize.Log(filepath.Base(jpegName)), filepath.Base(cmd.Path)) + + // Log exact command for debugging in trace mode. + log.Trace(cmd.String()) + + // Run convert command. + if err := cmd.Run(); err != nil { + if stderr.String() != "" { + return nil, errors.New(stderr.String()) + } else { + return nil, err + } + } + + log.Infof("convert: %s created in %s (%s)", sanitize.Log(filepath.Base(jpegName)), time.Since(start), filepath.Base(cmd.Path)) + + return NewMediaFile(jpegName) +} + +// JpegConvertCommand returns the command for converting files to JPEG, depending on the format. +func (c *Convert) JpegConvertCommand(f *MediaFile, jpegName string, xmpName string) (result *exec.Cmd, useMutex bool, err error) { + if f == nil { + return result, useMutex, fmt.Errorf("file is nil - possible bug") + } + + fileExt := f.Extension() + maxSize := strconv.Itoa(c.conf.JpegSize()) + + // Select conversion command depending on the file type and runtime environment. + if c.conf.SipsEnabled() && (f.IsRaw() || f.IsHEIF()) { + result = exec.Command(c.conf.SipsBin(), "-Z", maxSize, "-s", "format", "jpeg", "--out", jpegName, f.FileName()) + } else if f.IsRaw() && c.conf.RawEnabled() { + if c.conf.DarktableEnabled() && c.darktableBlacklist.Ok(fileExt) { + cachePath, configPath := conf.DarktableCachePath(), conf.DarktableConfigPath() + + var args []string + + // Set RAW, XMP, and JPEG filenames. + if xmpName != "" { + args = []string{f.FileName(), xmpName, jpegName} + } else { + args = []string{f.FileName(), jpegName} + } + + // Set RAW to JPEG conversion options. + if c.conf.RawPresets() { + useMutex = true // can run one instance only with presets enabled + args = append(args, "--width", maxSize, "--height", maxSize, "--hq", "true", "--upscale", "false") + } else { + useMutex = false // --apply-custom-presets=false disables locking + args = append(args, "--apply-custom-presets", "false", "--width", maxSize, "--height", maxSize, "--hq", "true", "--upscale", "false") + } + + // Set Darktable core storage paths. + args = append(args, "--core", "--configdir", configPath, "--cachedir", cachePath, "--library", ":memory:") + + result = exec.Command(c.conf.DarktableBin(), args...) + } else if c.conf.RawtherapeeEnabled() && c.rawtherapeeBlacklist.Ok(fileExt) { + jpegQuality := fmt.Sprintf("-j%d", c.conf.JpegQuality()) + profile := filepath.Join(conf.AssetsPath(), "profiles", "raw.pp3") + + args := []string{"-o", jpegName, "-p", profile, "-s", "-d", jpegQuality, "-js3", "-b8", "-c", f.FileName()} + + result = exec.Command(c.conf.RawtherapeeBin(), args...) + } else { + return nil, useMutex, fmt.Errorf("no suitable converter found") + } + } else if f.IsVideo() && c.conf.FFmpegEnabled() { + result = exec.Command(c.conf.FFmpegBin(), "-y", "-i", f.FileName(), "-ss", "00:00:00.001", "-vframes", "1", jpegName) + } else if f.IsHEIF() && c.conf.HeifConvertEnabled() { + result = exec.Command(c.conf.HeifConvertBin(), f.FileName(), jpegName) + } else { + return nil, useMutex, fmt.Errorf("file type %s not supported", f.FileType()) + } + + // Log convert command in trace mode only as it exposes server internals. + if result != nil { + log.Tracef("convert: %s", result.String()) + } + + return result, useMutex, nil +} diff --git a/internal/photoprism/convert_json.go b/internal/photoprism/convert_json.go new file mode 100644 index 000000000..cc36d2e25 --- /dev/null +++ b/internal/photoprism/convert_json.go @@ -0,0 +1,64 @@ +package photoprism + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/sanitize" +) + +// ToJson uses exiftool to export metadata to a json file. +func (c *Convert) ToJson(f *MediaFile) (jsonName string, err error) { + if f == nil { + return "", fmt.Errorf("exiftool: file is nil - possible bug") + } + + jsonName, err = f.ExifToolJsonName() + + if err != nil { + return "", nil + } + + if fs.FileExists(jsonName) { + return jsonName, nil + } + + log.Debugf("exiftool: extracting metadata from %s", sanitize.Log(f.RootRelName())) + + cmd := exec.Command(c.conf.ExifToolBin(), "-n", "-m", "-api", "LargeFileSupport", "-j", f.FileName()) + + // Fetch command output. + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + // Log exact command for debugging in trace mode. + log.Trace(cmd.String()) + + // Run convert command. + if err := cmd.Run(); err != nil { + if stderr.String() != "" { + return "", errors.New(stderr.String()) + } else { + return "", err + } + } + + // Write output to file. + if err := os.WriteFile(jsonName, []byte(out.String()), os.ModePerm); err != nil { + return "", err + } + + // Check if file exists. + if !fs.FileExists(jsonName) { + return "", fmt.Errorf("exiftool: failed creating %s", filepath.Base(jsonName)) + } + + return jsonName, err +} diff --git a/internal/photoprism/convert_worker.go b/internal/photoprism/convert_worker.go index 5a29b6937..86467bc0d 100644 --- a/internal/photoprism/convert_worker.go +++ b/internal/photoprism/convert_worker.go @@ -32,7 +32,7 @@ func ConvertWorker(jobs <-chan ConvertJob) { logError(err, job) } else if metaData := job.file.MetaData(); metaData.CodecAvc() { continue - } else if _, err := job.convert.ToAvc(job.file, job.convert.conf.FFmpegEncoder()); err != nil { + } else if _, err := job.convert.ToAvc(job.file, job.convert.conf.FFmpegEncoder(), false, false); err != nil { logError(err, job) } default: diff --git a/internal/photoprism/faces_cluster.go b/internal/photoprism/faces_cluster.go index afd6284ec..7cf086c34 100644 --- a/internal/photoprism/faces_cluster.go +++ b/internal/photoprism/faces_cluster.go @@ -72,7 +72,7 @@ func (w *Faces) Cluster(opt FacesOptions) (added entity.Faces, err error) { for _, cluster := range results { if f := entity.NewFace("", entity.SrcAuto, cluster); f == nil { - log.Errorf("faces: face should not be nil - bug?") + log.Errorf("faces: face should not be nil - possible bug") } else if f.SkipMatching() { log.Infof("faces: skipped cluster %s, embedding not distinct enough", f.ID) } else if err := f.Create(); err == nil { diff --git a/internal/photoprism/import.go b/internal/photoprism/import.go index 60a38faa8..74f73f961 100644 --- a/internal/photoprism/import.go +++ b/internal/photoprism/import.go @@ -102,7 +102,7 @@ func (imp *Import) Start(opt ImportOptions) fs.Done { convert := imp.conf.Settings().Index.Convert && imp.conf.SidecarWritable() indexOpt := NewIndexOptions("/", true, convert, true, false) - + skipRaw := imp.conf.DisableRaw() ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false) if err := ignore.Dir(importPath); err != nil { @@ -156,20 +156,23 @@ func (imp *Import) Start(opt ImportOptions) fs.Done { mf, err := NewMediaFile(fileName) + // Check if file exists and is not empty. if err != nil { + log.Warnf("import: %s", err) return nil } - if mf.FileSize() == 0 { - log.Infof("import: skipped empty file %s", sanitize.Log(mf.BaseName())) + // Ignore RAW images? + if mf.IsRaw() && skipRaw { + log.Infof("import: skipped raw %s", sanitize.Log(mf.RootRelName())) return nil } + // Find related files to import. related, err := mf.RelatedFiles(imp.conf.Settings().StackSequences()) if err != nil { event.Error(fmt.Sprintf("import: %s", err.Error())) - return nil } diff --git a/internal/photoprism/index.go b/internal/photoprism/index.go index 1abfccc73..1bd4f8f47 100644 --- a/internal/photoprism/index.go +++ b/internal/photoprism/index.go @@ -126,6 +126,7 @@ func (ind *Index) Start(o IndexOptions) fs.Done { defer ind.files.Done() filesIndexed := 0 + skipRaw := ind.conf.DisableRaw() ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false) if err := ignore.Dir(originalsPath); err != nil { @@ -176,25 +177,28 @@ func (ind *Index) Start(o IndexOptions) fs.Done { mf, err := NewMediaFile(fileName) + // Check if file exists and is not empty. if err != nil { - log.Error(err) + log.Errorf("index: %s", err) return nil } - if mf.FileSize() == 0 { - log.Infof("index: skipped empty file %s", sanitize.Log(mf.BaseName())) + // Ignore RAW images? + if mf.IsRaw() && skipRaw { + log.Infof("index: skipped raw %s", sanitize.Log(mf.RootRelName())) return nil } + // Skip? if ind.files.Indexed(relName, entity.RootOriginals, mf.modTime, o.Rescan) { return nil } + // Find related files to index. related, err := mf.RelatedFiles(ind.conf.Settings().StackSequences()) if err != nil { log.Warnf("index: %s", err.Error()) - return nil } diff --git a/internal/photoprism/index_mediafile.go b/internal/photoprism/index_mediafile.go index cd545a4bf..9beae1ac5 100644 --- a/internal/photoprism/index_mediafile.go +++ b/internal/photoprism/index_mediafile.go @@ -23,7 +23,7 @@ import ( // MediaFile indexes a single media file. func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName, photoUID string) (result IndexResult) { if m == nil { - err := errors.New("index: media file is nil - you might have found a bug") + err := errors.New("index: media file is nil - possible bug") log.Error(err) result.Err = err result.Status = IndexFailed diff --git a/internal/photoprism/mediafile.go b/internal/photoprism/mediafile.go index 82f5352d7..0db18e00e 100644 --- a/internal/photoprism/mediafile.go +++ b/internal/photoprism/mediafile.go @@ -74,8 +74,11 @@ func NewMediaFile(fileName string) (*MediaFile, error) { height: -1, } - if _, _, err := m.Stat(); err != nil { - return m, fmt.Errorf("media: %s not found", sanitize.Log(m.BaseName())) + // Check if file exists and is not empty. + if size, _, err := m.Stat(); err != nil { + return m, fmt.Errorf("%s not found", sanitize.Log(m.RootRelName())) + } else if size == 0 { + return m, fmt.Errorf("%s is empty", sanitize.Log(m.RootRelName())) } return m, nil @@ -297,6 +300,9 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e sidecarPrefix := Config().SidecarPath() + "/" originalsPrefix := Config().OriginalsPath() + "/" + // Ignore RAW images? + skipRaw := Config().DisableRaw() + // Replace sidecar with originals path in search prefix. if len(sidecarPrefix) > 1 && sidecarPrefix != originalsPrefix && strings.HasPrefix(prefix, sidecarPrefix) { prefix = strings.Replace(prefix, sidecarPrefix, originalsPrefix, 1) @@ -326,15 +332,16 @@ func (m *MediaFile) RelatedFiles(stripSequence bool) (result RelatedFiles, err e isHEIF := false for _, fileName := range matches { - f, err := NewMediaFile(fileName) + f, fileErr := NewMediaFile(fileName) - if err != nil { - log.Warnf("media: %s in %s", err, sanitize.Log(filepath.Base(fileName))) + if fileErr != nil { + log.Warn(fileErr) continue } - if f.FileSize() == 0 { - log.Warnf("media: %s is empty", sanitize.Log(filepath.Base(fileName))) + // Ignore RAW images? + if f.IsRaw() && skipRaw { + log.Debugf("media: skipped related raw file %s", sanitize.Log(f.RootRelName())) continue } diff --git a/internal/photoprism/mediafile_test.go b/internal/photoprism/mediafile_test.go index 20441d5d5..d987ede68 100644 --- a/internal/photoprism/mediafile_test.go +++ b/internal/photoprism/mediafile_test.go @@ -1742,10 +1742,10 @@ func TestMediaFile_Megapixels(t *testing.T) { } }) t.Run("2018-04-12 19_24_49.mov", func(t *testing.T) { - if f, err := NewMediaFile("testdata/2018-04-12 19_24_49.mov"); err != nil { - t.Fatal(err) + if _, err := NewMediaFile("testdata/2018-04-12 19_24_49.mov"); err != nil { + assert.EqualError(t, err, "'testdata/2018-04-12 19_24_49.mov' is empty") } else { - assert.Equal(t, 0, f.Megapixels()) + t.Errorf("error expected") } }) t.Run("rotate/6.png", func(t *testing.T) { diff --git a/internal/workers/sync_refresh.go b/internal/workers/sync_refresh.go index cb30f7a12..086e54e71 100644 --- a/internal/workers/sync_refresh.go +++ b/internal/workers/sync_refresh.go @@ -62,7 +62,7 @@ func (worker *Sync) refresh(a entity.Account) (complete bool, err error) { f = entity.FirstOrCreateFileSync(f) if f == nil { - log.Errorf("sync: file sync entity should not be nil - bug?") + log.Errorf("sync: file sync entity should not be nil - possible bug") continue }