CI: Fix prod build scripts and add docker-compose.local.yml #2262

2022-04-18 15:49:30 +02:00 · 2022-04-18 15:49:30 +02:00 · f093b498fa
commit f093b498fa
parent 975c4cdca3
16 changed files with 150 additions and 38 deletions
--- a/.gitignore
+++ b/.gitignore
@ -20,7 +20,6 @@
 *.db
 *.db-journal
 docker-compose.override.yml
-docker-compose.local.yml
 docker-compose.tmp.yml

 # Binaries for programs and plugins
--- a/49
+++ b/49
@ -35,10 +35,10 @@ all: dep build-js
 dep: dep-tensorflow dep-npm dep-js dep-go
 build: build-go
 test: test-js test-go
-test-go: reset-testdb run-test-go
-test-pkg: reset-testdb run-test-pkg
-test-api: reset-testdb run-test-api
-test-short: reset-testdb run-test-short
+test-go: reset-sqlite run-test-go
+test-pkg: reset-sqlite run-test-pkg
+test-api: reset-sqlite run-test-api
+test-short: reset-sqlite run-test-short
 test-mariadb: reset-acceptance run-test-mariadb
 acceptance-private-run-chromium: acceptance-private-restart acceptance-private acceptance-private-stop
 acceptance-public-run-chromium: acceptance-restart acceptance acceptance-stop
@ -53,6 +53,8 @@ clean-local: clean-local-config clean-local-cache
 upgrade: dep-upgrade-js dep-upgrade
 devtools: install-go dep-npm
 .SILENT: help;
+logs:
+	docker-compose logs -f
 help:
 	@echo "For build instructions, visit <https://docs.photoprism.app/developer-guide/>."
 fix-permissions:
@ -85,7 +87,7 @@ install:
 	(cd $(DESTDIR) && mkdir -p bin sbin lib assets config config/examples)
 	./scripts/build.sh prod "$(DESTDIR)/bin/$(BINARY_NAME)"
 	[ -f "$(GOBIN)/gosu" ] || go install github.com/tianon/gosu@latest
-	cp $(GOBIN)/gosu $(DESTDIR)/usr/local/sbin/gosu
+	cp $(GOBIN)/gosu $(DESTDIR)/sbin/gosu
 	[ ! -f "$(GOBIN)/exif-read-tool" ] || cp $(GOBIN)/exif-read-tool $(DESTDIR)/bin/exif-read-tool
 	rsync -r -l --safe-links --exclude-from=assets/.buildignore --chmod=a+r,u+rw ./assets/ $(DESTDIR)/assets
 	cp scripts/dist/heif-convert.sh $(DESTDIR)/bin/heif-convert
@ -211,13 +213,22 @@ acceptance-private-smoke:
 acceptance-private-firefox:
 	$(info Running JS acceptance-private tests in Firefox...)
 	(cd frontend &&	npm run acceptance-private-firefox && cd ..)
-reset-mariadb:
-	$(info Resetting photoprism database...)
-	mysql < scripts/sql/reset-mariadb.sql
-reset-acceptance:
+reset-mariadb-testdb:
+	$(info Resetting testdb database...)
+	mysql < scripts/sql/reset-testdb.sql
+reset-mariadb-local:
+	$(info Resetting local database...)
+	mysql < scripts/sql/reset-local.sql
+reset-mariadb-acceptance:
 	$(info Resetting acceptance database...)
-	echo "DROP DATABASE IF EXISTS acceptance;\nCREATE DATABASE IF NOT EXISTS acceptance;" | mysql
-reset-testdb:
+	mysql < scripts/sql/reset-acceptance.sql
+reset-mariadb-photoprism:
+	$(info Resetting photoprism database...)
+	mysql < scripts/sql/reset-photoprism.sql
+reset-mariadb: reset-mariadb-testdb reset-mariadb-local reset-mariadb-acceptance reset-mariadb-photoprism
+reset-testdb: reset-sqlite reset-mariadb-testdb
+reset-acceptance: reset-mariadb-acceptance
+reset-sqlite:
 	$(info Removing test database files...)
 	find ./internal -type f -name ".test.*" -delete
 run-test-short:
@ -334,7 +345,7 @@ docker-release-bookworm:
 	docker pull --platform=amd64 photoprism/develop:bookworm-slim
 	docker pull --platform=arm64 photoprism/develop:bookworm
 	docker pull --platform=arm64 photoprism/develop:bookworm-slim
-	scripts/docker/buildx-multi.sh photoprism linux/amd64,linux/arm64 bookworm /bookworm  "-t photoprism/photoprism:latest"
+	scripts/docker/buildx-multi.sh photoprism linux/amd64,linux/arm64 bookworm /bookworm "-t photoprism/photoprism:latest"
 docker-release-armv7:
 	docker pull --platform=arm photoprism/develop:armv7
 	docker pull --platform=arm debian:bookworm-slim
@ -367,28 +378,32 @@ docker-release-impish:
 	docker pull --platform=amd64 ubuntu:impish
 	docker pull --platform=arm64 ubuntu:impish
 	scripts/docker/buildx-multi.sh photoprism linux/amd64,linux/arm64 impish /impish
+start-local:
+	docker-compose -f docker-compose.local.yml up -d
+stop-local:
+	docker-compose -f docker-compose.local.yml stop
 docker-local: docker-local-bookworm
 docker-local-all: docker-local-bookworm docker-local-bullseye docker-local-buster docker-local-jammy
 docker-local-bookworm:
 	docker pull photoprism/develop:bookworm
 	docker pull photoprism/develop:bookworm-slim
-	scripts/docker/build.sh photoprism bookworm /bookworm
+	scripts/docker/build.sh photoprism bookworm /bookworm "-t photoprism/photoprism:local"
 docker-local-bullseye:
 	docker pull photoprism/develop:bullseye
 	docker pull photoprism/develop:bullseye-slim
-	scripts/docker/build.sh photoprism bullseye /bullseye
+	scripts/docker/build.sh photoprism bullseye /bullseye "-t photoprism/photoprism:local"
 docker-local-buster:
 	docker pull photoprism/develop:buster
 	docker pull debian:buster-slim
-	scripts/docker/build.sh photoprism buster /buster
+	scripts/docker/build.sh photoprism buster /buster "-t photoprism/photoprism:local"
 docker-local-jammy:
 	docker pull photoprism/develop:jammy
 	docker pull ubuntu:jammy
-	scripts/docker/build.sh photoprism jammy /jammy
+	scripts/docker/build.sh photoprism jammy /jammy "-t photoprism/photoprism:local"
 docker-local-impish:
 	docker pull photoprism/develop:impish
 	docker pull ubuntu:impish
-	scripts/docker/build.sh photoprism impish /impish
+	scripts/docker/build.sh photoprism impish /impish "-t photoprism/photoprism:local"
 docker-local-develop: docker-local-develop-bookworm
 docker-local-develop-all: docker-local-develop-bookworm docker-local-develop-bullseye docker-local-develop-buster docker-local-develop-impish
 docker-local-develop-bookworm:
--- a/docker-compose.latest.yml
+++ b/docker-compose.latest.yml
@ -38,9 +38,9 @@ services:
      PHOTOPRISM_HTTP_COMPRESSION: "gzip"     # improves transfer speed and bandwidth utilization (none or gzip)
      PHOTOPRISM_DATABASE_DRIVER: "mysql"
      PHOTOPRISM_DATABASE_SERVER: "mariadb:4001"
-      PHOTOPRISM_DATABASE_NAME: "photoprism_latest"
-      PHOTOPRISM_DATABASE_USER: "photoprism_latest"
-      PHOTOPRISM_DATABASE_PASSWORD: "photoprism_latest"
+      PHOTOPRISM_DATABASE_NAME: "latest"
+      PHOTOPRISM_DATABASE_USER: "latest"
+      PHOTOPRISM_DATABASE_PASSWORD: "latest"
      PHOTOPRISM_DISABLE_CHOWN: "false"       # disables storage permission updates on startup
      PHOTOPRISM_DISABLE_BACKUPS: "false"     # disables backing up albums and photo metadata to YAML files
      PHOTOPRISM_DISABLE_WEBDAV: "false"      # disables built-in WebDAV server
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@ -0,0 +1,77 @@
+version: '3.5'
+
+services:
+  ## Local Test Build
+  ## Docs: https://docs.photoprism.org/
+  photoprism-local:
+    image: photoprism/photoprism:local
+    security_opt:
+      - seccomp:unconfined
+      - apparmor:unconfined
+    ports:
+      - "2345:2342" # HTTP port (host:container)
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.latest.loadbalancer.server.port=2342"
+      - "traefik.http.routers.latest.entrypoints=websecure"
+      - "traefik.http.routers.latest.rule=Host(`local.localssl.dev`)"
+      - "traefik.http.routers.latest.tls.domains[0].main=localssl.dev"
+      - "traefik.http.routers.latest.tls.domains[0].sans=*.localssl.dev"
+      - "traefik.http.routers.latest.tls=true"
+    environment:
+      PHOTOPRISM_UID: ${UID:-1000}                   # user id, should match your host user id
+      PHOTOPRISM_GID: ${GID:-1000}                   # group id
+      PHOTOPRISM_ADMIN_PASSWORD: "photoprism"        # initial "admin" password (minimum 8 characters)
+      ## Public server URL incl http:// or https:// and /path, :port is optional
+      PHOTOPRISM_SITE_URL: "https://latest.localssl.dev/"
+      PHOTOPRISM_SITE_TITLE: "PhotoPrism"
+      PHOTOPRISM_SITE_CAPTION: "AI-Powered Photos App"
+      PHOTOPRISM_SITE_DESCRIPTION: "Open-Source Photo Management"
+      PHOTOPRISM_SITE_AUTHOR: "@photoprism_app"
+      PHOTOPRISM_DEBUG: "true"
+      PHOTOPRISM_READONLY: "false"
+      PHOTOPRISM_PUBLIC: "true"
+      PHOTOPRISM_EXPERIMENTAL: "false"
+      PHOTOPRISM_SERVER_MODE: "debug"
+      PHOTOPRISM_HTTP_HOST: "0.0.0.0"
+      PHOTOPRISM_HTTP_PORT: 2342
+      PHOTOPRISM_HTTP_COMPRESSION: "gzip"     # improves transfer speed and bandwidth utilization (none or gzip)
+      PHOTOPRISM_DATABASE_DRIVER: "mysql"
+      PHOTOPRISM_DATABASE_SERVER: "mariadb:4001"
+      PHOTOPRISM_DATABASE_NAME: "local"
+      PHOTOPRISM_DATABASE_USER: "local"
+      PHOTOPRISM_DATABASE_PASSWORD: "local"
+      PHOTOPRISM_DISABLE_CHOWN: "false"       # disables storage permission updates on startup
+      PHOTOPRISM_DISABLE_BACKUPS: "false"     # disables backing up albums and photo metadata to YAML files
+      PHOTOPRISM_DISABLE_WEBDAV: "false"      # disables built-in WebDAV server
+      PHOTOPRISM_DISABLE_SETTINGS: "false"    # disables settings UI and API
+      PHOTOPRISM_DISABLE_PLACES: "false"      # disables reverse geocoding and maps
+      PHOTOPRISM_DISABLE_EXIFTOOL: "false"    # disables creating JSON metadata sidecar files with ExifTool
+      PHOTOPRISM_DISABLE_TENSORFLOW: "false"  # disables all features depending on TensorFlow
+      PHOTOPRISM_DETECT_NSFW: "false"         # flag photos as private that MAY be offensive (requires TensorFlow)
+      PHOTOPRISM_UPLOAD_NSFW: "false"         # allows uploads that may be offensive
+      PHOTOPRISM_RAW_PRESETS: "false"         # enables applying user presets when converting RAW files (reduces performance)
+      PHOTOPRISM_THUMB_FILTER: "lanczos"      # resample filter, best to worst: blackman, lanczos, cubic, linear
+      PHOTOPRISM_THUMB_UNCACHED: "true"       # enables on-demand thumbnail rendering (high memory and cpu usage)
+      PHOTOPRISM_THUMB_SIZE: 2048             # pre-rendered thumbnail size limit (default 2048, min 720, max 7680)
+      # PHOTOPRISM_THUMB_SIZE: 4096           # Retina 4K, DCI 4K (requires more storage); 7680 for 8K Ultra HD
+      PHOTOPRISM_THUMB_SIZE_UNCACHED: 7680    # on-demand rendering size limit (default 7680, min 720, max 7680)
+      PHOTOPRISM_JPEG_SIZE: 7680              # size limit for converted image files in pixels (720-30000)
+      PHOTOPRISM_JPEG_QUALITY: 85             # image quality, a higher value reduces compression (25-100)
+      TF_CPP_MIN_LOG_LEVEL: 0                 # show TensorFlow log messages for development
+      PHOTOPRISM_INIT: "gpu tensorflow"       # Options: "update gpu tensorflow davfs clitools clean"
+      PHOTOPRISM_FFMPEG_ENCODER: "nvidia"     # Options: "software", "intel", "nvidia", "apple", "raspberry"
+      PHOTOPRISM_STORAGE_PATH: "/photoprism/storage"
+      PHOTOPRISM_ORIGINALS_PATH: "/photoprism/storage/originals"
+      PHOTOPRISM_IMPORT_PATH: "/photoprism/storage/import"
+    devices:
+      - "/dev/dri:/dev/dri" # Intel QuickSync
+    working_dir: "/photoprism"
+    volumes:
+      - "./storage:/photoprism/storage"
+
+## Join shared "photoprism-develop" network
+networks:
+  default:
+    external:
+      name: photoprism-develop
--- a/docker/photoprism/armv7/Dockerfile
+++ b/docker/photoprism/armv7/Dockerfile
@ -87,7 +87,7 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
    echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests && \
    echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes && \
    echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissing && \
-    mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+    mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu && \
    groupadd -f -r -g 44 video && groupadd -f -r -g 109 render && groupadd -f -g 1000 photoprism && \
--- a/docker/photoprism/bookworm/Dockerfile
+++ b/docker/photoprism/bookworm/Dockerfile
@ -87,7 +87,7 @@ EXPOSE 2342

 # copy dist files
 COPY --from=build /opt/photoprism/ /opt/photoprism
-RUN mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+RUN mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu

--- a/docker/photoprism/bullseye/Dockerfile
+++ b/docker/photoprism/bullseye/Dockerfile
@ -87,7 +87,7 @@ EXPOSE 2342

 # copy dist files
 COPY --from=build /opt/photoprism/ /opt/photoprism
-RUN mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+RUN mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu

--- a/docker/photoprism/buster/Dockerfile
+++ b/docker/photoprism/buster/Dockerfile
@ -87,7 +87,7 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
    echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests && \
    echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes && \
    echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissing && \
-    mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+    mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu && \
    groupadd -f -r -g 44 video && groupadd -f -r -g 109 render && groupadd -f -g 1000 photoprism && \
--- a/docker/photoprism/impish/Dockerfile
+++ b/docker/photoprism/impish/Dockerfile
@ -86,7 +86,7 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
    echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests && \
    echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes && \
    echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissing && \
-    mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+    mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu && \
    groupadd -f -r -g 44 video && groupadd -f -r -g 109 render && groupadd -f -g 1000 photoprism && \
--- a/docker/photoprism/jammy/Dockerfile
+++ b/docker/photoprism/jammy/Dockerfile
@ -86,7 +86,7 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
    echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests && \
    echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes && \
    echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissing && \
-    mv /opt/photoprism/usr/local/sbin/gosu /usr/local/sbin/gosu && \
+    mv /opt/photoprism/sbin/gosu /usr/local/sbin/gosu && \
    chown root:root /usr/local/sbin/gosu && \
    chmod 755 /usr/local/sbin/gosu && \
    groupadd -f -r -g 44 video && groupadd -f -r -g 109 render && groupadd -f -g 1000 photoprism && \
--- a/scripts/dist/entrypoint-init.sh
+++ b/scripts/dist/entrypoint-init.sh
@ -44,8 +44,8 @@ if [[ ${PHOTOPRISM_UID} =~ $re ]] && [[ ${PHOTOPRISM_UID} != "0" ]]; then
  if [[ -z ${PHOTOPRISM_DISABLE_CHOWN} ]] || [[ ${PHOTOPRISM_DISABLE_CHOWN} == "false" ]]; then
    echo "init: updating filesystem permissions"
    echo "PHOTOPRISM_DISABLE_CHOWN=\"true\" disables permission updates"
-    chown --preserve-root --no-dereference --silent -R "${CHOWN}" "${CHOWN_DIRS[@]}"
-    chmod --preserve-root --no-dereference --silent -R u+rwX "${CHMOD_DIRS[@]}"
+    chown --preserve-root --silent -R "${CHOWN}" "${CHOWN_DIRS[@]}"
+    chmod --preserve-root --silent -R u+rwX "${CHMOD_DIRS[@]}"
  fi
 fi

--- a/scripts/sql/mariadb-init.sql
+++ b/scripts/sql/mariadb-init.sql
@ -13,13 +13,21 @@ CREATE DATABASE IF NOT EXISTS keycloak;
 CREATE USER IF NOT EXISTS keycloak@'%' IDENTIFIED BY 'keycloak';
 GRANT ALL PRIVILEGES ON keycloak.* TO keycloak@'%';

-CREATE DATABASE IF NOT EXISTS photoprism_latest;
-CREATE USER IF NOT EXISTS photoprism_latest@'%' IDENTIFIED BY 'photoprism_latest';
-GRANT ALL PRIVILEGES ON photoprism_latest.* TO photoprism_latest@'%';
+CREATE DATABASE IF NOT EXISTS `local`;
+CREATE USER IF NOT EXISTS 'local'@'%' IDENTIFIED BY 'local';
+GRANT ALL PRIVILEGES ON `local`.* TO 'local'@'%';

-CREATE DATABASE IF NOT EXISTS photoprism_preview;
-CREATE USER IF NOT EXISTS photoprism_preview@'%' IDENTIFIED BY 'photoprism_preview';
-GRANT ALL PRIVILEGES ON photoprism_preview.* TO photoprism_preview@'%';
+CREATE DATABASE IF NOT EXISTS latest;
+CREATE USER IF NOT EXISTS latest@'%' IDENTIFIED BY 'latest';
+GRANT ALL PRIVILEGES ON latest.* TO latest@'%';
+
+CREATE DATABASE IF NOT EXISTS preview;
+CREATE USER IF NOT EXISTS preview@'%' IDENTIFIED BY 'preview';
+GRANT ALL PRIVILEGES ON preview.* TO preview@preview;
+
+CREATE DATABASE IF NOT EXISTS testdb;
+CREATE USER IF NOT EXISTS testdb@'%' IDENTIFIED BY 'testdb';
+GRANT ALL PRIVILEGES ON testdb.* TO testdb@'%';

 CREATE DATABASE IF NOT EXISTS acceptance;
 CREATE USER IF NOT EXISTS acceptance@'%' IDENTIFIED BY 'acceptance';
--- a/scripts/sql/reset-acceptance.sql
+++ b/scripts/sql/reset-acceptance.sql
@ -1,6 +1,3 @@
-DROP DATABASE IF EXISTS photoprism;
-CREATE DATABASE IF NOT EXISTS photoprism;
-
 DROP DATABASE IF EXISTS acceptance;
 CREATE DATABASE IF NOT EXISTS acceptance;
 CREATE USER IF NOT EXISTS acceptance@'%' IDENTIFIED BY 'acceptance';
--- a/scripts/sql/reset-local.sql
+++ b/scripts/sql/reset-local.sql
@ -0,0 +1,6 @@
+DROP DATABASE IF EXISTS `local`;
+CREATE DATABASE IF NOT EXISTS `local`;
+CREATE USER IF NOT EXISTS 'local'@'%' IDENTIFIED BY 'local';
+GRANT ALL PRIVILEGES ON `local`.* TO 'local'@'%';
+
+FLUSH PRIVILEGES;
--- a/scripts/sql/reset-photoprism.sql
+++ b/scripts/sql/reset-photoprism.sql
@ -0,0 +1,4 @@
+DROP DATABASE IF EXISTS photoprism;
+CREATE DATABASE IF NOT EXISTS photoprism;
+
+FLUSH PRIVILEGES;
--- a/scripts/sql/reset-testdb.sql
+++ b/scripts/sql/reset-testdb.sql
@ -0,0 +1,6 @@
+DROP DATABASE IF EXISTS testdb;
+CREATE DATABASE IF NOT EXISTS testdb;
+CREATE USER IF NOT EXISTS testdb@'%' IDENTIFIED BY 'testdb';
+GRANT ALL PRIVILEGES ON testdb.* TO testdb@'%';
+
+FLUSH PRIVILEGES;