Smaller docker images with multi-stage build #39

This commit is contained in:
Michael Mayer 2018-11-08 10:56:11 +01:00
parent f657bc5543
commit ac31141669
19 changed files with 279 additions and 130 deletions

View file

@ -7,14 +7,15 @@ before_script:
- docker-compose -f docker-compose.travis.yml up -d --build
script:
- docker-compose -f docker-compose.travis.yml exec photoprism make migrate test-codecov
- if [ "$TRAVIS_BRANCH" == "develop" ]; then docker-compose -f docker-compose.travis.yml exec photoprism make migrate test-codecov; fi
- if [ "$TRAVIS_BRANCH" != "develop" ]; then docker-compose -f docker-compose.travis.yml exec photoprism make migrate test; fi
after_script:
- docker-compose -f docker-compose.travis.yml down
deploy:
provider: script
script: make docker-push
script: make deploy-photoprism
skip_cleanup: true
on:
branch: master

View file

@ -1,81 +1,4 @@
FROM ubuntu:18.04
LABEL maintainer="Michael Mayer <michael@liquidbytes.net>"
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
curl \
libfreetype6-dev \
libhdf5-serial-dev \
libpng-dev \
libzmq3-dev \
pkg-config \
rsync \
software-properties-common \
unzip \
g++ \
gcc \
libc6-dev \
gpg-agent \
apt-utils \
make \
nano \
wget \
darktable \
git \
mysql-client \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN add-apt-repository ppa:pmjdebruijn/darktable-release
RUN apt-get update && apt-get install -y --no-install-recommends \
darktable \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN apt-get upgrade -y
# Install TensorFlow C library
RUN curl -L \
"https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.11.0.tar.gz" | \
tar -C "/usr/local" -xz
RUN ldconfig
# Hide some warnings
ENV TF_CPP_MIN_LOG_LEVEL 2
# Install NPM (NodeJS)
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
RUN apt-get install -y nodejs
# Install YARN (Package Manager)
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
RUN apt-get update && apt-get install yarn
ENV GOLANG_VERSION 1.11.2
RUN set -eux; \
\
url="https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz"; \
wget -O go.tgz "$url"; \
echo "1dfe664fa3d8ad714bbd15a36627992effd150ddabd7523931f077b3926d736d *go.tgz" | sha256sum -c -; \
tar -C /usr/local -xzf go.tgz; \
rm go.tgz; \
export PATH="/usr/local/go/bin:$PATH"; \
go version
ENV GOPATH /go
ENV GOBIN $GOPATH/bin
ENV PATH $GOBIN:/usr/local/go/bin:$PATH
ENV GO111MODULE on
ENV NODE_ENV production
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
RUN env GO111MODULE=off /usr/local/go/bin/go get golang.org/x/tools/cmd/goimports
FROM photoprism/development:20181107
# Set up project directory
WORKDIR "/go/src/github.com/photoprism/photoprism"
@ -84,8 +7,5 @@ COPY . .
# Build PhotoPrism
RUN make all install
# Expose HTTP port
EXPOSE 80
# Start PhotoPrism server
CMD photoprism start

View file

@ -11,24 +11,25 @@ GOGET=$(GOCMD) get
GOFMT=$(GOCMD) fmt
GOIMPORTS=goimports
BINARY_NAME=photoprism
DOCKER_TAG=`date -u +%Y%m%d`
all: tensorflow-model dep js build
install: install-bin install-assets install-config
install-bin:
$(GOINSTALL) cmd/photoprism/photoprism.go
cp $(BINARY_NAME) /usr/local/bin/$(BINARY_NAME)
install-assets:
mkdir -p /var/photoprism
mkdir -p /var/photoprism/photos
mkdir -p /var/photoprism/thumbnails
cp -r assets/favicons /var/photoprism
cp -r assets/public /var/photoprism
cp -r assets/templates /var/photoprism
cp -r assets/tensorflow /var/photoprism
mkdir -p /srv/photoprism
mkdir -p /srv/photoprism/photos
mkdir -p /srv/photoprism/thumbnails
cp -r assets/favicons /srv/photoprism
cp -r assets/public /srv/photoprism
cp -r assets/templates /srv/photoprism
cp -r assets/tensorflow /srv/photoprism
install-config:
mkdir -p /etc/photoprism
test -e /etc/photoprism/photoprism.yml || cp -n configs/photoprism.yml /etc/photoprism/photoprism.yml
build:
$(GOBUILD) cmd/photoprism/photoprism.go
scripts/build.sh
js:
(cd frontend && yarn install --prod)
(cd frontend && env NODE_ENV=production npm run build)
@ -51,8 +52,15 @@ clean:
rm -f $(BINARY_NAME)
tensorflow-model:
scripts/download-tf-model.sh
docker-push:
scripts/docker-push.sh
deploy-photoprism:
scripts/docker-build.sh photoprism $(DOCKER_TAG)
scripts/docker-push.sh photoprism $(DOCKER_TAG)
deploy-development:
scripts/docker-build.sh development $(DOCKER_TAG)
scripts/docker-push.sh development $(DOCKER_TAG)
deploy-tensorflow:
scripts/docker-build.sh tensorflow $(DOCKER_TAG)
scripts/docker-push.sh tensorflow $(DOCKER_TAG)
fmt:
$(GOIMPORTS) -w internal cmd
$(GOFMT) ./internal/... ./cmd/...

View file

@ -15,8 +15,8 @@
<script>
window.appConfig = {
appName: "{{ .title }}",
appVersion: "1.0.0",
appName: "{{ .appName }}",
appVersion: "{{ .appVersion }}",
debug: {{ .debug }},
cameras: {{ .cameras }},
countries: {{ .countries }}

View file

@ -21,4 +21,4 @@ CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
*/
package main
package main

View file

@ -7,11 +7,13 @@ import (
"github.com/urfave/cli"
)
var version = "development"
func main() {
app := cli.NewApp()
app.Name = "PhotoPrism"
app.Usage = "Browse your life in pictures"
app.Version = "0.0.0"
app.Version = version
app.Copyright = "Copyright (c) 2018 The PhotoPrism contributors"
app.EnableBashCompletion = true
app.Flags = commands.GlobalFlags

View file

@ -1,21 +1,21 @@
version: '3.3'
# Example docker-compose config file for production use
# Usage: docker-compose -f docker-compose.prod.yml up
# Usage: docker-compose up
services:
photoprism: # change if needed
image: photoprism/photoprism # use pre-built image from docker hub: https://hub.docker.com/r/photoprism/photoprism/
image: photoprism/photoprism:latest # use pre-built image from docker hub: https://hub.docker.com/r/photoprism/photoprism/
restart: always
ports:
- 2342:80 # left side is your local port (change if port 2342 is already used or you want to use port 80)
volumes:
- ~/Photos:/Photos # change ~/Photos to whatever directory you want to use on your local computer
- photoprism-thumbnails:/var/photoprism/thumbnails # keep this (thumbnail cache)
- ~/Photos:/srv/photoprism/photos # change ~/Photos to whatever directory you want to use on your local computer
- photoprism-thumbnails:/srv/photoprism/thumbnails # keep this (thumbnail cache)
environment:
PHOTOPRISM_IMPORT_PATH: "/Photos/Import" # ~/Photos/Import (files to be imported to originals)
PHOTOPRISM_EXPORT_PATH: "/Photos/Export" # ~/Photos/Export (files exported from originals)
PHOTOPRISM_ORIGINALS_PATH: "/Photos/Originals" # ~/Photos/Originals (original jpeg, raw and meta files)
PHOTOPRISM_IMPORT_PATH: /srv/photoprism/photos/Import # ~/Photos/Import (files to be imported to originals)
PHOTOPRISM_EXPORT_PATH: /srv/photoprism/photos/Export # ~/Photos/Export (files exported from originals)
PHOTOPRISM_ORIGINALS_PATH: /srv/photoprism/photos/Originals # ~/Photos/Originals (original jpeg, raw and meta files)
database: # keep this
image: mysql:latest

View file

@ -1,10 +1,10 @@
debug: false
darktable-cli: /usr/bin/darktable-cli
assets-path: /var/photoprism
thumbnails-path: /var/photoprism/thumbnails
originals-path: /var/photoprism/photos/originals
import-path: /var/photoprism/photos/import
export-path: /var/photoprism/photos/export
assets-path: /srv/photoprism
thumbnails-path: /srv/photoprism/thumbnails
originals-path: /srv/photoprism/photos/originals
import-path: /srv/photoprism/photos/import
export-path: /srv/photoprism/photos/export
server-host:
server-mode: release
server-port: 80

View file

@ -3,7 +3,7 @@ version: '3.3'
services:
photoprism:
build: .
image: photoprism/photoprism
image: photoprism/photoprism:develop
command: tail -f /dev/null
environment:
- CODECOV_TOKEN

View file

@ -3,7 +3,7 @@ version: '3.3'
services:
photoprism:
build: .
image: photoprism/photoprism
image: photoprism/photoprism:develop
command: tail -f /dev/null
ports:
- 2342:80

View file

@ -0,0 +1,79 @@
FROM ubuntu:18.04
LABEL maintainer="Michael Mayer <michael@liquidbytes.net>"
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
curl \
libfreetype6-dev \
libhdf5-serial-dev \
libpng-dev \
libzmq3-dev \
pkg-config \
software-properties-common \
rsync \
unzip \
g++ \
gcc \
libc6-dev \
gpg-agent \
apt-utils \
make \
nano \
wget \
git \
mysql-client
# Install darktable (RAW to JPEG converter)
RUN add-apt-repository ppa:pmjdebruijn/darktable-release && \
apt-get update && \
apt-get install -y --no-install-recommends darktable && \
apt-get upgrade -y
# Install TensorFlow C library
RUN curl -L \
"https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.11.0.tar.gz" | \
tar -C "/usr/local" -xz
RUN ldconfig
# Hide some warnings
ENV TF_CPP_MIN_LOG_LEVEL 2
# Install NPM (NodeJS)
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
RUN apt-get install -y nodejs
# Install YARN (Package Manager)
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
RUN apt-get update && \
apt-get install yarn && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV GOLANG_VERSION 1.11.2
RUN set -eux; \
\
url="https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz"; \
wget -O go.tgz "$url"; \
echo "1dfe664fa3d8ad714bbd15a36627992effd150ddabd7523931f077b3926d736d *go.tgz" | sha256sum -c -; \
tar -C /usr/local -xzf go.tgz; \
rm go.tgz; \
export PATH="/usr/local/go/bin:$PATH"; \
go version
ENV GOPATH /go
ENV GOBIN $GOPATH/bin
ENV PATH $GOBIN:/usr/local/go/bin:$PATH
ENV GO111MODULE on
ENV NODE_ENV production
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
RUN env GO111MODULE=off /usr/local/go/bin/go get golang.org/x/tools/cmd/goimports
# Expose HTTP port
EXPOSE 80
# Start PhotoPrism server
CMD tail -f /dev/null

View file

@ -0,0 +1,50 @@
FROM photoprism/development:20181107 as build
# Set up project directory
WORKDIR "/go/src/github.com/photoprism/photoprism"
COPY . .
# Build PhotoPrism
RUN make all install
# Base base image as photoprism/development
FROM ubuntu:18.04
WORKDIR /srv/photoprism
# Install additional distribution packages
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
unzip \
nano \
wget \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Copy built binaries and assets to this image
COPY --from=build /etc/apt/sources.list.d/pmjdebruijn-ubuntu-darktable-release-bionic.list /etc/apt/sources.list.d/pmjdebruijn-ubuntu-darktable-release-bionic.list
COPY --from=build /etc/apt/trusted.gpg.d/pmjdebruijn_ubuntu_darktable-release.gpg /etc/apt/trusted.gpg.d/pmjdebruijn_ubuntu_darktable-release.gpg
COPY --from=build /usr/local/bin/photoprism /usr/local/bin/photoprism
COPY --from=build /etc/photoprism/photoprism.yml /etc/photoprism/photoprism.yml
COPY --from=build /usr/local/lib/libtensorflow.so /usr/local/lib/libtensorflow.so
COPY --from=build /usr/local/lib/libtensorflow_framework.so /usr/local/lib/libtensorflow_framework.so
COPY --from=build /srv/photoprism /srv/photoprism
# Configure dynamic linker run-time bindings
RUN ldconfig
# Install darktable (RAW to JPEG converter)
RUN apt-get update && \
apt-get install -y --no-install-recommends darktable && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Show photoprism version
RUN photoprism -v
# Expose HTTP port
EXPOSE 80
# Start PhotoPrism server
CMD photoprism start

View file

@ -0,0 +1,41 @@
FROM photoprism/development:20181107
# Install Python and TensorFlow
RUN apt-get update && apt-get install -y --no-install-recommends \
python3 \
python-setuptools \
python3-dev \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
python get-pip.py && \
rm get-pip.py
RUN pip --no-cache-dir install --upgrade \
tensorflow \
requests \
Pillow \
h5py \
ipykernel \
jupyter \
keras_applications \
keras_preprocessing \
matplotlib \
numpy \
pandas \
scipy \
sklearn \
&& \
python -m ipykernel.kernelspec
# Set up project directory
WORKDIR "/go/src/github.com/photoprism/photoprism"
COPY . .
# Build PhotoPrism
RUN make all install
# Start PhotoPrism server
CMD photoprism start

View file

@ -23,31 +23,31 @@ var GlobalFlags = []cli.Flag{
cli.StringFlag{
Name: "originals-path",
Usage: "originals `PATH`",
Value: "/var/photoprism/photos/originals",
Value: "/srv/photoprism/photos/originals",
EnvVar: "PHOTOPRISM_ORIGINALS_PATH",
},
cli.StringFlag{
Name: "thumbnails-path",
Usage: "thumbnails `PATH`",
Value: "/var/photoprism/photos/thumbnails",
Value: "/srv/photoprism/photos/thumbnails",
EnvVar: "PHOTOPRISM_THUMBNAILS_PATH",
},
cli.StringFlag{
Name: "import-path",
Usage: "import `PATH`",
Value: "/var/photoprism/photos/import",
Value: "/srv/photoprism/photos/import",
EnvVar: "PHOTOPRISM_IMPORT_PATH",
},
cli.StringFlag{
Name: "export-path",
Usage: "export `PATH`",
Value: "/var/photoprism/photos/export",
Value: "/srv/photoprism/photos/export",
EnvVar: "PHOTOPRISM_EXPORT_PATH",
},
cli.StringFlag{
Name: "assets-path",
Usage: "assets `PATH`",
Value: "/var/photoprism",
Value: "/srv/photoprism",
EnvVar: "PHOTOPRISM_ASSETS_PATH",
},
cli.StringFlag{

View file

@ -19,6 +19,9 @@ import (
// Config provides a struct in which application configuration is stored.
type Config struct {
AppName string
AppVersion string
Copyright string
Debug bool
ConfigFile string
ServerIP string
@ -43,6 +46,9 @@ type configValues map[string]interface{}
// any previous values giving an option two override file configs through the CLI.
func NewConfig(context *cli.Context) *Config {
c := &Config{}
c.AppName = context.App.Name
c.Copyright = context.App.Copyright
c.AppVersion = context.App.Version
c.SetValuesFromFile(GetExpandedFilename(context.GlobalString("config-file")))
c.SetValuesFromCliContext(context)
@ -280,12 +286,13 @@ func (c *Config) GetClientConfig() map[string]interface{} {
cssHash := fileHash(c.GetPublicBuildPath() + "/app.css")
result := configValues{
"title": "PhotoPrism",
"debug": c.Debug,
"cameras": cameras,
"countries": countries,
"jsHash": jsHash,
"cssHash": cssHash,
"appName": c.AppName,
"appVersion": c.AppVersion,
"debug": c.Debug,
"cameras": cameras,
"countries": countries,
"jsHash": jsHash,
"cssHash": cssHash,
}
return result

View file

@ -96,7 +96,9 @@ func getTestCliContext() *cli.Context {
globalSet.String("originals-path", originalsPath, "doc")
globalSet.String("darktable-cli", darktableCli, "doc")
c := cli.NewContext(nil, globalSet, nil)
app := cli.NewApp()
c := cli.NewContext(app, globalSet, nil)
c.Set("config-file", testConfigFile)
c.Set("assets-path", assetsPath)
@ -125,11 +127,11 @@ func TestConfig_SetValuesFromFile(t *testing.T) {
c.SetValuesFromFile(GetExpandedFilename(testConfigFile))
assert.Equal(t, "/var/photoprism", c.AssetsPath)
assert.Equal(t, "/var/photoprism/thumbnails", c.ThumbnailsPath)
assert.Equal(t, "/var/photoprism/photos/originals", c.OriginalsPath)
assert.Equal(t, "/var/photoprism/photos/import", c.ImportPath)
assert.Equal(t, "/var/photoprism/photos/export", c.ExportPath)
assert.Equal(t, "/srv/photoprism", c.AssetsPath)
assert.Equal(t, "/srv/photoprism/thumbnails", c.ThumbnailsPath)
assert.Equal(t, "/srv/photoprism/photos/originals", c.OriginalsPath)
assert.Equal(t, "/srv/photoprism/photos/import", c.ImportPath)
assert.Equal(t, "/srv/photoprism/photos/export", c.ExportPath)
assert.Equal(t, databaseDriver, c.DatabaseDriver)
assert.Equal(t, databaseDsn, c.DatabaseDsn)
}

14
scripts/build.sh Executable file
View file

@ -0,0 +1,14 @@
#!/usr/bin/env bash
VERSION=`date -u +0.%Y%m%d.%H%M%S`
BRANCH=`git rev-parse --abbrev-ref HEAD`
if [ ${BRANCH} == "master" ]; then
echo "Building production binary..."
go build -ldflags "-s -w -X main.version=${VERSION}" cmd/photoprism/photoprism.go
echo "Done."
else
echo "Building development binary..."
go build -ldflags "-X main.version=${VERSION}-${BRANCH}" cmd/photoprism/photoprism.go
echo "Done."
fi

10
scripts/docker-build.sh Executable file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
if [ -z "$1" ] || [ -z "$2" ]; then
echo "Please provide a container image name and version" 1>&2
exit 1
else
echo "Building 'photoprism/$1:$2'...";
docker build -t photoprism/$1:latest -t photoprism/$1:$2 -f docker/$1/Dockerfile .
echo "Done"
fi

View file

@ -1,4 +1,19 @@
#!/usr/bin/env bash
if [ -z "$DOCKER_PASSWORD" ] || [ -z "$DOCKER_USERNAME" ]; then
echo "DOCKER_PASSWORD and DOCKER_USERNAME not set in your environment";
exit 1
fi
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
docker push photoprism/photoprism
if [ -z "$1" ] || [ -z "$2" ]; then
echo "Please provide a container image name and version" 1>&2
exit 1
else
echo "Pushing 'photoprism/$1:$2' to Docker hub...";
docker push photoprism/$1:latest
docker push photoprism/$1:$2
echo "Done"
fi