luisgulo/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use hash to keep TensorFlow model up to date

Browse source
This commit is contained in:
Michael Mayer 2019-05-04 00:53:02 +02:00
parent 3589a410c2
commit 3389b5115e
11 changed files with 70 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -5,6 +5,7 @@
/frontend/node_modules/*
/assets/server/public/build/*
/assets/testdata
/assets/backups
/assets/tensorflow
Dockerfile
/photoprism

1
.gitignore vendored
View file

@ -6,6 +6,7 @@
/frontend/node_modules/*
/frontend/tests/result.html
/assets/testdata
/assets/backups
/assets/tensorflow
*.log

2
.travis.yml
View file

@ -21,7 +21,7 @@ after_script:
deploy:
provider: script
script: make deploy-photoprism deploy-demo
script: make docker-photoprism docker-demo
skip_cleanup: true
on:
branch: master

2
Dockerfile
View file

@ -1,4 +1,4 @@
FROM photoprism/development:20190501
FROM photoprism/development:20190503
# Set up project directory
WORKDIR "/go/src/github.com/photoprism/photoprism"

24
Makefile
View file

@ -42,24 +42,26 @@ test-coverage:
go tool cover -html=coverage.txt -o coverage.html
clean:
rm -f $(BINARY_NAME)
rm -rf assets/testdata
rm -rf assets/backups
download:
scripts/download-assets.sh
deploy-photoprism:
scripts/docker-build.sh photoprism $(DOCKER_TAG)
scripts/docker-push.sh photoprism $(DOCKER_TAG)
deploy-demo:
scripts/docker-build.sh demo $(DOCKER_TAG)
scripts/docker-push.sh demo $(DOCKER_TAG)
deploy-development:
scripts/download-nasnet.sh
docker-development:
scripts/docker-build.sh development $(DOCKER_TAG)
scripts/docker-push.sh development $(DOCKER_TAG)
deploy-tensorflow:
docker-photoprism:
scripts/docker-build.sh photoprism $(DOCKER_TAG)
scripts/docker-push.sh photoprism $(DOCKER_TAG)
docker-demo:
scripts/docker-build.sh demo $(DOCKER_TAG)
scripts/docker-push.sh demo $(DOCKER_TAG)
docker-tensorflow:
scripts/docker-build.sh tensorflow $(DOCKER_TAG)
scripts/docker-push.sh tensorflow $(DOCKER_TAG)
deploy-darktable:
docker-darktable:
scripts/docker-build.sh darktable $(DARKTABLE_VERSION)
scripts/docker-push.sh darktable $(DARKTABLE_VERSION)
deploy-tidb:
docker-tidb:
scripts/docker-build.sh tidb $(TIDB_VERSION)
scripts/docker-push.sh tidb $(TIDB_VERSION)
fmt:

6
docker/development/Dockerfile
View file

@ -84,10 +84,12 @@ ENV PATH $GOBIN:/usr/local/go/bin:$PATH
ENV GO111MODULE on
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
ENV BUILD_DATE `date -u +%Y%m%d`
# Download TensorFlow model and test files
RUN rm -rf /tmp/* && mkdir -p /tmp/photoprism
RUN wget "https://dl.photoprism.org/tensorflow/nasnet-20190501.zip" -O /tmp/photoprism/nasnet.zip
RUN wget "https://dl.photoprism.org/fixtures/testdata-20190501.zip" -O /tmp/photoprism/testdata.zip
RUN wget "https://dl.photoprism.org/tensorflow/nasnet.zip?$BUILD_DATE" -O /tmp/photoprism/nasnet.zip
RUN wget "https://dl.photoprism.org/fixtures/testdata.zip?$BUILD_DATE" -O /tmp/photoprism/testdata.zip
# Install goimports
RUN env GO111MODULE=off /usr/local/go/bin/go get golang.org/x/tools/cmd/goimports

2
docker/photoprism/Dockerfile
View file

@ -1,4 +1,4 @@
FROM photoprism/development:20190501 as build
FROM photoprism/development:20190503 as build
# Set up project directory
WORKDIR "/go/src/github.com/photoprism/photoprism"

2
docker/tensorflow/Dockerfile
View file

@ -1,4 +1,4 @@
FROM photoprism/development:20190418
FROM photoprism/development:20190503
# Install Python and TensorFlow
RUN apt-get update && apt-get install -y --no-install-recommends \

2
internal/context/test_context.go
View file

@ -16,7 +16,7 @@ import (
const (
TestDataZip = "/tmp/photoprism/testdata.zip"
TestDataURL = "https://dl.photoprism.org/fixtures/testdata-20190501.zip"
TestDataURL = "https://dl.photoprism.org/fixtures/testdata.zip"
TestDataHash = "1a59b358b80221ab3e76efb683ad72402f0b0844"
)

15
scripts/download-assets.sh
View file

@ -1,15 +0,0 @@
#!/usr/bin/env bash
FILENAME="/tmp/photoprism/nasnet.zip"
if [[ ! -e assets/tensorflow/nasnet/saved_model.pb ]]; then
if [[ ! -e ${FILENAME} ]]; then
mkdir -p /tmp/photoprism
wget "https://dl.photoprism.org/tensorflow/nasnet.zip" -O ${FILENAME}
fi
mkdir -p assets/tensorflow
unzip ${FILENAME} -d assets/tensorflow
else
echo "TensorFlow model already downloaded."
fi

45
scripts/download-nasnet.sh Executable file
View file

@ -0,0 +1,45 @@
#!/usr/bin/env bash
TODAY=`date -u +%Y%m%d`
MODEL_NAME="NASNet Mobile"
MODEL_URL="https://dl.photoprism.org/tensorflow/nasnet.zip?$TODAY"
MODEL_PATH="assets/tensorflow/nasnet"
MODEL_ZIP="/tmp/photoprism/nasnet.zip"
MODEL_HASH="6a9450f89afa56b4539c0d7188f108f083c10fc9 $MODEL_ZIP"
MODEL_VERSION="$MODEL_PATH/version.txt"
MODEL_BACKUP="assets/backups/nasnet-$TODAY"
echo "Installing $MODEL_NAME for TensorFlow..."
# Check for update
if [[ -f ${MODEL_ZIP} ]] && [[ `sha1sum ${MODEL_ZIP}` == ${MODEL_HASH} ]]; then
echo "Already up to date."
exit
fi
# Create directories
mkdir -p /tmp/photoprism
mkdir -p assets/tensorflow
mkdir -p assets/backups
# Download model
echo "Downloading latest model from $MODEL_URL..."
wget ${MODEL_URL} -O ${MODEL_ZIP}
TMP_HASH=`sha1sum ${MODEL_ZIP}`
echo ${TMP_HASH}
# Create backup
if [[ -e ${MODEL_PATH} ]]; then
echo "Creating backup of existing directory: $MODEL_BACKUP"
rm -rf ${MODEL_BACKUP}
mv ${MODEL_PATH} ${MODEL_BACKUP}
fi
# Unzip model
unzip ${MODEL_ZIP} -d assets/tensorflow
echo "$MODEL_NAME $TODAY $MODEL_HASH" > ${MODEL_VERSION}
echo "Latest $MODEL_NAME installed."