Initial proof-of-concept

2018-02-04 17:34:07 +01:00 · 2018-02-04 17:34:07 +01:00 · c8b7dbbe01
commit c8b7dbbe01
parent ae91906416
26 changed files with 1433 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -12,3 +12,26 @@

 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
+
+# Compiled, config and log files
+*.log
+config.yml
+/tweethog
+
+# Generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+.heartbeat
+.idea
+*~
+.goutputstream*
+.c9revisions
+.settings
+.swp
+.tmp
+/vendor/
--- a/.travis.yml
+++ b/.travis.yml
@ -0,0 +1,4 @@
+language: go
+
+go:
+    - 1.9
--- a/Gopkg.lock
+++ b/Gopkg.lock
@ -0,0 +1,159 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  name = "github.com/bamiaux/rez"
+  packages = ["."]
+  revision = "29f4463c688b986c11f166b12734f69b58b5555f"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/brett-lempereur/ish"
+  packages = ["."]
+  revision = "bbdc45bcf55de61b38b4108871199a117aecd1be"
+
+[[projects]]
+  name = "github.com/cenkalti/backoff"
+  packages = ["."]
+  revision = "61153c768f31ee5f130071d08fc82b85208528de"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/ddliu/go-httpclient"
+  packages = ["."]
+  revision = "52a7afc73c57c5b898b5514a5467f8d38decd3ed"
+  version = "v0.5.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/dghubble/go-twitter"
+  packages = ["twitter"]
+  revision = "c4115fa44a928413e0b857e0eb47376ffde3a61a"
+
+[[projects]]
+  name = "github.com/dghubble/oauth1"
+  packages = ["."]
+  revision = "70562a5920ad9b6ff03ef697c0f90ae569abbd2b"
+  version = "v0.4.0"
+
+[[projects]]
+  name = "github.com/dghubble/sling"
+  packages = ["."]
+  revision = "eb56e89ac5088bebb12eef3cb4b293300f43608b"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/disintegration/imaging"
+  packages = ["."]
+  revision = "1884593a19ddc6f2ea050403430d02c1d0fc1283"
+  version = "v1.3.0"
+
+[[projects]]
+  name = "github.com/djherbis/times"
+  packages = ["."]
+  revision = "95292e44976d1217cf3611dc7c8d9466877d3ed5"
+  version = "v1.0.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/google/go-querystring"
+  packages = ["query"]
+  revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a"
+
+[[projects]]
+  name = "github.com/julienschmidt/httprouter"
+  packages = ["."]
+  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
+  version = "v1.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kylelemons/go-gypsy"
+  packages = ["yaml"]
+  revision = "08cad365cd28a7fba23bb1e57aa43c5e18ad8bb8"
+
+[[projects]]
+  name = "github.com/lastzero/tweethog"
+  packages = ["."]
+  revision = "ac3ce5feaebcb1320109e02d5e477d0a97711793"
+  version = "v0.7.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mailru/easyjson"
+  packages = [".","buffer","jlexer","jwriter"]
+  revision = "32fa128f234d041f196a9f3e0fea5ac9772c08e1"
+
+[[projects]]
+  name = "github.com/olivere/elastic"
+  packages = ["config","uritemplates"]
+  revision = "c51e74f9bcab8906a2f6cf5660dac396ba51b3d6"
+  version = "v6.1.4"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  branch = "go1"
+  name = "github.com/rwcarlsen/goexif"
+  packages = ["exif","mknote","tiff"]
+  revision = "17202558c8d9c3fd047859f1a5e73fd9ae709187"
+
+[[projects]]
+  name = "github.com/steakknife/hamming"
+  packages = ["."]
+  revision = "5ac3f73b8842df21423978fbbeb5166670f6f73e"
+  version = "0.2.3"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/subosito/shorturl"
+  packages = [".","adfly","base","bitly","catchy","cligs","gggg","gitio","googl","isgd","moourl","pendekin","shorl","snipurl","tinyurl","vamu"]
+  revision = "3dc4cee684914f665399d6a5cddc13b1864b36dd"
+
+[[projects]]
+  name = "github.com/tensorflow/tensorflow"
+  packages = ["tensorflow/go","tensorflow/go/op"]
+  revision = "37aa430d84ced579342a4044c89c236664be7f68"
+  version = "v1.5.0"
+
+[[projects]]
+  name = "github.com/urfave/cli"
+  packages = ["."]
+  revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1"
+  version = "v1.20.0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/image"
+  packages = ["bmp","riff","tiff","tiff/lzw","vp8","vp8l","webp"]
+  revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/net"
+  packages = ["context","html","html/atom"]
+  revision = "309822c5b9b9f80db67f016069a12628d94fad34"
+
+[[projects]]
+  name = "gopkg.in/olivere/elastic.v6"
+  packages = ["."]
+  revision = "c51e74f9bcab8906a2f6cf5660dac396ba51b3d6"
+  version = "v6.1.4"
+
+[[projects]]
+  name = "mvdan.cc/xurls"
+  packages = ["."]
+  revision = "d315b61cf6727664f310fa87b3197e9faf2a8513"
+  version = "v1.1.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "70fdd0684ae209f6735c04d5ecffa4c3ad3dc102214afc431b79c530fb2d1d65"
+  solver-name = "gps-cdcl"
+  solver-version = 1
--- a/Gopkg.toml
+++ b/Gopkg.toml
@ -0,0 +1,30 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/kylelemons/go-gypsy"
+
+[[constraint]]
+  name = "github.com/urfave/cli"
+  version = "1.20.0"
--- a/browse.go
+++ b/browse.go
@ -0,0 +1 @@
+package photoprism
--- a/cmd/photoprism/photoprism.go
+++ b/cmd/photoprism/photoprism.go
@ -0,0 +1,92 @@
+package main
+
+import (
+	"github.com/photoprism/photoprism"
+	"github.com/urfave/cli"
+	"os"
+	"fmt"
+	"gopkg.in/olivere/elastic.v6"
+	"github.com/lastzero/tweethog"
+)
+
+func main() {
+	config := photoprism.NewConfig()
+
+	app := cli.NewApp()
+	app.Name = "PhotoPrism"
+	app.Usage = "Sort, view and archive photos on your local computer"
+	app.Version = "0.0.1"
+	app.Copyright = "Michael Mayer <michael@liquidbytes.net>"
+	app.Flags = globalCliFlags
+	app.Commands = []cli.Command{
+		{
+			Name:  "config",
+			Usage: "Displays global configuration values",
+			Action: func(c *cli.Context) error {
+				config.SetValuesFromFile(tweethog.GetExpandedFilename(c.GlobalString("config-file")))
+
+				config.SetValuesFromCliContext(c)
+
+				fmt.Printf("<name>                <value>\n")
+				fmt.Printf("config-file           %s\n", config.ConfigFile)
+				fmt.Printf("darktable-cli         %s\n", config.DarktableCli)
+				fmt.Printf("originals-path        %s\n", config.OriginalsPath)
+				fmt.Printf("thumbnails-path       %s\n", config.ThumbnailsPath)
+				fmt.Printf("import-path           %s\n", config.ImportPath)
+				fmt.Printf("export-path           %s\n", config.ExportPath)
+
+				return nil
+			},
+		},
+		{
+			Name:  "import",
+			Usage: "Imports photo from a directory",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "import-directory, d",
+					Usage: "Import directory",
+					Value: "~/Pictures/Import",
+				},
+			},
+			Action: func(c *cli.Context) error {
+				config.SetValuesFromFile(tweethog.GetExpandedFilename(c.GlobalString("config-file")))
+
+				config.SetValuesFromCliContext(c)
+
+				fmt.Println("Welcome to PhotoPrism")
+
+				client, err := elastic.NewClient(elastic.SetURL("http://localhost:9200"))
+
+				if err != nil {
+					fmt.Println("Problem with elasticsearch :-(")
+
+					return err
+				}
+
+				client.ClusterState()
+
+				return nil
+			},
+		},
+	}
+
+	app.Run(os.Args)
+}
+
+var globalCliFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:  "config-file, c",
+		Usage: "Config filename",
+		Value: "~/.photoprism",
+	},
+	cli.StringFlag{
+		Name:  "darktable-cli",
+		Usage: "Darktable CLI app",
+		Value: "/Applications/darktable.app/Contents/MacOS/darktable-cli",
+	},
+	cli.StringFlag{
+		Name:  "storage-path",
+		Usage: "Storage path",
+		Value: "~/Photos",
+	},
+}
--- a/cmd/tensorflowapi/Dockerfile
+++ b/cmd/tensorflowapi/Dockerfile
@ -0,0 +1,82 @@
+FROM tensorflow/tensorflow
+
+# Install TensorFlow C library
+RUN curl -L \
+   "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.3.0.tar.gz" | \
+   tar -C "/usr/local" -xz
+RUN ldconfig
+# Hide some warnings
+ENV TF_CPP_MIN_LOG_LEVEL 2
+
+# Install Go (https://github.com/docker-library/golang/blob/221ee92559f2963c1fe55646d3516f5b8f4c91a4/1.9/stretch/Dockerfile)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		g++ \
+		gcc \
+		libc6-dev \
+		make \
+		pkg-config \
+    wget \
+    git \
+	&& rm -rf /var/lib/apt/lists/*
+
+ENV GOLANG_VERSION 1.9.1
+RUN set -eux; \
+	\
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "${dpkgArch##*-}" in \
+		amd64) goRelArch='linux-amd64'; goRelSha256='07d81c6b6b4c2dcf1b5ef7c27aaebd3691cdb40548500941f92b221147c5d9c7' ;; \
+		armhf) goRelArch='linux-armv6l'; goRelSha256='65a0495a50c7c240a6487b1170939586332f6c8f3526abdbb9140935b3cff14c' ;; \
+		arm64) goRelArch='linux-arm64'; goRelSha256='d31ecae36efea5197af271ccce86ccc2baf10d2e04f20d0fb75556ecf0614dad' ;; \
+		i386) goRelArch='linux-386'; goRelSha256='2cea1ce9325cb40839601b566bc02b11c92b2942c21110b1b254c7e72e5581e7' ;; \
+		ppc64el) goRelArch='linux-ppc64le'; goRelSha256='de57b6439ce9d4dd8b528599317a35fa1e09d6aa93b0a80e3945018658d963b8' ;; \
+		s390x) goRelArch='linux-s390x'; goRelSha256='9adf03574549db82a72e0d721ef2178ec5e51d1ce4f309b271a2bca4dcf206f6' ;; \
+		*) goRelArch='src'; goRelSha256='a84afc9dc7d64fe0fa84d4d735e2ece23831a22117b50dafc75c1484f1cb550e'; \
+			echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \
+	esac; \
+	\
+	url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \
+	wget -O go.tgz "$url"; \
+	echo "${goRelSha256} *go.tgz" | sha256sum -c -; \
+	tar -C /usr/local -xzf go.tgz; \
+	rm go.tgz; \
+	\
+	if [ "$goRelArch" = 'src' ]; then \
+		echo >&2; \
+		echo >&2 'error: UNIMPLEMENTED'; \
+		echo >&2 'TODO install golang-any from jessie-backports for GOROOT_BOOTSTRAP (and uninstall after build)'; \
+		echo >&2; \
+		exit 1; \
+	fi; \
+	\
+	export PATH="/usr/local/go/bin:$PATH"; \
+	go version
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Install dependencies
+RUN go get github.com/tensorflow/tensorflow/tensorflow/go \
+  github.com/tensorflow/tensorflow/tensorflow/go/op \
+  github.com/julienschmidt/httprouter
+
+# Download InceptionV3 model
+RUN mkdir -p /model && \
+  wget "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip" -O /model/inception.zip && \
+  unzip /model/inception.zip -d /model && \
+  chmod -R 777 /model
+
+# Create user
+RUN adduser --disabled-password --gecos '' api
+USER api
+
+# Set up project directory
+WORKDIR "/go/src/tensorflowapi"
+COPY . .
+
+# Install the app
+RUN go install -v ./...
+
+# Run the app
+CMD [ "tensorflowapi" ]
--- a/cmd/tensorflowapi/image_tensor.go
+++ b/cmd/tensorflowapi/image_tensor.go
@ -0,0 +1,65 @@
+package main
+
+import (
+	"bytes"
+
+	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+	"github.com/tensorflow/tensorflow/tensorflow/go/op"
+)
+
+func makeTensorFromImage(imageBuffer *bytes.Buffer, imageFormat string) (*tf.Tensor, error) {
+	tensor, err := tf.NewTensor(imageBuffer.String())
+	if err != nil {
+		return nil, err
+	}
+	graph, input, output, err := makeTransformImageGraph(imageFormat)
+	if err != nil {
+		return nil, err
+	}
+	session, err := tf.NewSession(graph, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+	normalized, err := session.Run(
+		map[tf.Output]*tf.Tensor{input: tensor},
+		[]tf.Output{output},
+		nil)
+	if err != nil {
+		return nil, err
+	}
+	return normalized[0], nil
+}
+
+// Creates a graph to decode, rezise and normalize an image
+func makeTransformImageGraph(imageFormat string) (graph *tf.Graph, input, output tf.Output, err error) {
+	const (
+		H, W  = 224, 224
+		Mean  = float32(117)
+		Scale = float32(1)
+	)
+	s := op.NewScope()
+	input = op.Placeholder(s, tf.String)
+	// Decode PNG or JPEG
+	var decode tf.Output
+	if imageFormat == "png" {
+		decode = op.DecodePng(s, input, op.DecodePngChannels(3))
+	} else {
+		decode = op.DecodeJpeg(s, input, op.DecodeJpegChannels(3))
+	}
+	// Div and Sub perform (value-Mean)/Scale for each pixel
+	output = op.Div(s,
+		op.Sub(s,
+			// Resize to 224x224 with bilinear interpolation
+			op.ResizeBilinear(s,
+				// Create a batch containing a single image
+				op.ExpandDims(s,
+					// Use decoded pixel values
+					op.Cast(s, decode, tf.Float),
+					op.Const(s.SubScope("make_batch"), int32(0))),
+				op.Const(s.SubScope("size"), []int32{H, W})),
+			op.Const(s.SubScope("mean"), Mean)),
+		op.Const(s.SubScope("scale"), Scale))
+	graph, err = s.Finalize()
+	return graph, input, output, err
+}
--- a/cmd/tensorflowapi/tensorflowapi.go
+++ b/cmd/tensorflowapi/tensorflowapi.go
@ -0,0 +1,138 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+)
+
+type ClassifyResult struct {
+	Filename string        `json:"filename"`
+	Labels   []LabelResult `json:"labels"`
+}
+
+type LabelResult struct {
+	Label       string  `json:"label"`
+	Probability float32 `json:"probability"`
+}
+
+var (
+	graph  *tf.Graph
+	labels []string
+)
+
+func main() {
+	if err := loadModel(); err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	r := httprouter.New()
+	r.POST("/recognize", recognizeHandler)
+	log.Fatal(http.ListenAndServe(":8080", r))
+}
+
+func loadModel() error {
+	// Load inception model
+	model, err := ioutil.ReadFile("/model/tensorflow_inception_graph.pb")
+	if err != nil {
+		return err
+	}
+	graph = tf.NewGraph()
+	if err := graph.Import(model, ""); err != nil {
+		return err
+	}
+	// Load labels
+	labelsFile, err := os.Open("/model/imagenet_comp_graph_label_strings.txt")
+	if err != nil {
+		return err
+	}
+	defer labelsFile.Close()
+	scanner := bufio.NewScanner(labelsFile)
+	// Labels are separated by newlines
+	for scanner.Scan() {
+		labels = append(labels, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func recognizeHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	// Read image
+	imageFile, header, err := r.FormFile("image")
+	// Will contain filename and extension
+	imageName := strings.Split(header.Filename, ".")
+	if err != nil {
+		responseError(w, "Could not read image", http.StatusBadRequest)
+		return
+	}
+	defer imageFile.Close()
+	var imageBuffer bytes.Buffer
+	// Copy image data to a buffer
+	io.Copy(&imageBuffer, imageFile)
+
+	// ...
+	// Make tensor
+	tensor, err := makeTensorFromImage(&imageBuffer, imageName[:1][0])
+	if err != nil {
+		responseError(w, "Invalid image", http.StatusBadRequest)
+		return
+	}
+
+	// Run inference
+	session, err := tf.NewSession(graph, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer session.Close()
+	output, err := session.Run(
+		map[tf.Output]*tf.Tensor{
+			graph.Operation("input").Output(0): tensor,
+		},
+		[]tf.Output{
+			graph.Operation("output").Output(0),
+		},
+		nil)
+	if err != nil {
+		responseError(w, "Could not run inference", http.StatusInternalServerError)
+		return
+	}
+
+	// Return best labels
+	responseJSON(w, ClassifyResult{
+		Filename: header.Filename,
+		Labels:   findBestLabels(output[0].Value().([][]float32)[0]),
+	})
+}
+
+type ByProbability []LabelResult
+
+func (a ByProbability) Len() int           { return len(a) }
+func (a ByProbability) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByProbability) Less(i, j int) bool { return a[i].Probability > a[j].Probability }
+
+func findBestLabels(probabilities []float32) []LabelResult {
+	// Make a list of label/probability pairs
+	var resultLabels []LabelResult
+	for i, p := range probabilities {
+		if i >= len(labels) {
+			break
+		}
+		resultLabels = append(resultLabels, LabelResult{Label: labels[i], Probability: p})
+	}
+	// Sort by probability
+	sort.Sort(ByProbability(resultLabels))
+	// Return top 5 labels
+	return resultLabels[:5]
+}
--- a/cmd/tensorflowapi/utilities.go
+++ b/cmd/tensorflowapi/utilities.go
@ -0,0 +1,17 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+func responseError(w http.ResponseWriter, message string, code int) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	json.NewEncoder(w).Encode(map[string]string{"error": message})
+}
+
+func responseJSON(w http.ResponseWriter, data interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
--- a/config.example.yml
+++ b/config.example.yml
@ -0,0 +1,5 @@
+darktable-cli: /Applications/darktable.app/Contents/MacOS/darktable-cli
+originals-path: photos/originals
+thumbnails-path: photos/thumbnails
+import-path: photos/import
+export-path: photos/export
--- a/config.go
+++ b/config.go
@ -0,0 +1,75 @@
+package photoprism
+
+import (
+	"github.com/kylelemons/go-gypsy/yaml"
+	"github.com/urfave/cli"
+)
+
+type Config struct {
+	ConfigFile     string
+	DarktableCli   string
+	OriginalsPath  string
+	ThumbnailsPath string
+	ImportPath     string
+	ExportPath     string
+}
+
+func NewConfig() *Config {
+	return &Config{}
+}
+
+func (config *Config) SetValuesFromFile(fileName string) error {
+	yamlConfig, err := yaml.ReadFile(fileName)
+
+	if err != nil {
+		return err
+	}
+
+	config.ConfigFile = fileName
+
+	if OriginalsPath, err := yamlConfig.Get("originals-path"); err == nil {
+		config.OriginalsPath = GetExpandedFilename(OriginalsPath)
+	}
+
+	if ThumbnailsPath, err := yamlConfig.Get("thumbnails-path"); err == nil {
+		config.ThumbnailsPath = GetExpandedFilename(ThumbnailsPath)
+	}
+
+	if ImportPath, err := yamlConfig.Get("import-path"); err == nil {
+		config.ImportPath = GetExpandedFilename(ImportPath)
+	}
+
+	if ExportPath, err := yamlConfig.Get("export-path"); err == nil {
+		config.ExportPath = GetExpandedFilename(ExportPath)
+	}
+
+	if DarktableCli, err := yamlConfig.Get("darktable-cli"); err == nil {
+		config.DarktableCli = GetExpandedFilename(DarktableCli)
+	}
+
+	return nil
+}
+
+func (config *Config) SetValuesFromCliContext(c *cli.Context) error {
+	if c.IsSet("originals-path") {
+		config.OriginalsPath = GetExpandedFilename(c.String("originals-path"))
+	}
+
+	if c.IsSet("thumbnails-path") {
+		config.ThumbnailsPath = GetExpandedFilename(c.String("thumbnails-path"))
+	}
+
+	if c.IsSet("import-path") {
+		config.ImportPath = GetExpandedFilename(c.String("import-path"))
+	}
+
+	if c.IsSet("export-path") {
+		config.ExportPath = GetExpandedFilename(c.String("export-path"))
+	}
+
+	if c.IsSet("darktable-cli") {
+		config.DarktableCli = GetExpandedFilename(c.String("darktable-cli"))
+	}
+
+	return nil
+}
--- a/config_test.go
+++ b/config_test.go
@ -0,0 +1,11 @@
+package photoprism
+
+func NewTestConfig() *Config {
+	return &Config{
+		DarktableCli:   "/Applications/darktable.app/Contents/MacOS/darktable-cli",
+		OriginalsPath:  GetExpandedFilename("photos/originals"),
+		ThumbnailsPath: GetExpandedFilename("photos/thumbnails"),
+		ImportPath:     GetExpandedFilename("photos/import"),
+		ExportPath:     GetExpandedFilename("photos/export"),
+	}
+}
--- a/converter.go
+++ b/converter.go
@ -0,0 +1,53 @@
+package photoprism
+
+import (
+	"os"
+	"os/exec"
+	"log"
+)
+
+type Converter struct {
+	darktableCli string
+}
+
+func NewConverter(darktableCli string) *Converter {
+	if stat, err := os.Stat(darktableCli); err != nil {
+		log.Print("Darktable CLI binary could not be found at " + darktableCli)
+	} else if stat.IsDir() {
+		log.Print("Darktable CLI must be a file, not a directory")
+	}
+
+	return &Converter{darktableCli: darktableCli}
+}
+
+func (converter *Converter) ConvertToJpeg(image *MediaFile) (*MediaFile, error) {
+	if image.IsJpeg() {
+		return image, nil
+	}
+
+	extension := image.GetExtension()
+
+	baseFilename := image.filename[0:len(image.filename)-len(extension)]
+
+	jpegFilename := baseFilename + ".jpg"
+
+	if _, err := os.Stat(jpegFilename); err == nil {
+		return NewMediaFile(jpegFilename), nil
+	}
+
+	xmpFilename := baseFilename + ".xmp"
+
+	var convertCommand *exec.Cmd
+
+	if _, err := os.Stat(xmpFilename); err == nil {
+		convertCommand = exec.Command(converter.darktableCli, image.filename, xmpFilename, jpegFilename)
+	} else {
+		convertCommand = exec.Command(converter.darktableCli, image.filename, jpegFilename)
+	}
+
+	if err := convertCommand.Run(); err != nil {
+		return nil, err
+	}
+
+	return NewMediaFile(jpegFilename), nil
+}
--- a/converter_test.go
+++ b/converter_test.go
@ -0,0 +1,10 @@
+package photoprism
+
+import (
+	"testing"
+
+	)
+
+func TestNewConverter(t *testing.T) {
+	NewConverter("storage")
+}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,25 @@
+version: '3.3'
+
+services:
+  tensorflowapi:
+    build: './cmd/tensorflowapi'
+    ports:
+      - 8080:8080
+    volumes:
+      - './cmd/tensorflowapi:/go/src/tensorflowapi'
+  database:
+    image: mysql:latest
+    command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=1024
+    ports:
+      - 3306:3306
+    volumes:
+      - database-data:/var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: photoprism
+      MYSQL_USER: photoprism
+      MYSQL_PASSWORD: photoprism
+      MYSQL_DATABASE: photoprism
+
+volumes:
+  database-data:
+    driver: local
--- a/importer.go
+++ b/importer.go
@ -0,0 +1,135 @@
+package photoprism
+
+import (
+	"path/filepath"
+	"os"
+	"log"
+	"fmt"
+	"github.com/pkg/errors"
+	"bytes"
+	"path"
+)
+
+type Importer struct {
+	originalsPath string
+	converter     *Converter
+}
+
+func NewImporter(originalsPath string, converter *Converter) *Importer {
+	instance := &Importer{
+		originalsPath: originalsPath,
+		converter:     converter,
+	}
+
+	return instance
+}
+
+func (importer *Importer) CreateJpegFromRaw(sourcePath string) {
+	err := filepath.Walk(sourcePath, func(filename string, fileInfo os.FileInfo, err error) error {
+
+		if err != nil {
+			log.Print(err.Error())
+			return nil
+		}
+
+		if fileInfo.IsDir() {
+			return nil
+		}
+
+		mediaFile := NewMediaFile(filename)
+
+		if !mediaFile.Exists() || !mediaFile.IsRaw() {
+			return nil
+		}
+
+		log.Printf("Converting %s \n", filename)
+
+		if _, err := importer.converter.ConvertToJpeg(mediaFile); err != nil {
+			log.Print(err.Error())
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		log.Print(err.Error())
+	}
+}
+
+func (importer *Importer) ImportJpegFromDirectory(sourcePath string) {
+	err := filepath.Walk(sourcePath, func(filename string, fileInfo os.FileInfo, err error) error {
+
+		if err != nil {
+			log.Print(err.Error())
+			return nil
+		}
+
+		if fileInfo.IsDir() {
+			return nil
+		}
+
+		jpegFile := NewMediaFile(filename)
+
+		if !jpegFile.Exists() || !jpegFile.IsJpeg() {
+			return nil
+		}
+
+		log.Println(jpegFile.GetFilename() + " -> " + jpegFile.GetCanonicalName())
+
+		log.Println("Getting related files")
+
+		relatedFiles, _ := jpegFile.GetRelatedFiles()
+
+		for _, relatedMediaFile := range relatedFiles {
+			log.Println("Processing " + relatedMediaFile.GetFilename())
+			if destinationFilename, err := importer.GetDestinationFilename(jpegFile, relatedMediaFile); err == nil {
+				log.Println("Creating directories")
+				os.MkdirAll(path.Dir(destinationFilename), os.ModePerm)
+				log.Println("Moving file " + relatedMediaFile.GetFilename())
+				relatedMediaFile.Move(destinationFilename)
+				log.Println("Moved file to  " + destinationFilename)
+			} else {
+				log.Println("File already exists: " + relatedMediaFile.GetFilename() + " -> " + destinationFilename)
+			}
+		}
+
+		// mediaFile.Move(importer.originalsPath)
+
+		return nil
+	})
+
+	if err != nil {
+		log.Print(err.Error())
+	}
+}
+
+func (importer *Importer) GetDestinationFilename(jpegFile *MediaFile, mediaFile *MediaFile) (string, error) {
+	canonicalName := jpegFile.GetCanonicalName()
+	fileExtension := mediaFile.GetExtension()
+	dateCreated := jpegFile.GetDateCreated()
+
+	//	Mon Jan 2 15:04:05 -0700 MST 2006
+	path := importer.originalsPath + "/" + dateCreated.UTC().Format("2006/01")
+
+	i := 1
+
+	result := path + "/" + canonicalName + fileExtension
+
+	for FileExists(result) {
+		if bytes.Compare(mediaFile.GetHash(), Md5Sum(result)) == 0 {
+			return result, errors.New("File already exists")
+		}
+
+		i++
+		result = path + "/" + canonicalName + "_" + fmt.Sprintf("%02d", i) + fileExtension
+//		log.Println(result)
+	}
+
+	// os.MkdirAll(folderPath, os.ModePerm)
+
+	return result, nil
+}
+
+func (importer *Importer) MoveRelatedFiles() {
+
+}
--- a/importer_test.go
+++ b/importer_test.go
@ -0,0 +1,15 @@
+package photoprism
+
+import (
+	"testing"
+)
+
+func TestImporter_ImportFromDirectory(t *testing.T) {
+	config := NewTestConfig()
+
+	converter := NewConverter(config.DarktableCli)
+	importer := NewImporter(config.OriginalsPath, converter)
+
+	importer.CreateJpegFromRaw(config.ImportPath)
+	importer.ImportJpegFromDirectory(config.ImportPath)
+}
--- a/indexer.go
+++ b/indexer.go
@ -0,0 +1 @@
+package photoprism
--- a/mediafile.go
+++ b/mediafile.go
@ -0,0 +1,245 @@
+package photoprism
+
+import (
+	"path/filepath"
+	"encoding/hex"
+	"github.com/brett-lempereur/ish"
+	"net/http"
+	"os"
+	"log"
+	"strings"
+	"time"
+	"github.com/djherbis/times"
+	"fmt"
+)
+
+const (
+	FileTypeOther = ""
+	FileTypeYaml  = "yml"
+	FileTypeJpeg  = "jpg"
+	FileTypeRaw   = "raw"
+	FileTypeXmp   = "xmp"
+	FileTypeAae   = "aae"
+	FileTypeMovie = "mov"
+)
+
+const (
+	MimeTypeJpeg = "image/jpeg"
+)
+
+var FileExtensions = map[string]string {
+	".crw": FileTypeRaw,
+	".cr2": FileTypeRaw,
+	".nef": FileTypeRaw,
+	".arw": FileTypeRaw,
+	".dng": FileTypeRaw,
+	".mov": FileTypeMovie,
+	".avi": FileTypeMovie,
+	".yml": FileTypeYaml,
+	".jpg": FileTypeJpeg,
+	".jpeg": FileTypeJpeg,
+	".xmp": FileTypeXmp,
+	".aae": FileTypeAae,
+}
+
+type MediaFile struct {
+	filename string
+	dateCreated time.Time
+	hash     []byte
+	fileType string
+	mimeType string
+	tags     []string
+	exifData *ExifData
+}
+
+func NewMediaFile(filename string) *MediaFile {
+	instance := &MediaFile{filename: filename}
+
+	return instance
+}
+
+func (mediaFile *MediaFile) GetDateCreated() time.Time {
+	if !mediaFile.dateCreated.IsZero() {
+		return mediaFile.dateCreated
+	}
+
+	info, err := mediaFile.GetExifData()
+
+	if err == nil {
+		mediaFile.dateCreated = info.DateTime
+		return info.DateTime
+	}
+
+	t, err := times.Stat(mediaFile.GetFilename())
+
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	if t.HasBirthTime() {
+		mediaFile.dateCreated = t.BirthTime()
+		return t.BirthTime()
+	}
+
+	mediaFile.dateCreated = t.ModTime()
+	return t.ModTime()
+}
+
+func (mediaFile *MediaFile) GetCameraModel () string {
+	info, err := mediaFile.GetExifData()
+
+	var result string
+
+	if err == nil {
+		result = info.CameraModel
+	}
+
+	return result
+}
+
+func (mediaFile *MediaFile) GetCanonicalName() string {
+	dateCreated := mediaFile.GetDateCreated().UTC()
+	cameraModel := strings.Replace(mediaFile.GetCameraModel(), " ", "_", -1)
+
+	result := dateCreated.Format("20060102_150405_") + strings.ToUpper(mediaFile.GetHashString()[:8])
+
+	if cameraModel != "" {
+		result = result + "_" + cameraModel
+	}
+
+	return result
+}
+
+func (mediaFile *MediaFile) GetPerceptiveHash() (string, error) {
+	hasher := ish.NewDifferenceHash(8, 8)
+	img, _, err := ish.LoadFile(mediaFile.GetFilename())
+
+	if err != nil {
+		return "", err
+	}
+
+	dh, err := hasher.Hash(img)
+
+	if err != nil {
+		return "", err
+	}
+
+	dhs := hex.EncodeToString(dh)
+
+	return dhs, nil
+}
+
+func (mediaFile *MediaFile) GetHash() []byte  {
+	if len(mediaFile.hash) == 0 {
+		mediaFile.hash = Md5Sum(mediaFile.GetFilename())
+	}
+
+	return mediaFile.hash
+}
+
+func (mediaFile *MediaFile) GetHashString() string {
+	return fmt.Sprintf("%x", mediaFile.GetHash())
+}
+
+func (mediaFile *MediaFile) GetRelatedFiles() (result []*MediaFile, err error) {
+	extension := mediaFile.GetExtension()
+
+	baseFilename := mediaFile.filename[0:len(mediaFile.filename)-len(extension)]
+
+	matches, err := filepath.Glob(baseFilename + "*")
+
+	if err != nil {
+		return result, err
+	}
+
+	for _, filename := range matches {
+		result = append(result, NewMediaFile(filename))
+	}
+
+	return result, nil
+}
+
+
+func (mediaFile *MediaFile) GetFilename() string {
+	return mediaFile.filename
+}
+
+func (mediaFile *MediaFile) SetFilename(filename string)  {
+	mediaFile.filename = filename
+}
+
+func (mediaFile *MediaFile) GetMimeType() string {
+	if mediaFile.mimeType != "" {
+		return mediaFile.mimeType
+	}
+
+	handle, err := mediaFile.openFile()
+
+	if err != nil {
+		log.Println("Error: Could not open file to determine mime type")
+		return ""
+	}
+
+	defer handle.Close()
+
+	// Only the first 512 bytes are used to sniff the content type.
+	buffer := make([]byte, 512)
+
+	_, err = handle.Read(buffer)
+
+	if err != nil {
+		log.Println("Error: Could not read file to determine mime type: " + mediaFile.GetFilename())
+		return ""
+	}
+
+	mediaFile.mimeType = http.DetectContentType(buffer)
+
+	return mediaFile.mimeType
+}
+
+func (mediaFile *MediaFile) openFile() (*os.File, error) {
+	if handle, err := os.Open(mediaFile.filename); err == nil {
+		return handle, nil
+	} else {
+		log.Println(err.Error())
+		return nil, err
+	}
+}
+
+func (mediaFile *MediaFile) Exists() bool {
+	return FileExists(mediaFile.GetFilename())
+}
+
+func (mediaFile *MediaFile) Move(newFilename string) error {
+	if err := os.Rename(mediaFile.filename, newFilename); err != nil {
+		return err
+	}
+
+	mediaFile.filename = newFilename
+
+	return nil
+}
+
+func (mediaFile *MediaFile) GetExtension() string {
+	return strings.ToLower(filepath.Ext(mediaFile.filename))
+}
+
+func (mediaFile *MediaFile) IsJpeg() bool {
+	return mediaFile.GetMimeType() == MimeTypeJpeg
+}
+
+func (mediaFile *MediaFile) HasType(typeString string) bool {
+	if typeString == FileTypeJpeg {
+		return mediaFile.IsJpeg()
+	}
+
+	return FileExtensions[mediaFile.GetExtension()] == typeString
+}
+
+func (mediaFile *MediaFile) IsRaw() bool {
+	return mediaFile.HasType(FileTypeRaw)
+}
+
+func (mediaFile *MediaFile) IsPhoto() bool {
+	return mediaFile.IsJpeg() || mediaFile.IsRaw()
+}
--- a/mediafile_exif.go
+++ b/mediafile_exif.go
@ -0,0 +1,62 @@
+package photoprism
+
+import (
+	"github.com/rwcarlsen/goexif/exif"
+	"github.com/rwcarlsen/goexif/mknote"
+	"time"
+	"errors"
+	"strings"
+	"log"
+)
+
+type ExifData struct {
+	DateTime    time.Time
+	CameraModel string
+	Lat         float64
+	Long        float64
+}
+
+func (mediaFile *MediaFile) GetExifData() (*ExifData, error) {
+	if mediaFile.exifData != nil {
+		log.Printf("GetExifData() Cache Hit %s", mediaFile.filename)
+		return mediaFile.exifData, nil
+	}
+
+	log.Printf("GetExifData() Cache Miss %s", mediaFile.filename)
+
+	if !mediaFile.IsJpeg() {
+		// EXIF only works for JPEG
+		return nil, errors.New("MediaFile is not a JPEG")
+	}
+
+	mediaFile.exifData = &ExifData{}
+
+	log.Printf("GetExifData() Open File %s", mediaFile.filename)
+	file, err := mediaFile.openFile()
+
+	if err != nil {
+		return mediaFile.exifData, err
+	}
+
+	defer file.Close()
+
+	exif.RegisterParsers(mknote.All...)
+
+	x, err := exif.Decode(file)
+
+	if err != nil {
+		return mediaFile.exifData, err
+	}
+
+	camModel, _ := x.Get(exif.Model)
+	mediaFile.exifData.CameraModel = strings.Replace(camModel.String(), "\"", "", -1)
+
+	tm, _ := x.DateTime()
+	mediaFile.exifData.DateTime = tm
+
+	lat, long, _ := x.LatLong()
+	mediaFile.exifData.Lat = lat
+	mediaFile.exifData.Long = long
+
+	return mediaFile.exifData, nil
+}
--- a/mediafile_exif_test.go
+++ b/mediafile_exif_test.go
@ -0,0 +1,31 @@
+package photoprism
+
+import (
+	"testing"
+	"fmt"
+)
+
+func TestImage_GetExifData(t *testing.T) {
+	config := NewTestConfig()
+
+	converter := NewConverter(config.DarktableCli)
+
+	image1 := NewMediaFile("storage/import/IMG_9083.jpg")
+
+	info1, _ := image1.GetExifData()
+
+	fmt.Printf("%+v\n", info1)
+
+	image2,_ := converter.ConvertToJpeg(NewMediaFile("storage/import/IMG_5901.JPG"))
+
+	info2, _ := image2.GetExifData()
+
+	fmt.Printf("%+v\n", info2)
+
+	image3, _ := converter.ConvertToJpeg(NewMediaFile("storage/import/IMG_9087.CR2"))
+
+	info3, _ := image3.GetExifData()
+
+	fmt.Printf("%+v\n", info3)
+}
+
--- a/mediafile_test.go
+++ b/mediafile_test.go
@ -0,0 +1,62 @@
+package photoprism
+
+import (
+	"testing"
+	"fmt"
+)
+
+func TestMediaFile_ConvertToJpeg(t *testing.T) {
+	converterCommand := "/Applications/darktable.app/Contents/MacOS/darktable-cli"
+
+	converter := NewConverter(converterCommand)
+
+	image1,_ := converter.ConvertToJpeg(NewMediaFile("storage/import/IMG_5901.JPG"))
+
+	info1, _ := image1.GetExifData()
+
+	fmt.Printf("%+v\n", info1)
+
+	image2, _ := converter.ConvertToJpeg(NewMediaFile("storage/import/IMG_9087.CR2"))
+
+	info2, _ := image2.GetExifData()
+
+	fmt.Printf("%+v\n", info2)
+}
+
+func TestMediaFile_FindRelatedImages(t *testing.T) {
+	image := NewMediaFile("storage/import/IMG_9079.jpg")
+
+	related, err := image.GetRelatedFiles()
+
+	if err != nil {
+		t.Error(err)
+	}
+
+	for _, result := range related {
+		info, _ := result.GetExifData()
+		fmt.Printf("%s %+v\n", result.GetFilename(), info)
+	}
+}
+
+func TestMediaFile_GetPerceptiveHash(t *testing.T) {
+	image := NewMediaFile("storage/import/IMG_9079.jpg")
+
+	hash, _ := image.GetPerceptiveHash()
+	fmt.Printf("Perceptive Hash (large): %s\n", hash)
+
+	image2 := NewMediaFile("storage/import/IMG_9079_small.jpg")
+
+	hash2, _ := image2.GetPerceptiveHash()
+	fmt.Printf("Perceptive Hash (small): %s\n", hash2)
+}
+
+
+func TestMediaFile_GetMimeType(t *testing.T) {
+	image1 := NewMediaFile("storage/import/IMG_9083.jpg")
+
+	fmt.Println("MimeType: " + image1.GetMimeType())
+
+	image2 := NewMediaFile("storage/import/IMG_9082.CR2")
+
+	fmt.Println("MimeType: " + image2.GetMimeType())
+}
--- a/thumbnails.go
+++ b/thumbnails.go
@ -0,0 +1,19 @@
+package photoprism
+
+import (
+	"github.com/disintegration/imaging"
+	"log"
+)
+
+func CreateThumbnail () {
+	src, err := imaging.Open("testdata/lena_512.png")
+	if err != nil {
+		log.Print("Open failed: %v", err)
+	}
+
+	// Crop the original image to 350x350px size using the center anchor.
+	src = imaging.CropAnchor(src, 350, 350, imaging.Center)
+
+	// Resize the cropped image to width = 256px preserving the aspect ratio.
+	src = imaging.Resize(src, 256, 0, imaging.Lanczos)
+}
--- a/util.go
+++ b/util.go
@ -0,0 +1,55 @@
+package photoprism
+
+import (
+	"math/rand"
+	"time"
+	"os/user"
+	"path/filepath"
+	"os"
+	"crypto/md5"
+	"io"
+)
+
+func GetRandomInt(min, max int) int {
+	rand.Seed(time.Now().Unix())
+	return rand.Intn(max-min) + min
+}
+
+func GetExpandedFilename(filename string) string {
+	usr, _ := user.Current()
+	dir := usr.HomeDir
+
+	if filename[:2] == "~/" {
+		filename = filepath.Join(dir, filename[2:])
+	}
+
+	result, _ := filepath.Abs(filename)
+
+	return result
+}
+
+func FileExists (filename string) bool {
+	_, err := os.Stat(filename)
+
+	return err == nil
+}
+
+func Md5Sum (filename string) []byte {
+	var result []byte
+
+	file, err := os.Open(filename)
+
+	if err != nil {
+		return result
+	}
+
+	defer file.Close()
+
+	hash := md5.New()
+
+	if _, err := io.Copy(hash, file); err != nil {
+		return result
+	}
+
+	return hash.Sum(result)
+}
--- a/util_test.go
+++ b/util_test.go
@ -0,0 +1,18 @@
+package photoprism
+
+import "testing"
+
+func TestGetRandomInt(t *testing.T) {
+	min := 5
+	max := 50
+
+	for i := 0; i < 10; i++ {
+		result := GetRandomInt(min, max)
+
+		if result > max {
+			t.Errorf("Random result must not be bigger than %d", max)
+		} else if result < min {
+			t.Errorf("Random result must not be smaller than %d", min)
+		}
+	}
+}