Implemented migrate-db command

Gopkg.lock

@@ -1,6 +1,12 @@
 # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
 
 
+[[projects]]
+  name = "cloud.google.com/go"
+  packages = ["civil"]
+  revision = "777200caa7fb8936aed0f12b1fd79af64cc83ec9"
+  version = "v0.24.0"
+
 [[projects]]
   branch = "master"
   name = "github.com/araddon/dateparse"
@@ -25,6 +31,15 @@
   revision = "346938d642f2ec3594ed81d874461961cd0faa76"
   version = "v1.1.0"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/denisenkom/go-mssqldb"
+  packages = [
+    ".",
+    "internal/cp"
+  ]
+  revision = "94c9c97e8c9f9844d15c846854a7a6031ae2132c"
+
 [[projects]]
   name = "github.com/disintegration/imaging"
   packages = ["."]
@@ -38,10 +53,28 @@
   version = "v1.0.1"
 
 [[projects]]
-  name = "github.com/julienschmidt/httprouter"
+  name = "github.com/go-sql-driver/mysql"
   packages = ["."]
-  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
-  version = "v1.1"
+  revision = "d523deb1b23d913de5bdada721a6071e71283618"
+  version = "v1.4.0"
+
+[[projects]]
+  name = "github.com/jinzhu/gorm"
+  packages = [
+    ".",
+    "dialects/mssql",
+    "dialects/mysql",
+    "dialects/postgres",
+    "dialects/sqlite"
+  ]
+  revision = "6ed508ec6a4ecb3531899a69cbc746ccf65a4166"
+  version = "v1.9.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/jinzhu/inflection"
+  packages = ["."]
+  revision = "04140366298a54a039076d798123ffa108fff46c"
 
 [[projects]]
   branch = "master"
@@ -49,6 +82,28 @@
   packages = ["yaml"]
   revision = "08cad365cd28a7fba23bb1e57aa43c5e18ad8bb8"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/lib/pq"
+  packages = [
+    ".",
+    "hstore",
+    "oid"
+  ]
+  revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8"
+
+[[projects]]
+  name = "github.com/mattn/go-sqlite3"
+  packages = ["."]
+  revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4"
+  version = "v1.9.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/photoprism/photoprism"
+  packages = ["."]
+  revision = "b2659ba5ce48b223490b8f51db065d93ae8f0cf5"
+
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -98,6 +153,12 @@
   revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1"
   version = "v1.20.0"
 
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["md4"]
+  revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
+
 [[projects]]
   branch = "master"
   name = "golang.org/x/image"
@@ -112,9 +173,15 @@
   ]
   revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
 
+[[projects]]
+  name = "google.golang.org/appengine"
+  packages = ["cloudsql"]
+  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
+  version = "v1.1.0"
+
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "1ffc35574da6350baeabae12e2aed7a151f4db332e9e73656bd10b8db55de7d2"
+  inputs-digest = "8aa59b793f2c56ca48723acc6a2b517cbd2dd323af673b71c1bcc74d491951bf"
   solver-name = "gps-cdcl"
   solver-version = 1

album.go

@@ -0,0 +1,11 @@
+package photoprism
+
+import (
+	"github.com/jinzhu/gorm"
+)
+
+type Album struct {
+	gorm.Model
+	Name   string
+	Photos []Photo `gorm:"many2many:album_photos;"`
+}

cmd/photoprism/photoprism.go

@@ -36,6 +36,23 @@ func main() {
 				return nil
 			},
 		},
+		{
+			Name:  "migrate-db",
+			Usage: "Automatically migrates / initializes database",
+			Action: func(context *cli.Context) error {
+				conf.SetValuesFromFile(photoprism.GetExpandedFilename(context.GlobalString("config-file")))
+
+				conf.SetValuesFromCliContext(context)
+
+				fmt.Println("Migrating database...")
+
+				conf.MigrateDb()
+
+				fmt.Println("Done.")
+
+				return nil
+			},
+		},
 		{
 			Name:  "import",
 			Usage: "Imports photos",
@@ -48,7 +65,7 @@ func main() {
 
 				fmt.Printf("Importing photos from %s...\n", conf.ImportPath)
 
-				importer := photoprism.NewImporter(conf.OriginalsPath)
+				importer := photoprism.NewImporter(conf.OriginalsPath, conf.GetDb())
 
 				importer.ImportPhotosFromDirectory(conf.ImportPath)
 
@@ -226,4 +243,14 @@ var globalCliFlags = []cli.Flag{
 		Usage: "thumbnails path",
 		Value: "~/Photos/Thumbnails",
 	},
+	cli.StringFlag{
+		Name:  "database-driver",
+		Usage: "database driver (mysql, mssql, postgres or sqlite)",
+		Value: "mysql",
+	},
+	cli.StringFlag{
+		Name:  "database-dsn",
+		Usage: "database data source name (DSN)",
+		Value: "photoprism:photoprism@tcp(database:3306)/photoprism",
+	},
 }

cmd/tensorflowapi/utilities.go

@@ -1,17 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"net/http"
-)
-
-func responseError(w http.ResponseWriter, message string, code int) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(code)
-	json.NewEncoder(w).Encode(map[string]string{"error": message})
-}
-
-func responseJSON(w http.ResponseWriter, data interface{}) {
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(data)
-}

config.example.yml

@@ -2,4 +2,6 @@ darktable-cli: /Applications/darktable.app/Contents/MacOS/darktable-cli
 originals-path: photos/originals
 thumbnails-path: photos/thumbnails
 import-path: photos/import
-export-path: photos/export
+export-path: photos/export
+database-driver: mysql
+database-dsn: photoprism:photoprism@tcp(database:3306)/photoprism

config.go

@@ -1,8 +1,14 @@
 package photoprism
 
 import (
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mssql"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/kylelemons/go-gypsy/yaml"
 	"github.com/urfave/cli"
+	"log"
 	"os"
 	"path"
 )
@@ -14,6 +20,9 @@ type Config struct {
 	ThumbnailsPath string
 	ImportPath     string
 	ExportPath     string
+	DatabaseDriver string
+	DatabaseDsn    string
+	db             *gorm.DB
 }
 
 func NewConfig() *Config {
@@ -49,6 +58,14 @@ func (c *Config) SetValuesFromFile(fileName string) error {
 		c.DarktableCli = GetExpandedFilename(DarktableCli)
 	}
 
+	if DatabaseDriver, err := yamlConfig.Get("database-driver"); err == nil {
+		c.DatabaseDriver = DatabaseDriver
+	}
+
+	if DatabaseDsn, err := yamlConfig.Get("database-dsn"); err == nil {
+		c.DatabaseDsn = DatabaseDsn
+	}
+
 	return nil
 }
 
@@ -73,6 +90,14 @@ func (c *Config) SetValuesFromCliContext(context *cli.Context) error {
 		c.DarktableCli = GetExpandedFilename(context.String("darktable-cli"))
 	}
 
+	if context.IsSet("database-driver") {
+		c.DatabaseDriver = context.String("database-driver")
+	}
+
+	if context.IsSet("database-dsn") {
+		c.DatabaseDsn = context.String("database-dsn")
+	}
+
 	return nil
 }
 
@@ -82,3 +107,31 @@ func (c *Config) CreateDirectories() {
 	os.MkdirAll(path.Dir(c.ImportPath), os.ModePerm)
 	os.MkdirAll(path.Dir(c.ExportPath), os.ModePerm)
 }
+
+func (c *Config) ConnectToDatabase() error {
+	db, err := gorm.Open(c.DatabaseDriver, c.DatabaseDsn)
+
+	if err != nil || db == nil {
+		log.Fatal(err)
+	}
+
+	c.db = db
+
+	c.MigrateDb()
+
+	return err
+}
+
+func (c *Config) GetDb() *gorm.DB {
+	if c.db == nil {
+		c.ConnectToDatabase()
+	}
+
+	return c.db
+}
+
+func (c *Config) MigrateDb() {
+	db := c.GetDb()
+
+	db.AutoMigrate(&File{}, &Photo{}, &Tag{}, &Album{})
+}

config_test.go

@@ -2,6 +2,7 @@ package photoprism
 
 import (
 	"fmt"
+	"github.com/jinzhu/gorm"
 	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
@@ -17,6 +18,8 @@ var originalsPath = GetExpandedFilename(testDataPath + "/originals")
 var thumbnailsPath = GetExpandedFilename(testDataPath + "/thumbnails")
 var importPath = GetExpandedFilename(testDataPath + "/import")
 var exportPath = GetExpandedFilename(testDataPath + "/export")
+var databaseDriver = "mysql"
+var databaseDsn = "photoprism:photoprism@tcp(database:3306)/photoprism"
 
 func (c *Config) RemoveTestData(t *testing.T) {
 	os.RemoveAll(c.ImportPath)
@@ -67,6 +70,8 @@ func NewTestConfig() *Config {
 		ThumbnailsPath: thumbnailsPath,
 		ImportPath:     importPath,
 		ExportPath:     exportPath,
+		DatabaseDriver: databaseDriver,
+		DatabaseDsn:    databaseDsn,
 	}
 }
 
@@ -85,4 +90,16 @@ func TestConfig_SetValuesFromFile(t *testing.T) {
 	assert.Equal(t, GetExpandedFilename("photos/thumbnails"), c.ThumbnailsPath)
 	assert.Equal(t, GetExpandedFilename("photos/import"), c.ImportPath)
 	assert.Equal(t, GetExpandedFilename("photos/export"), c.ExportPath)
+	assert.Equal(t, databaseDriver, c.DatabaseDriver)
+	assert.Equal(t, databaseDsn, c.DatabaseDsn)
+}
+
+func TestConfig_ConnectToDatabase(t *testing.T) {
+	c := NewTestConfig()
+
+	c.ConnectToDatabase()
+
+	db := c.GetDb()
+
+	assert.IsType(t, &gorm.DB{}, db)
 }

docker-compose.yml

@@ -8,6 +8,7 @@ services:
       - 8080:8080
     volumes:
       - .:/go/src/photoprism
+
   database:
     image: mysql:latest
     command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=1024

export.go

@@ -1,9 +1,9 @@
 package photoprism
 
 import (
-	"os"
-	"log"
 	"fmt"
+	"log"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -46,7 +46,7 @@ func ExportPhotosFromOriginals(originals []*MediaFile, thumbnailsPath string, ex
 
 		thumbnail, err := mediaFile.GetThumbnail(thumbnailsPath, size)
 
-		if err != nil  {
+		if err != nil {
 			log.Print(err.Error())
 		}
 

file.go

@@ -0,0 +1,14 @@
+package photoprism
+
+import (
+	"github.com/jinzhu/gorm"
+)
+
+type File struct {
+	gorm.Model
+	PhotoID  uint
+	Filename string
+	Hash     string
+	FileType string
+	MimeType string
+}

importer.go

@@ -2,6 +2,7 @@ package photoprism
 
 import (
 	"fmt"
+	"github.com/jinzhu/gorm"
 	"github.com/pkg/errors"
 	"log"
 	"os"
@@ -13,14 +14,16 @@ import (
 
 type Importer struct {
 	originalsPath          string
+	db                     *gorm.DB
 	removeDotFiles         bool
 	removeExistingFiles    bool
 	removeEmptyDirectories bool
 }
 
-func NewImporter(originalsPath string) *Importer {
+func NewImporter(originalsPath string, db *gorm.DB) *Importer {
 	instance := &Importer{
 		originalsPath:          originalsPath,
+		db:                     db,
 		removeDotFiles:         true,
 		removeExistingFiles:    true,
 		removeEmptyDirectories: true,

mediafile.go

@@ -5,13 +5,13 @@ import (
 	"github.com/brett-lempereur/ish"
 	"github.com/djherbis/times"
 	"github.com/steakknife/hamming"
+	"io"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
-	"io"
 )
 
 const (

photo.go

@@ -0,0 +1,22 @@
+package photoprism
+
+import (
+	"github.com/jinzhu/gorm"
+)
+
+type Photo struct {
+	gorm.Model
+	CanonicalName  string
+	PerceptualHash string
+	Tags           []Tag `gorm:"many2many:photo_tags;"`
+	Files          []File
+	Albums         []Album `gorm:"many2many:album_photos;"`
+	Author         string
+	CameraModel    string
+	LocationName   string
+	Lat            float64
+	Long           float64
+	Liked          bool
+	Private        bool
+	Deleted        bool
+}

tag.go

@@ -0,0 +1,10 @@
+package photoprism
+
+import (
+	"github.com/jinzhu/gorm"
+)
+
+type Tag struct {
+	gorm.Model
+	Label string
+}

tensorflow/image.go

@@ -1,14 +1,12 @@
-package main
+package tensorflow
 
 import (
-	"bytes"
-
 	tf "github.com/tensorflow/tensorflow/tensorflow/go"
 	"github.com/tensorflow/tensorflow/tensorflow/go/op"
 )
 
-func makeTensorFromImage(imageBuffer *bytes.Buffer, imageFormat string) (*tf.Tensor, error) {
-	tensor, err := tf.NewTensor(imageBuffer.String())
+func makeTensorFromImage(image string, imageFormat string) (*tf.Tensor, error) {
+	tensor, err := tf.NewTensor(image)
 	if err != nil {
 		return nil, err
 	}

tensorflow/recognize.go

@@ -1,18 +1,13 @@
-package main
+package tensorflow
 
 import (
 	"bufio"
-	"bytes"
-	"io"
 	"io/ioutil"
-	"log"
-	"net/http"
 	"os"
 	"sort"
-	"strings"
-
-	"github.com/julienschmidt/httprouter"
 	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+	"log"
+	"errors"
 )
 
 type ClassifyResult struct {
@@ -30,15 +25,41 @@ var (
 	labels []string
 )
 
-func main() {
+func RecognizeImage(image string) (result []LabelResult, err error) {
 	if err := loadModel(); err != nil {
-		log.Fatal(err)
-		return
+		return nil, err
 	}
 
-	r := httprouter.New()
-	r.POST("/recognize", recognizeHandler)
-	log.Fatal(http.ListenAndServe(":8080", r))
+	// Make tensor
+	tensor, err := makeTensorFromImage(image, "jpeg")
+
+	if err != nil {
+		return nil, errors.New("invalid image")
+	}
+
+	// Run inference
+	session, err := tf.NewSession(graph, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer session.Close()
+
+	output, err := session.Run(
+		map[tf.Output]*tf.Tensor{
+			graph.Operation("input").Output(0): tensor,
+		},
+		[]tf.Output{
+			graph.Operation("output").Output(0),
+		},
+		nil)
+
+	if err != nil {
+		return nil, errors.New("could not run inference")
+	}
+
+	// Return best labels
+	return findBestLabels(output[0].Value().([][]float32)[0]), nil
 }
 
 func loadModel() error {
@@ -51,6 +72,7 @@ func loadModel() error {
 	if err := graph.Import(model, ""); err != nil {
 		return err
 	}
+
 	// Load labels
 	labelsFile, err := os.Open("/model/imagenet_comp_graph_label_strings.txt")
 	if err != nil {
@@ -58,6 +80,7 @@ func loadModel() error {
 	}
 	defer labelsFile.Close()
 	scanner := bufio.NewScanner(labelsFile)
+
 	// Labels are separated by newlines
 	for scanner.Scan() {
 		labels = append(labels, scanner.Text())
@@ -68,54 +91,6 @@ func loadModel() error {
 	return nil
 }
 
-func recognizeHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-	// Read image
-	imageFile, header, err := r.FormFile("image")
-	// Will contain filename and extension
-	imageName := strings.Split(header.Filename, ".")
-	if err != nil {
-		responseError(w, "Could not read image", http.StatusBadRequest)
-		return
-	}
-	defer imageFile.Close()
-	var imageBuffer bytes.Buffer
-	// Copy image data to a buffer
-	io.Copy(&imageBuffer, imageFile)
-
-	// ...
-	// Make tensor
-	tensor, err := makeTensorFromImage(&imageBuffer, imageName[:1][0])
-	if err != nil {
-		responseError(w, "Invalid image", http.StatusBadRequest)
-		return
-	}
-
-	// Run inference
-	session, err := tf.NewSession(graph, nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer session.Close()
-	output, err := session.Run(
-		map[tf.Output]*tf.Tensor{
-			graph.Operation("input").Output(0): tensor,
-		},
-		[]tf.Output{
-			graph.Operation("output").Output(0),
-		},
-		nil)
-	if err != nil {
-		responseError(w, "Could not run inference", http.StatusInternalServerError)
-		return
-	}
-
-	// Return best labels
-	responseJSON(w, ClassifyResult{
-		Filename: header.Filename,
-		Labels:   findBestLabels(output[0].Value().([][]float32)[0]),
-	})
-}
-
 type ByProbability []LabelResult
 
 func (a ByProbability) Len() int           { return len(a) }
@@ -131,8 +106,10 @@ func findBestLabels(probabilities []float32) []LabelResult {
 		}
 		resultLabels = append(resultLabels, LabelResult{Label: labels[i], Probability: p})
 	}
+
 	// Sort by probability
 	sort.Sort(ByProbability(resultLabels))
+
 	// Return top 5 labels
 	return resultLabels[:5]
 }

tensorflow/recognize_test.go

@@ -0,0 +1,25 @@
+package tensorflow
+
+import (
+	"testing"
+	"io/ioutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRecognizeImage(t *testing.T) {
+	if imageBuffer, err := ioutil.ReadFile("cat.jpg"); err != nil {
+		t.Error(err)
+	} else {
+		result, err := RecognizeImage(string(imageBuffer))
+
+		assert.NotNil(t, result)
+		assert.Nil(t, err)
+		assert.IsType(t, []LabelResult{}, result)
+		assert.Equal(t, 5, len(result))
+
+		assert.Equal(t, "tabby", result[0].Label)
+		assert.Equal(t, "tiger cat", result[1].Label)
+
+		assert.Equal(t, float32(0.23251747), result[1].Probability)
+	}
+}

thumbnails.go

@@ -1,10 +1,10 @@
 package photoprism
 
 import (
+	"fmt"
 	"github.com/disintegration/imaging"
 	"log"
 	"os"
-	"fmt"
 	"path/filepath"
 	"strings"
 )
@@ -30,7 +30,7 @@ func CreateThumbnailsFromOriginals(originalsPath string, thumbnailsPath string,
 		} else {
 			log.Printf("Creating %dpx thumbnail for %s", size, filename)
 
-			if _, err := mediaFile.GetThumbnail(thumbnailsPath, size); err != nil  {
+			if _, err := mediaFile.GetThumbnail(thumbnailsPath, size); err != nil {
 				log.Print(err.Error())
 			}
 		}

thumbnails_test.go

@@ -51,4 +51,4 @@ func TestCreateThumbnailsFromOriginals(t *testing.T) {
 	CreateThumbnailsFromOriginals(conf.OriginalsPath, conf.ThumbnailsPath, 600, false)
 
 	CreateThumbnailsFromOriginals(conf.OriginalsPath, conf.ThumbnailsPath, 300, true)
-}
+}