From 774b9ae12d43542caeafdfd181fdf4dbe44c21c0 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Sat, 19 Dec 2020 00:49:36 -0600
Subject: [PATCH 1/9] Added thumbnail task and database table

---
 .gitignore             |   1 +
 cps.py                 |   4 ++
 cps/constants.py       |   1 +
 cps/helper.py          |   9 ++-
 cps/tasks/thumbnail.py | 154 +++++++++++++++++++++++++++++++++++++++++
 cps/thumbnails.py      |  63 +++++++++++++++++
 cps/ub.py              |  36 +++++++++-
 7 files changed, 264 insertions(+), 4 deletions(-)
 create mode 100644 cps/tasks/thumbnail.py
 create mode 100644 cps/thumbnails.py

diff --git a/.gitignore b/.gitignore
index f06dcd44..cef58094 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ vendor/
 # calibre-web
 *.db
 *.log
+cps/cache
 
 .idea/
 *.bak
diff --git a/cps.py b/cps.py
index 50ab0076..e90a38d9 100755
--- a/cps.py
+++ b/cps.py
@@ -43,6 +43,7 @@ from cps.gdrive import gdrive
 from cps.editbooks import editbook
 from cps.remotelogin import remotelogin
 from cps.error_handler import init_errorhandler
+from cps.thumbnails import generate_thumbnails
 
 try:
     from cps.kobo import kobo, get_kobo_activated
@@ -78,6 +79,9 @@ def main():
         app.register_blueprint(kobo_auth)
     if oauth_available:
         app.register_blueprint(oauth)
+
+    generate_thumbnails()
+
     success = web_server.start()
     sys.exit(0 if success else 1)
 
diff --git a/cps/constants.py b/cps/constants.py
index c1bcbe59..0a9f9cd5 100644
--- a/cps/constants.py
+++ b/cps/constants.py
@@ -33,6 +33,7 @@ else:
 STATIC_DIR          = os.path.join(BASE_DIR, 'cps', 'static')
 TEMPLATES_DIR       = os.path.join(BASE_DIR, 'cps', 'templates')
 TRANSLATIONS_DIR    = os.path.join(BASE_DIR, 'cps', 'translations')
+CACHE_DIR           = os.path.join(BASE_DIR, 'cps', 'cache')
 
 if HOME_CONFIG:
     home_dir = os.path.join(os.path.expanduser("~"),".calibre-web")
diff --git a/cps/helper.py b/cps/helper.py
index da5ea2b3..6fc6b02a 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -551,6 +551,11 @@ def get_book_cover_with_uuid(book_uuid,
 
 def get_book_cover_internal(book, use_generic_cover_on_failure):
     if book and book.has_cover:
+        # if thumbnails.cover_thumbnail_exists_for_book(book):
+        #     thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
+        #     return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
+        # else:
+            # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title)))
         if config.config_use_google_drive:
             try:
                 if not gd.is_gdrive_ready():
@@ -561,8 +566,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
                 else:
                     log.error('%s/cover.jpg not found on Google Drive', book.path)
                     return get_cover_on_failure(use_generic_cover_on_failure)
-            except Exception as e:
-                log.debug_or_exception(e)
+            except Exception as ex:
+                log.debug_or_exception(ex)
                 return get_cover_on_failure(use_generic_cover_on_failure)
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
new file mode 100644
index 00000000..c452ab41
--- /dev/null
+++ b/cps/tasks/thumbnail.py
@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import os
+
+from cps import config, db, gdriveutils, logger, ub
+from cps.constants import CACHE_DIR as _CACHE_DIR
+from cps.services.worker import CalibreTask
+from datetime import datetime, timedelta
+from sqlalchemy import func
+
+try:
+    from wand.image import Image
+    use_IM = True
+except (ImportError, RuntimeError) as e:
+    use_IM = False
+
+THUMBNAIL_RESOLUTION_1X = 1.0
+THUMBNAIL_RESOLUTION_2X = 2.0
+
+
+class TaskThumbnail(CalibreTask):
+    def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
+        super(TaskThumbnail, self).__init__(task_message)
+        self.limit = limit
+        self.log = logger.create()
+        self.app_db_session = ub.get_new_session_instance()
+        self.worker_db = db.CalibreDB(expire_on_commit=False)
+
+    def run(self, worker_thread):
+        if self.worker_db.session and use_IM:
+            thumbnails = self.get_thumbnail_book_ids()
+            thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
+            self.log.info(','.join([str(elem) for elem in thumbnail_book_ids]))
+            self.log.info(len(thumbnail_book_ids))
+
+            books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
+
+            count = len(books_without_thumbnails)
+            for i, book in enumerate(books_without_thumbnails):
+                thumbnails = self.get_thumbnails_for_book(thumbnails, book)
+                if thumbnails:
+                    for thumbnail in thumbnails:
+                        self.update_book_thumbnail(book, thumbnail)
+
+                else:
+                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X)
+                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X)
+
+                self.progress = (1.0 / count) * i
+
+        self._handleSuccess()
+        self.app_db_session.close()
+
+    def get_thumbnail_book_ids(self):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .group_by(ub.Thumbnail.book_id)\
+            .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
+            .all()
+
+    def get_books_without_thumbnails(self, thumbnail_book_ids):
+        return self.worker_db.session\
+            .query(db.Books)\
+            .filter(db.Books.has_cover == 1)\
+            .filter(db.Books.id.notin_(thumbnail_book_ids))\
+            .limit(self.limit)\
+            .all()
+
+    def get_thumbnails_for_book(self, thumbnails, book):
+        results = list()
+        for thumbnail in thumbnails:
+            if thumbnail.book_id == book.id:
+                results.append(thumbnail)
+
+        return results
+
+    def update_book_thumbnail(self, book, thumbnail):
+        thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
+
+        try:
+            self.app_db_session.commit()
+            self.generate_book_thumbnail(book, thumbnail)
+        except Exception as ex:
+            self._handleError(u'Error updating book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def create_book_thumbnail(self, book, resolution):
+        thumbnail = ub.Thumbnail()
+        thumbnail.book_id = book.id
+        thumbnail.resolution = resolution
+
+        self.app_db_session.add(thumbnail)
+        try:
+            self.app_db_session.commit()
+            self.generate_book_thumbnail(book, thumbnail)
+        except Exception as ex:
+            self._handleError(u'Error creating book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def generate_book_thumbnail(self, book, thumbnail):
+        if book and thumbnail:
+            if config.config_use_google_drive:
+                self.log.info('google drive thumbnail')
+            else:
+                book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg')
+                if os.path.isfile(book_cover_filepath):
+                    with Image(filename=book_cover_filepath) as img:
+                        height = self.get_thumbnail_height(thumbnail)
+                        if img.height > height:
+                            width = self.get_thumbnail_width(height, img)
+                            img.resize(width=width, height=height, filter='lanczos')
+                            img.save(filename=self.get_thumbnail_cache_path(thumbnail))
+
+    def get_thumbnail_height(self, thumbnail):
+        return int(225 * thumbnail.resolution)
+
+    def get_thumbnail_width(self, height, img):
+        percent = (height / float(img.height))
+        return int((float(img.width) * float(percent)))
+
+    def get_thumbnail_cache_dir(self):
+        if not os.path.isdir(_CACHE_DIR):
+            os.makedirs(_CACHE_DIR)
+
+        if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
+            os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
+
+        return os.path.join(_CACHE_DIR, 'thumbnails')
+
+    def get_thumbnail_cache_path(self, thumbnail):
+        if thumbnail:
+            return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename)
+        return None
+
+    @property
+    def name(self):
+        return "Thumbnail"
diff --git a/cps/thumbnails.py b/cps/thumbnails.py
new file mode 100644
index 00000000..6ccff56f
--- /dev/null
+++ b/cps/thumbnails.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import os
+
+from . import logger, ub
+from .constants import CACHE_DIR as _CACHE_DIR
+from .services.worker import WorkerThread
+from .tasks.thumbnail import TaskThumbnail
+
+from datetime import datetime
+
+THUMBNAIL_RESOLUTION_1X = 1.0
+THUMBNAIL_RESOLUTION_2X = 2.0
+
+log = logger.create()
+
+
+def get_thumbnail_cache_dir():
+    if not os.path.isdir(_CACHE_DIR):
+        os.makedirs(_CACHE_DIR)
+
+    if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
+        os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
+
+    return os.path.join(_CACHE_DIR, 'thumbnails')
+
+
+def get_thumbnail_cache_path(thumbnail):
+    if thumbnail:
+        return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename)
+
+    return None
+
+
+def cover_thumbnail_exists_for_book(book):
+    if book and book.has_cover:
+        thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
+        if thumbnail and thumbnail.expiration > datetime.utcnow():
+            thumbnail_path = get_thumbnail_cache_path(thumbnail)
+            return thumbnail_path and os.path.isfile(thumbnail_path)
+
+    return False
+
+
+def generate_thumbnails():
+    WorkerThread.add(None, TaskThumbnail())
diff --git a/cps/ub.py b/cps/ub.py
index dbc3b419..4500160f 100644
--- a/cps/ub.py
+++ b/cps/ub.py
@@ -40,13 +40,14 @@ except ImportError:
         oauth_support = False
 from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
+from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session
 from werkzeug.security import generate_password_hash
 
-from . import constants
+from . import cli, constants
 
 
 session = None
@@ -434,6 +435,28 @@ class RemoteAuthToken(Base):
         return '<Token %r>' % self.id
 
 
+class Thumbnail(Base):
+    __tablename__ = 'thumbnail'
+
+    id = Column(Integer, primary_key=True)
+    book_id = Column(Integer)
+    uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True)
+    format = Column(String, default='jpeg')
+    resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0)
+    expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30))
+
+    @hybrid_property
+    def extension(self):
+        if self.format == 'jpeg':
+            return 'jpg'
+        else:
+            return self.format
+
+    @hybrid_property
+    def filename(self):
+        return self.uuid + '.' + self.extension
+
+
 # Migrate database to current version, has to be updated after every database change. Currently migration from
 # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
 # rows with SQL commands
@@ -451,6 +474,8 @@ def migrate_Database(session):
         KoboStatistics.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "archived_book"):
         ArchivedBook.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "thumbnail"):
+        Thumbnail.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "registration"):
         ReadBook.__table__.create(bind=engine)
         with engine.connect() as conn:
@@ -676,6 +701,13 @@ def init_db(app_db_path):
         create_anonymous_user(session)
 
 
+def get_new_session_instance():
+    new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False)
+    new_session = scoped_session(sessionmaker())
+    new_session.configure(bind=new_engine)
+    return new_session
+
+
 def dispose():
     global session
 

From 21fce9a5b5b004c1fcf16aa74e96d27f7e4bde0f Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Sat, 19 Dec 2020 02:58:40 -0600
Subject: [PATCH 2/9] Added background scheduler and scheduled thumbnail
 generation job

---
 cps.py                               |  4 ---
 cps/__init__.py                      |  9 ++++-
 cps/services/background_scheduler.py | 52 ++++++++++++++++++++++++++++
 cps/tasks/thumbnail.py               | 12 +++----
 cps/thumbnails.py                    |  4 ---
 requirements.txt                     |  1 +
 6 files changed, 66 insertions(+), 16 deletions(-)
 create mode 100644 cps/services/background_scheduler.py

diff --git a/cps.py b/cps.py
index e90a38d9..50ab0076 100755
--- a/cps.py
+++ b/cps.py
@@ -43,7 +43,6 @@ from cps.gdrive import gdrive
 from cps.editbooks import editbook
 from cps.remotelogin import remotelogin
 from cps.error_handler import init_errorhandler
-from cps.thumbnails import generate_thumbnails
 
 try:
     from cps.kobo import kobo, get_kobo_activated
@@ -79,9 +78,6 @@ def main():
         app.register_blueprint(kobo_auth)
     if oauth_available:
         app.register_blueprint(oauth)
-
-    generate_thumbnails()
-
     success = web_server.start()
     sys.exit(0 if success else 1)
 
diff --git a/cps/__init__.py b/cps/__init__.py
index 1a7dc868..fa85e15c 100644
--- a/cps/__init__.py
+++ b/cps/__init__.py
@@ -36,6 +36,8 @@ from flask_principal import Principal
 from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
+from .services.background_scheduler import BackgroundScheduler
+from .tasks.thumbnail import TaskThumbnail
 
 
 mimetypes.init()
@@ -95,7 +97,7 @@ def create_app():
         app.instance_path = app.instance_path.decode('utf-8')
 
     if os.environ.get('FLASK_DEBUG'):
-    	cache_buster.init_cache_busting(app)
+        cache_buster.init_cache_busting(app)
 
     log.info('Starting Calibre Web...')
     Principal(app)
@@ -115,8 +117,13 @@ def create_app():
                                            config.config_goodreads_api_secret,
                                            config.config_use_goodreads)
 
+    scheduler = BackgroundScheduler()
+    # Generate 100 book cover thumbnails every 5 minutes
+    scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5)
+
     return app
 
+
 @babel.localeselector
 def get_locale():
     # if a user is logged in, use the locale from the user settings
diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py
new file mode 100644
index 00000000..efa57379
--- /dev/null
+++ b/cps/services/background_scheduler.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import atexit
+
+from .. import logger
+from .worker import WorkerThread
+from apscheduler.schedulers.background import BackgroundScheduler as BScheduler
+
+
+class BackgroundScheduler:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(BackgroundScheduler, cls).__new__(cls)
+
+            scheduler = BScheduler()
+            atexit.register(lambda: scheduler.shutdown())
+
+            cls.log = logger.create()
+            cls.scheduler = scheduler
+            cls.scheduler.start()
+
+        return cls._instance
+
+    def add(self, func, trigger, **trigger_args):
+        self.scheduler.add_job(func=func, trigger=trigger, **trigger_args)
+
+    def add_task(self, user, task, trigger, **trigger_args):
+        def scheduled_task():
+            worker_task = task()
+            self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message)
+            WorkerThread.add(user, worker_task)
+
+        self.add(func=scheduled_task, trigger=trigger, **trigger_args)
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index c452ab41..378b688e 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -19,7 +19,7 @@
 from __future__ import division, print_function, unicode_literals
 import os
 
-from cps import config, db, gdriveutils, logger, ub
+from cps import db, logger, ub
 from cps.constants import CACHE_DIR as _CACHE_DIR
 from cps.services.worker import CalibreTask
 from datetime import datetime, timedelta
@@ -36,8 +36,9 @@ THUMBNAIL_RESOLUTION_2X = 2.0
 
 
 class TaskThumbnail(CalibreTask):
-    def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
+    def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'):
         super(TaskThumbnail, self).__init__(task_message)
+        self.config = config
         self.limit = limit
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
@@ -47,9 +48,6 @@ class TaskThumbnail(CalibreTask):
         if self.worker_db.session and use_IM:
             thumbnails = self.get_thumbnail_book_ids()
             thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
-            self.log.info(','.join([str(elem) for elem in thumbnail_book_ids]))
-            self.log.info(len(thumbnail_book_ids))
-
             books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
 
             count = len(books_without_thumbnails)
@@ -116,10 +114,10 @@ class TaskThumbnail(CalibreTask):
 
     def generate_book_thumbnail(self, book, thumbnail):
         if book and thumbnail:
-            if config.config_use_google_drive:
+            if self.config.config_use_google_drive:
                 self.log.info('google drive thumbnail')
             else:
-                book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg')
+                book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg')
                 if os.path.isfile(book_cover_filepath):
                     with Image(filename=book_cover_filepath) as img:
                         height = self.get_thumbnail_height(thumbnail)
diff --git a/cps/thumbnails.py b/cps/thumbnails.py
index 6ccff56f..89e68f50 100644
--- a/cps/thumbnails.py
+++ b/cps/thumbnails.py
@@ -57,7 +57,3 @@ def cover_thumbnail_exists_for_book(book):
             return thumbnail_path and os.path.isfile(thumbnail_path)
 
     return False
-
-
-def generate_thumbnails():
-    WorkerThread.add(None, TaskThumbnail())
diff --git a/requirements.txt b/requirements.txt
index f154dd5b..9d0a8654 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+APScheduler==3.6.3
 Babel>=1.3, <2.9
 Flask-Babel>=0.11.1,<2.1.0
 Flask-Login>=0.3.2,<0.5.1

From e48bdf9d5a038d169046c0e5352d3c9d074769c6 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Sun, 20 Dec 2020 03:11:21 -0600
Subject: [PATCH 3/9] Display thumbnails on the frontend, generate thumbnails
 from google drive

---
 cps.py                        |  5 ++++
 cps/__init__.py               |  6 -----
 cps/helper.py                 | 37 ++++++++++++++++++++---------
 cps/schedule.py               | 34 +++++++++++++++++++++++++++
 cps/tasks/thumbnail.py        | 44 +++++++++++++++++++++++++----------
 cps/templates/author.html     |  2 +-
 cps/templates/book_cover.html |  8 +++++++
 cps/templates/book_edit.html  |  4 +++-
 cps/templates/detail.html     |  3 ++-
 cps/templates/discover.html   |  3 ++-
 cps/templates/fragment.html   |  1 +
 cps/templates/grid.html       |  3 ++-
 cps/templates/index.html      |  5 ++--
 cps/templates/layout.html     |  1 +
 cps/templates/search.html     |  3 ++-
 cps/templates/shelf.html      |  3 ++-
 cps/thumbnails.py             |  9 ++-----
 cps/ub.py                     |  4 ++--
 cps/web.py                    | 11 ++++-----
 19 files changed, 133 insertions(+), 53 deletions(-)
 create mode 100644 cps/schedule.py
 create mode 100644 cps/templates/book_cover.html

diff --git a/cps.py b/cps.py
index 50ab0076..d63771eb 100755
--- a/cps.py
+++ b/cps.py
@@ -43,6 +43,7 @@ from cps.gdrive import gdrive
 from cps.editbooks import editbook
 from cps.remotelogin import remotelogin
 from cps.error_handler import init_errorhandler
+from cps.schedule import register_jobs
 
 try:
     from cps.kobo import kobo, get_kobo_activated
@@ -78,6 +79,10 @@ def main():
         app.register_blueprint(kobo_auth)
     if oauth_available:
         app.register_blueprint(oauth)
+
+    # Register scheduled jobs
+    register_jobs()
+
     success = web_server.start()
     sys.exit(0 if success else 1)
 
diff --git a/cps/__init__.py b/cps/__init__.py
index fa85e15c..6a6d361a 100644
--- a/cps/__init__.py
+++ b/cps/__init__.py
@@ -36,8 +36,6 @@ from flask_principal import Principal
 from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
-from .services.background_scheduler import BackgroundScheduler
-from .tasks.thumbnail import TaskThumbnail
 
 
 mimetypes.init()
@@ -117,10 +115,6 @@ def create_app():
                                            config.config_goodreads_api_secret,
                                            config.config_use_goodreads)
 
-    scheduler = BackgroundScheduler()
-    # Generate 100 book cover thumbnails every 5 minutes
-    scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5)
-
     return app
 
 
diff --git a/cps/helper.py b/cps/helper.py
index 6fc6b02a..d3420a11 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -52,7 +52,7 @@ except ImportError:
 
 from . import calibre_db
 from .tasks.convert import TaskConvert
-from . import logger, config, get_locale, db, ub
+from . import logger, config, get_locale, db, thumbnails, ub
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR
 from .subproc_wrapper import process_wait
@@ -538,24 +538,27 @@ def get_cover_on_failure(use_generic_cover):
         return None
 
 
-def get_book_cover(book_id):
+def get_book_cover(book_id, resolution=1):
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
-    return get_book_cover_internal(book, use_generic_cover_on_failure=True)
+    return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
 
 
-def get_book_cover_with_uuid(book_uuid,
-                             use_generic_cover_on_failure=True):
+def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True):
     book = calibre_db.get_book_by_uuid(book_uuid)
     return get_book_cover_internal(book, use_generic_cover_on_failure)
 
 
-def get_book_cover_internal(book, use_generic_cover_on_failure):
+def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, disable_thumbnail=False):
     if book and book.has_cover:
-        # if thumbnails.cover_thumbnail_exists_for_book(book):
-        #     thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
-        #     return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
-        # else:
-            # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title)))
+
+        # Send the book cover thumbnail if it exists in cache
+        if not disable_thumbnail:
+            thumbnail = get_book_cover_thumbnail(book, resolution)
+            if thumbnail:
+                if os.path.isfile(thumbnails.get_thumbnail_cache_path(thumbnail)):
+                    return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
+
+        # Send the book cover from Google Drive if configured
         if config.config_use_google_drive:
             try:
                 if not gd.is_gdrive_ready():
@@ -569,6 +572,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
             except Exception as ex:
                 log.debug_or_exception(ex)
                 return get_cover_on_failure(use_generic_cover_on_failure)
+
+        # Send the book cover from the Calibre directory
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)
             if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")):
@@ -579,6 +584,16 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
         return get_cover_on_failure(use_generic_cover_on_failure)
 
 
+def get_book_cover_thumbnail(book, resolution=1):
+    if book and book.has_cover:
+        return ub.session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.book_id == book.id)\
+            .filter(ub.Thumbnail.resolution == resolution)\
+            .filter(ub.Thumbnail.expiration > datetime.utcnow())\
+            .first()
+
+
 # saves book cover from url
 def save_cover_from_url(url, book_path):
     try:
diff --git a/cps/schedule.py b/cps/schedule.py
new file mode 100644
index 00000000..5d2c94b9
--- /dev/null
+++ b/cps/schedule.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+
+from . import logger
+from .services.background_scheduler import BackgroundScheduler
+from .tasks.thumbnail import TaskThumbnail
+
+log = logger.create()
+
+
+def register_jobs():
+    scheduler = BackgroundScheduler()
+
+    # Generate 100 book cover thumbnails every 5 minutes
+    scheduler.add_task(user=None, task=lambda: TaskThumbnail(limit=100), trigger='interval', minutes=5)
+
+    # TODO: validate thumbnail scheduled task
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index 378b688e..4e0c6db4 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -19,11 +19,13 @@
 from __future__ import division, print_function, unicode_literals
 import os
 
-from cps import db, logger, ub
+from cps import config, db, gdriveutils, logger, ub
 from cps.constants import CACHE_DIR as _CACHE_DIR
 from cps.services.worker import CalibreTask
+from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X
 from datetime import datetime, timedelta
 from sqlalchemy import func
+from urllib.request import urlopen
 
 try:
     from wand.image import Image
@@ -31,14 +33,10 @@ try:
 except (ImportError, RuntimeError) as e:
     use_IM = False
 
-THUMBNAIL_RESOLUTION_1X = 1.0
-THUMBNAIL_RESOLUTION_2X = 2.0
-
 
 class TaskThumbnail(CalibreTask):
-    def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'):
+    def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
         super(TaskThumbnail, self).__init__(task_message)
-        self.config = config
         self.limit = limit
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
@@ -114,17 +112,39 @@ class TaskThumbnail(CalibreTask):
 
     def generate_book_thumbnail(self, book, thumbnail):
         if book and thumbnail:
-            if self.config.config_use_google_drive:
-                self.log.info('google drive thumbnail')
-            else:
-                book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg')
-                if os.path.isfile(book_cover_filepath):
-                    with Image(filename=book_cover_filepath) as img:
+            if config.config_use_google_drive:
+                if not gdriveutils.is_gdrive_ready():
+                    raise Exception('Google Drive is configured but not ready')
+
+                web_content_link = gdriveutils.get_cover_via_gdrive(book.path)
+                if not web_content_link:
+                    raise Exception('Google Drive cover url not found')
+
+                stream = None
+                try:
+                    stream = urlopen(web_content_link)
+                    with Image(file=stream) as img:
                         height = self.get_thumbnail_height(thumbnail)
                         if img.height > height:
                             width = self.get_thumbnail_width(height, img)
                             img.resize(width=width, height=height, filter='lanczos')
                             img.save(filename=self.get_thumbnail_cache_path(thumbnail))
+                except Exception as ex:
+                    # Bubble exception to calling function
+                    raise ex
+                finally:
+                    stream.close()
+            else:
+                book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg')
+                if not os.path.isfile(book_cover_filepath):
+                    raise Exception('Book cover file not found')
+
+                with Image(filename=book_cover_filepath) as img:
+                    height = self.get_thumbnail_height(thumbnail)
+                    if img.height > height:
+                        width = self.get_thumbnail_width(height, img)
+                        img.resize(width=width, height=height, filter='lanczos')
+                        img.save(filename=self.get_thumbnail_cache_path(thumbnail))
 
     def get_thumbnail_height(self, thumbnail):
         return int(225 * thumbnail.resolution)
diff --git a/cps/templates/author.html b/cps/templates/author.html
index 7887aa4a..24ce876a 100644
--- a/cps/templates/author.html
+++ b/cps/templates/author.html
@@ -36,7 +36,7 @@
     <div id="books" class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
         <a href="{{ url_for('web.show_book', book_id=entry.id) }}">
-          <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" />
+            {{ book_cover_image(entry.id, entry.title) }}
         </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html
new file mode 100644
index 00000000..878c14a8
--- /dev/null
+++ b/cps/templates/book_cover.html
@@ -0,0 +1,8 @@
+{% macro book_cover_image(book_id, book_title) -%}
+    <img
+        srcset="{{ url_for('web.get_cover', book_id=book_id, resolution=1) }} 1x,
+                {{ url_for('web.get_cover', book_id=book_id, resolution=2) }} 2x"
+        src="{{ url_for('web.get_cover', book_id=book_id) }}"
+        alt="{{ book_title }}"
+    />
+{%- endmacro %}
diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html
index 003b33f9..881fa8ff 100644
--- a/cps/templates/book_edit.html
+++ b/cps/templates/book_edit.html
@@ -1,9 +1,11 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 {% if book %}
   <div class="col-sm-3 col-lg-3 col-xs-12">
     <div class="cover">
-        <img src="{{ url_for('web.get_cover', book_id=book.id, edit=1|uuidfilter)  }}" alt="{{ book.title }}"/>
+        {{ book_cover_image(book.id, book.title) }}
+<!--        <img src="{{ url_for('web.get_cover', book_id=book.id, edit=1|uuidfilter)  }}" alt="{{ book.title }}"/>-->
     </div>
 {% if g.user.role_delete_books() %}
     <div class="text-center">
diff --git a/cps/templates/detail.html b/cps/templates/detail.html
index 503d1dbd..d3615563 100644
--- a/cps/templates/detail.html
+++ b/cps/templates/detail.html
@@ -4,7 +4,8 @@
   <div class="row">
     <div class="col-sm-3 col-lg-3 col-xs-5">
       <div class="cover">
-          <img src="{{ url_for('web.get_cover', book_id=entry.id, edit=1|uuidfilter) }}" alt="{{ entry.title }}" />
+          {{ book_cover_image(entry.id, entry.title) }}
+<!--          <img src="{{ url_for('web.get_cover', book_id=entry.id, edit=1|uuidfilter) }}" alt="{{ entry.title }}" />-->
       </div>
     </div>
     <div class="col-sm-9 col-lg-9 book-meta">
diff --git a/cps/templates/discover.html b/cps/templates/discover.html
index 3c858feb..33bafbbe 100644
--- a/cps/templates/discover.html
+++ b/cps/templates/discover.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 <div class="discover load-more">
@@ -8,7 +9,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-            <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="{{ entry.title }}" />
+              {{ book_cover_image(entry.id, entry.title) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html
index 1421ea6a..901dd193 100644
--- a/cps/templates/fragment.html
+++ b/cps/templates/fragment.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 <div class="container-fluid">
   {% block body %}{% endblock %}
 </div>
diff --git a/cps/templates/grid.html b/cps/templates/grid.html
index ce2c05ac..bc3ca4a2 100644
--- a/cps/templates/grid.html
+++ b/cps/templates/grid.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 <h1 class="{{page}}">{{_(title)}}</h1>
@@ -28,7 +29,7 @@
               <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
                   <div class="cover">
                       <a href="{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].series[0].id )}}">
-                          <img src="{{ url_for('web.get_cover', book_id=entry[0].id) }}" alt="{{ entry[0].name }}"/>
+                          {{ book_cover_image(entry[0].id, entry[0].name) }}
                           <span class="badge">{{entry.count}}</span>
                       </a>
                   </div>
diff --git a/cps/templates/index.html b/cps/templates/index.html
index 1db73c89..c536884f 100644
--- a/cps/templates/index.html
+++ b/cps/templates/index.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 {% if g.user.show_detail_random() %}
@@ -8,7 +9,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books_rand">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="{{ entry.title }}" />
+              {{ book_cover_image(entry.id, entry.title) }}
           </a>
       </div>
       <div class="meta">
@@ -82,7 +83,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="{{ entry.title }}"/>
+              {{ book_cover_image(entry.id, entry.title) }}
           </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/layout.html b/cps/templates/layout.html
index 3b89a7ce..5df471a9 100644
--- a/cps/templates/layout.html
+++ b/cps/templates/layout.html
@@ -1,4 +1,5 @@
 {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal %}
+{% from 'book_cover.html' import book_cover_image %}
 <!DOCTYPE html>
 <html lang="{{ g.user.locale }}">
   <head>
diff --git a/cps/templates/search.html b/cps/templates/search.html
index aedb6f45..56b12154 100644
--- a/cps/templates/search.html
+++ b/cps/templates/search.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 <div class="discover">
@@ -43,7 +44,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
            <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-            <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="{{ entry.title }}" />
+               {{ book_cover_image(entry.id, entry.title) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html
index f7e3c1ae..7a678ea6 100644
--- a/cps/templates/shelf.html
+++ b/cps/templates/shelf.html
@@ -1,3 +1,4 @@
+{% from 'book_cover.html' import book_cover_image %}
 {% extends "layout.html" %}
 {% block body %}
 <div class="discover">
@@ -30,7 +31,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
             <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              <img src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="{{ entry.title }}" />
+                {{ book_cover_image(entry.id, entry.title) }}
             </a>
       </div>
       <div class="meta">
diff --git a/cps/thumbnails.py b/cps/thumbnails.py
index 89e68f50..ea7aac86 100644
--- a/cps/thumbnails.py
+++ b/cps/thumbnails.py
@@ -21,13 +21,11 @@ import os
 
 from . import logger, ub
 from .constants import CACHE_DIR as _CACHE_DIR
-from .services.worker import WorkerThread
-from .tasks.thumbnail import TaskThumbnail
 
 from datetime import datetime
 
-THUMBNAIL_RESOLUTION_1X = 1.0
-THUMBNAIL_RESOLUTION_2X = 2.0
+THUMBNAIL_RESOLUTION_1X = 1
+THUMBNAIL_RESOLUTION_2X = 2
 
 log = logger.create()
 
@@ -35,17 +33,14 @@ log = logger.create()
 def get_thumbnail_cache_dir():
     if not os.path.isdir(_CACHE_DIR):
         os.makedirs(_CACHE_DIR)
-
     if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
         os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
-
     return os.path.join(_CACHE_DIR, 'thumbnails')
 
 
 def get_thumbnail_cache_path(thumbnail):
     if thumbnail:
         return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename)
-
     return None
 
 
diff --git a/cps/ub.py b/cps/ub.py
index 4500160f..0b5a65e7 100644
--- a/cps/ub.py
+++ b/cps/ub.py
@@ -40,7 +40,7 @@ except ImportError:
         oauth_support = False
 from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric
+from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm.attributes import flag_modified
@@ -442,7 +442,7 @@ class Thumbnail(Base):
     book_id = Column(Integer)
     uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True)
     format = Column(String, default='jpeg')
-    resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0)
+    resolution = Column(SmallInteger, default=1)
     expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30))
 
     @hybrid_property
diff --git a/cps/web.py b/cps/web.py
index 4baf82cb..27a5849b 100644
--- a/cps/web.py
+++ b/cps/web.py
@@ -1171,14 +1171,17 @@ def advanced_search_form():
 
 
 @web.route("/cover/<int:book_id>")
+@web.route("/cover/<int:book_id>/<int:resolution>")
 @login_required_if_no_ano
-def get_cover(book_id):
-    return get_book_cover(book_id)
+def get_cover(book_id, resolution=1):
+    return get_book_cover(book_id, resolution)
+
 
 @web.route("/robots.txt")
 def get_robots():
     return send_from_directory(constants.STATIC_DIR, "robots.txt")
 
+
 @web.route("/show/<int:book_id>/<book_format>", defaults={'anyname': 'None'})
 @web.route("/show/<int:book_id>/<book_format>/<anyname>")
 @login_required_if_no_ano
@@ -1205,7 +1208,6 @@ def serve_book(book_id, book_format, anyname):
         return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)
 
 
-
 @web.route("/download/<int:book_id>/<book_format>", defaults={'anyname': 'None'})
 @web.route("/download/<int:book_id>/<book_format>/<anyname>")
 @login_required_if_no_ano
@@ -1387,9 +1389,6 @@ def logout():
     return redirect(url_for('web.login'))
 
 
-
-
-
 # ################################### Users own configuration #########################################################
 
 

From 541fc7e14ebe3716fce58593a2ff3f5b23403e3d Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Tue, 22 Dec 2020 17:49:21 -0600
Subject: [PATCH 4/9] fixed thumbnail generate tasks, added thumbnail cleanup
 task, added reconnect db scheduled job

---
 cps/fs.py              |  61 ++++++++++++++
 cps/helper.py          |   7 +-
 cps/schedule.py        |  18 ++++-
 cps/services/worker.py |  17 +++-
 cps/tasks/thumbnail.py | 177 +++++++++++++++++++++++++++++++----------
 cps/thumbnails.py      |  54 -------------
 cps/ub.py              |  28 ++++---
 7 files changed, 245 insertions(+), 117 deletions(-)
 create mode 100644 cps/fs.py
 delete mode 100644 cps/thumbnails.py

diff --git a/cps/fs.py b/cps/fs.py
new file mode 100644
index 00000000..699d5991
--- /dev/null
+++ b/cps/fs.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+from .constants import CACHE_DIR
+from os import listdir, makedirs, remove
+from os.path import isdir, isfile, join
+from shutil import rmtree
+
+CACHE_TYPE_THUMBNAILS = 'thumbnails'
+
+
+class FileSystem:
+    _instance = None
+    _cache_dir = CACHE_DIR
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(FileSystem, cls).__new__(cls)
+        return cls._instance
+
+    def get_cache_dir(self, cache_type=None):
+        if not isdir(self._cache_dir):
+            makedirs(self._cache_dir)
+
+        if cache_type and not isdir(join(self._cache_dir, cache_type)):
+            makedirs(join(self._cache_dir, cache_type))
+
+        return join(self._cache_dir, cache_type) if cache_type else self._cache_dir
+
+    def get_cache_file_path(self, filename, cache_type=None):
+        return join(self.get_cache_dir(cache_type), filename) if filename else None
+
+    def list_cache_files(self, cache_type=None):
+        path = self.get_cache_dir(cache_type)
+        return [file for file in listdir(path) if isfile(join(path, file))]
+
+    def delete_cache_dir(self, cache_type=None):
+        if not cache_type and isdir(self._cache_dir):
+            rmtree(self._cache_dir)
+        if cache_type and isdir(join(self._cache_dir, cache_type)):
+            rmtree(join(self._cache_dir, cache_type))
+
+    def delete_cache_file(self, filename, cache_type=None):
+        if isfile(join(self.get_cache_dir(cache_type), filename)):
+            remove(join(self.get_cache_dir(cache_type), filename))
diff --git a/cps/helper.py b/cps/helper.py
index d3420a11..271ab3e9 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -52,7 +52,7 @@ except ImportError:
 
 from . import calibre_db
 from .tasks.convert import TaskConvert
-from . import logger, config, get_locale, db, thumbnails, ub
+from . import logger, config, get_locale, db, fs, ub
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR
 from .subproc_wrapper import process_wait
@@ -555,8 +555,9 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, di
         if not disable_thumbnail:
             thumbnail = get_book_cover_thumbnail(book, resolution)
             if thumbnail:
-                if os.path.isfile(thumbnails.get_thumbnail_cache_path(thumbnail)):
-                    return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
+                cache = fs.FileSystem()
+                if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS):
+                    return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename)
 
         # Send the book cover from Google Drive if configured
         if config.config_use_google_drive:
diff --git a/cps/schedule.py b/cps/schedule.py
index 5d2c94b9..5c658e41 100644
--- a/cps/schedule.py
+++ b/cps/schedule.py
@@ -18,9 +18,9 @@
 
 from __future__ import division, print_function, unicode_literals
 
-from . import logger
+from . import config, db, logger, ub
 from .services.background_scheduler import BackgroundScheduler
-from .tasks.thumbnail import TaskThumbnail
+from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails
 
 log = logger.create()
 
@@ -29,6 +29,16 @@ def register_jobs():
     scheduler = BackgroundScheduler()
 
     # Generate 100 book cover thumbnails every 5 minutes
-    scheduler.add_task(user=None, task=lambda: TaskThumbnail(limit=100), trigger='interval', minutes=5)
+    scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5)
 
-    # TODO: validate thumbnail scheduled task
+    # Cleanup book cover cache every day at 4am
+    scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4)
+
+    # Reconnect metadata.db every 4 hours
+    scheduler.add(func=reconnect_db_job, trigger='interval', hours=4)
+
+
+def reconnect_db_job():
+    log.info('Running background task: reconnect to calibre database')
+    calibre_db = db.CalibreDB()
+    calibre_db.reconnect_db(config, ub.app_DB_path)
diff --git a/cps/services/worker.py b/cps/services/worker.py
index 072674a0..2b6816db 100644
--- a/cps/services/worker.py
+++ b/cps/services/worker.py
@@ -35,7 +35,6 @@ def _get_main_thread():
     raise Exception("main thread not found?!")
 
 
-
 class ImprovedQueue(queue.Queue):
     def to_list(self):
         """
@@ -45,7 +44,8 @@ class ImprovedQueue(queue.Queue):
         with self.mutex:
             return list(self.queue)
 
-#Class for all worker tasks in the background
+
+# Class for all worker tasks in the background
 class WorkerThread(threading.Thread):
     _instance = None
 
@@ -127,6 +127,10 @@ class WorkerThread(threading.Thread):
                 # CalibreTask.start() should wrap all exceptions in it's own error handling
                 item.task.start(self)
 
+            # remove self_cleanup tasks from list
+            if item.task.self_cleanup:
+                self.dequeued.remove(item)
+
             self.queue.task_done()
 
 
@@ -141,6 +145,7 @@ class CalibreTask:
         self.end_time = None
         self.message = message
         self.id = uuid.uuid4()
+        self.self_cleanup = False
 
     @abc.abstractmethod
     def run(self, worker_thread):
@@ -209,6 +214,14 @@ class CalibreTask:
         # todo: throw error if outside of [0,1]
         self._progress = x
 
+    @property
+    def self_cleanup(self):
+        return self._self_cleanup
+
+    @self_cleanup.setter
+    def self_cleanup(self, is_self_cleanup):
+        self._self_cleanup = is_self_cleanup
+
     def _handleError(self, error_message):
         self.stat = STAT_FAIL
         self.progress = 1
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index 4e0c6db4..f61eb4a7 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -19,13 +19,15 @@
 from __future__ import division, print_function, unicode_literals
 import os
 
-from cps import config, db, gdriveutils, logger, ub
-from cps.constants import CACHE_DIR as _CACHE_DIR
+from cps import config, db, fs, gdriveutils, logger, ub
 from cps.services.worker import CalibreTask
-from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X
 from datetime import datetime, timedelta
 from sqlalchemy import func
-from urllib.request import urlopen
+
+try:
+    from urllib.request import urlopen
+except ImportError as e:
+    from urllib2 import urlopen
 
 try:
     from wand.image import Image
@@ -33,73 +35,92 @@ try:
 except (ImportError, RuntimeError) as e:
     use_IM = False
 
+THUMBNAIL_RESOLUTION_1X = 1
+THUMBNAIL_RESOLUTION_2X = 2
 
-class TaskThumbnail(CalibreTask):
+
+class TaskGenerateCoverThumbnails(CalibreTask):
     def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
-        super(TaskThumbnail, self).__init__(task_message)
+        super(TaskGenerateCoverThumbnails, self).__init__(task_message)
+        self.self_cleanup = True
         self.limit = limit
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
-        self.worker_db = db.CalibreDB(expire_on_commit=False)
+        self.calibre_db = db.CalibreDB(expire_on_commit=False)
+        self.cache = fs.FileSystem()
+        self.resolutions = [
+            THUMBNAIL_RESOLUTION_1X,
+            THUMBNAIL_RESOLUTION_2X
+        ]
 
     def run(self, worker_thread):
-        if self.worker_db.session and use_IM:
-            thumbnails = self.get_thumbnail_book_ids()
-            thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
+        if self.calibre_db.session and use_IM:
+            expired_thumbnails = self.get_expired_thumbnails()
+            thumbnail_book_ids = self.get_thumbnail_book_ids()
             books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
 
             count = len(books_without_thumbnails)
             for i, book in enumerate(books_without_thumbnails):
-                thumbnails = self.get_thumbnails_for_book(thumbnails, book)
-                if thumbnails:
-                    for thumbnail in thumbnails:
-                        self.update_book_thumbnail(book, thumbnail)
-
-                else:
-                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X)
-                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X)
+                for resolution in self.resolutions:
+                    expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution(
+                        book,
+                        resolution,
+                        expired_thumbnails
+                    )
+                    if expired_thumbnail:
+                        self.update_book_thumbnail(book, expired_thumbnail)
+                    else:
+                        self.create_book_thumbnail(book, resolution)
 
                 self.progress = (1.0 / count) * i
 
         self._handleSuccess()
-        self.app_db_session.close()
+        self.app_db_session.remove()
+
+    def get_expired_thumbnails(self):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.expiration < datetime.utcnow())\
+            .all()
 
     def get_thumbnail_book_ids(self):
         return self.app_db_session\
-            .query(ub.Thumbnail)\
+            .query(ub.Thumbnail.book_id)\
             .group_by(ub.Thumbnail.book_id)\
             .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
-            .all()
+            .distinct()
 
     def get_books_without_thumbnails(self, thumbnail_book_ids):
-        return self.worker_db.session\
+        return self.calibre_db.session\
             .query(db.Books)\
             .filter(db.Books.has_cover == 1)\
             .filter(db.Books.id.notin_(thumbnail_book_ids))\
             .limit(self.limit)\
             .all()
 
-    def get_thumbnails_for_book(self, thumbnails, book):
-        results = list()
-        for thumbnail in thumbnails:
-            if thumbnail.book_id == book.id:
-                results.append(thumbnail)
+    def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails):
+        for thumbnail in expired_thumbnails:
+            if thumbnail.book_id == book.id and thumbnail.resolution == resolution:
+                return thumbnail
 
-        return results
+        return None
 
     def update_book_thumbnail(self, book, thumbnail):
+        thumbnail.generated_at = datetime.utcnow()
         thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
 
         try:
             self.app_db_session.commit()
             self.generate_book_thumbnail(book, thumbnail)
         except Exception as ex:
+            self.log.info(u'Error updating book thumbnail: ' + str(ex))
             self._handleError(u'Error updating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
     def create_book_thumbnail(self, book, resolution):
         thumbnail = ub.Thumbnail()
         thumbnail.book_id = book.id
+        thumbnail.format = 'jpeg'
         thumbnail.resolution = resolution
 
         self.app_db_session.add(thumbnail)
@@ -107,6 +128,7 @@ class TaskThumbnail(CalibreTask):
             self.app_db_session.commit()
             self.generate_book_thumbnail(book, thumbnail)
         except Exception as ex:
+            self.log.info(u'Error creating book thumbnail: ' + str(ex))
             self._handleError(u'Error creating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
@@ -128,9 +150,12 @@ class TaskThumbnail(CalibreTask):
                         if img.height > height:
                             width = self.get_thumbnail_width(height, img)
                             img.resize(width=width, height=height, filter='lanczos')
-                            img.save(filename=self.get_thumbnail_cache_path(thumbnail))
+                            img.format = thumbnail.format
+                            filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
+                            img.save(filename=filename)
                 except Exception as ex:
                     # Bubble exception to calling function
+                    self.log.info(u'Error generating thumbnail file: ' + str(ex))
                     raise ex
                 finally:
                     stream.close()
@@ -144,7 +169,9 @@ class TaskThumbnail(CalibreTask):
                     if img.height > height:
                         width = self.get_thumbnail_width(height, img)
                         img.resize(width=width, height=height, filter='lanczos')
-                        img.save(filename=self.get_thumbnail_cache_path(thumbnail))
+                        img.format = thumbnail.format
+                        filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
+                        img.save(filename=filename)
 
     def get_thumbnail_height(self, thumbnail):
         return int(225 * thumbnail.resolution)
@@ -153,20 +180,88 @@ class TaskThumbnail(CalibreTask):
         percent = (height / float(img.height))
         return int((float(img.width) * float(percent)))
 
-    def get_thumbnail_cache_dir(self):
-        if not os.path.isdir(_CACHE_DIR):
-            os.makedirs(_CACHE_DIR)
+    @property
+    def name(self):
+        return "GenerateCoverThumbnails"
 
-        if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
-            os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
 
-        return os.path.join(_CACHE_DIR, 'thumbnails')
+class TaskCleanupCoverThumbnailCache(CalibreTask):
+    def __init__(self, task_message=u'Validating cover thumbnail cache'):
+        super(TaskCleanupCoverThumbnailCache, self).__init__(task_message)
+        self.log = logger.create()
+        self.app_db_session = ub.get_new_session_instance()
+        self.calibre_db = db.CalibreDB(expire_on_commit=False)
+        self.cache = fs.FileSystem()
 
-    def get_thumbnail_cache_path(self, thumbnail):
-        if thumbnail:
-            return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename)
-        return None
+    def run(self, worker_thread):
+        cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS)
+
+        # Expire thumbnails in the database if the cached file is missing
+        # This case will happen if a user deletes the cache dir or cached files
+        if self.app_db_session:
+            self.expire_missing_thumbnails(cached_thumbnail_files)
+            self.progress = 0.33
+
+        # Delete thumbnails in the database if the book has been removed
+        # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem
+        if self.app_db_session and self.calibre_db:
+            book_ids = self.get_book_ids()
+            self.delete_thumbnails_for_missing_books(book_ids)
+            self.progress = 0.66
+
+        # Delete extraneous cached thumbnail files
+        # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally
+        if self.app_db_session:
+            db_thumbnail_files = self.get_thumbnail_filenames()
+            self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files)
+
+        self._handleSuccess()
+        self.app_db_session.remove()
+
+    def expire_missing_thumbnails(self, filenames):
+        try:
+            self.app_db_session\
+                .query(ub.Thumbnail)\
+                .filter(ub.Thumbnail.filename.notin_(filenames))\
+                .update({"expiration": datetime.utcnow()}, synchronize_session=False)
+            self.app_db_session.commit()
+        except Exception as ex:
+            self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex))
+            self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def get_book_ids(self):
+        results = self.calibre_db.session\
+            .query(db.Books.id)\
+            .filter(db.Books.has_cover == 1)\
+            .distinct()
+
+        return [value for value, in results]
+
+    def delete_thumbnails_for_missing_books(self, book_ids):
+        try:
+            self.app_db_session\
+                .query(ub.Thumbnail)\
+                .filter(ub.Thumbnail.book_id.notin_(book_ids))\
+                .delete(synchronize_session=False)
+            self.app_db_session.commit()
+        except Exception as ex:
+            self.log.info(str(ex))
+            self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def get_thumbnail_filenames(self):
+        results = self.app_db_session\
+            .query(ub.Thumbnail.filename)\
+            .all()
+
+        return [thumbnail for thumbnail, in results]
+
+    def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files):
+        extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files))
+        for file in extraneous_files:
+            self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS)
 
     @property
     def name(self):
-        return "Thumbnail"
+        return "CleanupCoverThumbnailCache"
diff --git a/cps/thumbnails.py b/cps/thumbnails.py
deleted file mode 100644
index ea7aac86..00000000
--- a/cps/thumbnails.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-
-#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
-#     Copyright (C) 2020 mmonkey
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import division, print_function, unicode_literals
-import os
-
-from . import logger, ub
-from .constants import CACHE_DIR as _CACHE_DIR
-
-from datetime import datetime
-
-THUMBNAIL_RESOLUTION_1X = 1
-THUMBNAIL_RESOLUTION_2X = 2
-
-log = logger.create()
-
-
-def get_thumbnail_cache_dir():
-    if not os.path.isdir(_CACHE_DIR):
-        os.makedirs(_CACHE_DIR)
-    if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
-        os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
-    return os.path.join(_CACHE_DIR, 'thumbnails')
-
-
-def get_thumbnail_cache_path(thumbnail):
-    if thumbnail:
-        return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename)
-    return None
-
-
-def cover_thumbnail_exists_for_book(book):
-    if book and book.has_cover:
-        thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
-        if thumbnail and thumbnail.expiration > datetime.utcnow():
-            thumbnail_path = get_thumbnail_cache_path(thumbnail)
-            return thumbnail_path and os.path.isfile(thumbnail_path)
-
-    return False
diff --git a/cps/ub.py b/cps/ub.py
index 0b5a65e7..30abd728 100644
--- a/cps/ub.py
+++ b/cps/ub.py
@@ -18,6 +18,7 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import division, print_function, unicode_literals
+import atexit
 import os
 import sys
 import datetime
@@ -42,12 +43,11 @@ from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
 from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session
 from werkzeug.security import generate_password_hash
 
-from . import cli, constants
+from . import cli, constants, logger
 
 
 session = None
@@ -435,6 +435,14 @@ class RemoteAuthToken(Base):
         return '<Token %r>' % self.id
 
 
+def filename(context):
+    file_format = context.get_current_parameters()['format']
+    if file_format == 'jpeg':
+        return context.get_current_parameters()['uuid'] + '.jpg'
+    else:
+        return context.get_current_parameters()['uuid'] + '.' + file_format
+
+
 class Thumbnail(Base):
     __tablename__ = 'thumbnail'
 
@@ -443,19 +451,10 @@ class Thumbnail(Base):
     uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True)
     format = Column(String, default='jpeg')
     resolution = Column(SmallInteger, default=1)
+    filename = Column(String, default=filename)
+    generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow())
     expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30))
 
-    @hybrid_property
-    def extension(self):
-        if self.format == 'jpeg':
-            return 'jpg'
-        else:
-            return self.format
-
-    @hybrid_property
-    def filename(self):
-        return self.uuid + '.' + self.extension
-
 
 # Migrate database to current version, has to be updated after every database change. Currently migration from
 # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
@@ -705,6 +704,9 @@ def get_new_session_instance():
     new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False)
     new_session = scoped_session(sessionmaker())
     new_session.configure(bind=new_engine)
+
+    atexit.register(lambda: new_session.remove() if new_session else True)
+
     return new_session
 
 

From 626051e4892ef42d65e16f01932ae8ce48342c22 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Wed, 23 Dec 2020 03:25:25 -0600
Subject: [PATCH 5/9] Added clear cache button to admin settings, updated cache
 busting for book cover images

---
 cps/admin.py                  | 19 +++++++-
 cps/editbooks.py              |  2 +
 cps/helper.py                 | 23 +++++++---
 cps/jinjia.py                 |  6 +++
 cps/schedule.py               | 12 +----
 cps/static/css/caliBlur.css   | 82 ++++++++++++++++++++++++++++-------
 cps/static/js/main.js         | 12 +++++
 cps/tasks/database.py         | 49 +++++++++++++++++++++
 cps/tasks/thumbnail.py        | 58 ++++++++++++++++++++++++-
 cps/templates/admin.html      | 36 +++++++++++----
 cps/templates/author.html     |  2 +-
 cps/templates/book_cover.html |  8 ++--
 cps/templates/book_edit.html  |  3 +-
 cps/templates/detail.html     |  3 +-
 cps/templates/discover.html   |  2 +-
 cps/templates/grid.html       |  2 +-
 cps/templates/index.html      |  4 +-
 cps/templates/search.html     |  2 +-
 cps/templates/shelf.html      |  2 +-
 cps/web.py                    |  8 +++-
 20 files changed, 278 insertions(+), 57 deletions(-)
 create mode 100644 cps/tasks/database.py

diff --git a/cps/admin.py b/cps/admin.py
index 8b3ca247..dbc1b708 100644
--- a/cps/admin.py
+++ b/cps/admin.py
@@ -38,7 +38,7 @@ from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError
 from sqlalchemy.sql.expression import func, or_
 
-from . import constants, logger, helper, services
+from . import constants, logger, helper, services, fs
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
 from .gdriveutils import is_gdrive_ready, gdrive_support
@@ -157,6 +157,23 @@ def shutdown():
     return json.dumps(showtext), 400
 
 
+@admi.route("/clear-cache")
+@login_required
+@admin_required
+def clear_cache():
+    cache_type = request.args.get('cache_type'.strip())
+    showtext = {}
+
+    if cache_type == fs.CACHE_TYPE_THUMBNAILS:
+        log.info('clearing cover thumbnail cache')
+        showtext['text'] = _(u'Cleared cover thumbnail cache')
+        helper.clear_cover_thumbnail_cache()
+        return json.dumps(showtext)
+
+    showtext['text'] = _(u'Unknown command')
+    return json.dumps(showtext)
+
+
 @admi.route("/admin/view")
 @login_required
 @admin_required
diff --git a/cps/editbooks.py b/cps/editbooks.py
index 08ee93b1..6d26ebca 100644
--- a/cps/editbooks.py
+++ b/cps/editbooks.py
@@ -595,6 +595,7 @@ def upload_cover(request, book):
                 abort(403)
             ret, message = helper.save_cover(requested_file, book.path)
             if ret is True:
+                helper.clear_cover_thumbnail_cache(book.id)
                 return True
             else:
                 flash(message, category="error")
@@ -684,6 +685,7 @@ def edit_book(book_id):
                         if result is True:
                             book.has_cover = 1
                             modif_date = True
+                            helper.clear_cover_thumbnail_cache(book.id)
                         else:
                             flash(error, category="error")
 
diff --git a/cps/helper.py b/cps/helper.py
index 271ab3e9..0b0c675f 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -58,6 +58,7 @@ from .constants import STATIC_DIR as _STATIC_DIR
 from .subproc_wrapper import process_wait
 from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
 from .tasks.mail import TaskEmail
+from .tasks.thumbnail import TaskClearCoverThumbnailCache
 
 log = logger.create()
 
@@ -525,6 +526,7 @@ def update_dir_stucture(book_id, calibrepath, first_author=None, orignal_filepat
 
 
 def delete_book(book, calibrepath, book_format):
+    clear_cover_thumbnail_cache(book.id)
     if config.config_use_google_drive:
         return delete_book_gdrive(book, book_format)
     else:
@@ -538,9 +540,9 @@ def get_cover_on_failure(use_generic_cover):
         return None
 
 
-def get_book_cover(book_id, resolution=1):
+def get_book_cover(book_id):
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
-    return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
+    return get_book_cover_internal(book, use_generic_cover_on_failure=True)
 
 
 def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True):
@@ -548,11 +550,19 @@ def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True):
     return get_book_cover_internal(book, use_generic_cover_on_failure)
 
 
-def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, disable_thumbnail=False):
+def get_cached_book_cover(cache_id):
+    parts = cache_id.split('_')
+    book_uuid = parts[0] if len(parts) else None
+    resolution = parts[2] if len(parts) > 2 else None
+    book = calibre_db.get_book_by_uuid(book_uuid) if book_uuid else None
+    return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
+
+
+def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None):
     if book and book.has_cover:
 
         # Send the book cover thumbnail if it exists in cache
-        if not disable_thumbnail:
+        if resolution:
             thumbnail = get_book_cover_thumbnail(book, resolution)
             if thumbnail:
                 cache = fs.FileSystem()
@@ -585,7 +595,7 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, di
         return get_cover_on_failure(use_generic_cover_on_failure)
 
 
-def get_book_cover_thumbnail(book, resolution=1):
+def get_book_cover_thumbnail(book, resolution):
     if book and book.has_cover:
         return ub.session\
             .query(ub.Thumbnail)\
@@ -846,3 +856,6 @@ def get_download_link(book_id, book_format, client):
     else:
         abort(404)
 
+
+def clear_cover_thumbnail_cache(book_id=None):
+    WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id))
diff --git a/cps/jinjia.py b/cps/jinjia.py
index 688d1fba..bf81c059 100644
--- a/cps/jinjia.py
+++ b/cps/jinjia.py
@@ -128,8 +128,14 @@ def formatseriesindex_filter(series_index):
             return series_index
     return 0
 
+
 @jinjia.app_template_filter('uuidfilter')
 def uuidfilter(var):
     return uuid4()
 
 
+@jinjia.app_template_filter('book_cover_cache_id')
+def book_cover_cache_id(book, resolution=None):
+    timestamp = int(book.last_modified.timestamp() * 1000)
+    cache_bust = str(book.uuid) + '_' + str(timestamp)
+    return cache_bust if not resolution else cache_bust + '_' + str(resolution)
diff --git a/cps/schedule.py b/cps/schedule.py
index 5c658e41..f349a231 100644
--- a/cps/schedule.py
+++ b/cps/schedule.py
@@ -18,12 +18,10 @@
 
 from __future__ import division, print_function, unicode_literals
 
-from . import config, db, logger, ub
 from .services.background_scheduler import BackgroundScheduler
+from .tasks.database import TaskReconnectDatabase
 from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails
 
-log = logger.create()
-
 
 def register_jobs():
     scheduler = BackgroundScheduler()
@@ -35,10 +33,4 @@ def register_jobs():
     scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4)
 
     # Reconnect metadata.db every 4 hours
-    scheduler.add(func=reconnect_db_job, trigger='interval', hours=4)
-
-
-def reconnect_db_job():
-    log.info('Running background task: reconnect to calibre database')
-    calibre_db = db.CalibreDB()
-    calibre_db.reconnect_db(config, ub.app_DB_path)
+    scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4)
diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css
index d085608d..da5d2933 100644
--- a/cps/static/css/caliBlur.css
+++ b/cps/static/css/caliBlur.css
@@ -5167,7 +5167,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head
     pointer-events: none
 }
 
-#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover {
+#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #ClearCacheDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover {
     cursor: pointer
 }
 
@@ -5254,7 +5254,7 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d
     margin-bottom: 20px
 }
 
-body.admin:not(.modal-open) .btn-default {
+body.admin .btn-default {
     margin-bottom: 10px
 }
 
@@ -5485,7 +5485,7 @@ body.admin.modal-open .navbar {
     z-index: 0 !important
 }
 
-#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal {
+#RestartDialog, #ShutdownDialog, #StatusDialog, #ClearCacheDialog, #deleteModal {
     top: 0;
     overflow: hidden;
     padding-top: 70px;
@@ -5495,7 +5495,7 @@ body.admin.modal-open .navbar {
     background: rgba(0, 0, 0, .5)
 }
 
-#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before {
+#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #ClearCacheDialog:before, #deleteModal:before {
     content: "\E208";
     padding-right: 10px;
     display: block;
@@ -5517,18 +5517,18 @@ body.admin.modal-open .navbar {
     z-index: 99
 }
 
-#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before {
+#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before {
     -webkit-transform: translate(0, 0);
     -ms-transform: translate(0, 0);
     transform: translate(0, 0)
 }
 
-#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog {
+#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog {
     width: 450px;
     margin: auto
 }
 
-#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content {
+#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content {
     max-height: calc(100% - 90px);
     -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5);
     box-shadow: 0 5px 15px rgba(0, 0, 0, .5);
@@ -5539,7 +5539,7 @@ body.admin.modal-open .navbar {
     width: 450px
 }
 
-#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header {
+#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header {
     padding: 15px 20px;
     border-radius: 3px 3px 0 0;
     line-height: 1.71428571;
@@ -5552,7 +5552,7 @@ body.admin.modal-open .navbar {
     text-align: left
 }
 
-#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before {
+#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before {
     padding-right: 10px;
     font-size: 18px;
     color: #999;
@@ -5576,6 +5576,11 @@ body.admin.modal-open .navbar {
     font-family: plex-icons-new, serif
 }
 
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before {
+    content: "\EA15";
+    font-family: plex-icons-new, serif
+}
+
 #deleteModal > .modal-dialog > .modal-content > .modal-header:before {
     content: "\EA6D";
     font-family: plex-icons-new, serif
@@ -5599,6 +5604,12 @@ body.admin.modal-open .navbar {
     font-size: 20px
 }
 
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:after {
+    content: "Clear Cover Thumbnail Cache";
+    display: inline-block;
+    font-size: 20px
+}
+
 #deleteModal > .modal-dialog > .modal-content > .modal-header:after {
     content: "Delete Book";
     display: inline-block;
@@ -5629,7 +5640,17 @@ body.admin.modal-open .navbar {
     text-align: left
 }
 
-#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p {
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body {
+    padding: 20px 20px 10px;
+    font-size: 16px;
+    line-height: 1.6em;
+    font-family: Open Sans Regular, Helvetica Neue, Helvetica, Arial, sans-serif;
+    color: #eee;
+    background: #282828;
+    text-align: left
+}
+
+#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p {
     padding: 20px 20px 0 0;
     font-size: 16px;
     line-height: 1.6em;
@@ -5638,7 +5659,7 @@ body.admin.modal-open .navbar {
     background: #282828
 }
 
-#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default {
+#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default {
     float: right;
     z-index: 9;
     position: relative;
@@ -5674,6 +5695,18 @@ body.admin.modal-open .navbar {
     border-radius: 3px
 }
 
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > #clear_cache {
+    float: right;
+    z-index: 9;
+    position: relative;
+    margin: 25px 0 0 10px;
+    min-width: 80px;
+    padding: 10px 18px;
+    font-size: 16px;
+    line-height: 1.33;
+    border-radius: 3px
+}
+
 #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger {
     float: right;
     z-index: 9;
@@ -5694,11 +5727,15 @@ body.admin.modal-open .navbar {
     margin: 55px 0 0 10px
 }
 
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache) {
+    margin: 25px 0 0 10px
+}
+
 #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default {
     margin: 0 0 0 10px
 }
 
-#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover {
+#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover {
     background-color: hsla(0, 0%, 100%, .3)
 }
 
@@ -5732,6 +5769,21 @@ body.admin.modal-open .navbar {
     box-shadow: 0 5px 15px rgba(0, 0, 0, .5)
 }
 
+#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body:after {
+    content: '';
+    position: absolute;
+    width: 100%;
+    height: 72px;
+    background-color: #323232;
+    border-radius: 0 0 3px 3px;
+    left: 0;
+    margin-top: 10px;
+    z-index: 0;
+    border-top: 1px solid #222;
+    -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5);
+    box-shadow: 0 5px 15px rgba(0, 0, 0, .5)
+}
+
 #deleteButton {
     position: fixed;
     top: 60px;
@@ -7322,11 +7374,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         background-color: transparent !important
     }
 
-    #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog {
+    #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog {
         max-width: calc(100vw - 40px)
     }
 
-    #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content {
+    #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content {
         max-width: calc(100vw - 40px);
         left: 0
     }
@@ -7476,7 +7528,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         padding: 30px 15px
     }
 
-    #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before {
+    #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before {
         left: auto;
         right: 34px
     }
diff --git a/cps/static/js/main.js b/cps/static/js/main.js
index 3fbaed88..d8c1863f 100644
--- a/cps/static/js/main.js
+++ b/cps/static/js/main.js
@@ -405,6 +405,18 @@ $(function() {
             }
         });
     });
+    $("#clear_cache").click(function () {
+        $("#spinner3").show();
+        $.ajax({
+            dataType: "json",
+            url: window.location.pathname + "/../../clear-cache",
+            data: {"cache_type":"thumbnails"},
+            success: function(data) {
+                $("#spinner3").hide();
+                $("#ClearCacheDialog").modal("hide");
+            }
+        });
+    });
 
     // Init all data control handlers to default
     $("input[data-control]").trigger("change");
diff --git a/cps/tasks/database.py b/cps/tasks/database.py
new file mode 100644
index 00000000..11f0186d
--- /dev/null
+++ b/cps/tasks/database.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+
+from cps import config, logger
+from cps.services.worker import CalibreTask
+
+try:
+    from urllib.request import urlopen
+except ImportError as e:
+    from urllib2 import urlopen
+
+
+class TaskReconnectDatabase(CalibreTask):
+    def __init__(self, task_message=u'Reconnecting Calibre database'):
+        super(TaskReconnectDatabase, self).__init__(task_message)
+        self.log = logger.create()
+        self.listen_address = config.get_config_ipaddress()
+        self.listen_port = config.config_port
+
+    def run(self, worker_thread):
+        address = self.listen_address if self.listen_address else 'localhost'
+        port = self.listen_port if self.listen_port else 8083
+
+        try:
+            urlopen('http://' + address + ':' + str(port) + '/reconnect')
+            self._handleSuccess()
+        except Exception as ex:
+            self._handleError(u'Unable to reconnect Calibre database: ' + str(ex))
+
+    @property
+    def name(self):
+        return "Reconnect Database"
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index f61eb4a7..b541bc70 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -42,7 +42,6 @@ THUMBNAIL_RESOLUTION_2X = 2
 class TaskGenerateCoverThumbnails(CalibreTask):
     def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
         super(TaskGenerateCoverThumbnails, self).__init__(task_message)
-        self.self_cleanup = True
         self.limit = limit
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
@@ -186,7 +185,7 @@ class TaskGenerateCoverThumbnails(CalibreTask):
 
 
 class TaskCleanupCoverThumbnailCache(CalibreTask):
-    def __init__(self, task_message=u'Validating cover thumbnail cache'):
+    def __init__(self, task_message=u'Cleaning up cover thumbnail cache'):
         super(TaskCleanupCoverThumbnailCache, self).__init__(task_message)
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
@@ -265,3 +264,58 @@ class TaskCleanupCoverThumbnailCache(CalibreTask):
     @property
     def name(self):
         return "CleanupCoverThumbnailCache"
+
+
+class TaskClearCoverThumbnailCache(CalibreTask):
+    def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'):
+        super(TaskClearCoverThumbnailCache, self).__init__(task_message)
+        self.log = logger.create()
+        self.book_id = book_id
+        self.app_db_session = ub.get_new_session_instance()
+        self.cache = fs.FileSystem()
+
+    def run(self, worker_thread):
+        if self.app_db_session:
+            if self.book_id:
+                thumbnails = self.get_thumbnails_for_book(self.book_id)
+                for thumbnail in thumbnails:
+                    self.expire_and_delete_thumbnail(thumbnail)
+            else:
+                self.expire_and_delete_all_thumbnails()
+
+        self._handleSuccess()
+        self.app_db_session.remove()
+
+    def get_thumbnails_for_book(self, book_id):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.book_id == book_id)\
+            .all()
+
+    def expire_and_delete_thumbnail(self, thumbnail):
+        thumbnail.expiration = datetime.utcnow()
+
+        try:
+            self.app_db_session.commit()
+            self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
+        except Exception as ex:
+            self.log.info(u'Error expiring book thumbnail: ' + str(ex))
+            self._handleError(u'Error expiring book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def expire_and_delete_all_thumbnails(self):
+        self.app_db_session\
+            .query(ub.Thumbnail)\
+            .update({'expiration': datetime.utcnow()})
+
+        try:
+            self.app_db_session.commit()
+            self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS)
+        except Exception as ex:
+            self.log.info(u'Error expiring book thumbnails: ' + str(ex))
+            self._handleError(u'Error expiring book thumbnails: ' + str(ex))
+            self.app_db_session.rollback()
+
+    @property
+    def name(self):
+        return "ClearCoverThumbnailCache"
diff --git a/cps/templates/admin.html b/cps/templates/admin.html
index 1ef64157..e3d20db3 100644
--- a/cps/templates/admin.html
+++ b/cps/templates/admin.html
@@ -139,15 +139,18 @@
     </div>
   </div>
 
-    <div class="row form-group">
+  <div class="row form-group">
     <h2>{{_('Administration')}}</h2>
-      <div class="btn btn-default"><a id="debug" href="{{url_for('admin.download_debug')}}">{{_('Download Debug Package')}}</a></div>
-      <div class="btn btn-default"><a id="logfile" href="{{url_for('admin.view_logfile')}}">{{_('View Logs')}}</a></div>
-    </div>
-    <div class="row form-group">
-      <div class="btn btn-default" id="restart_database" data-toggle="modal" data-target="#StatusDialog">{{_('Reconnect Calibre Database')}}</div>
-      <div class="btn btn-default" id="admin_restart" data-toggle="modal" data-target="#RestartDialog">{{_('Restart')}}</div>
-      <div class="btn btn-default" id="admin_stop" data-toggle="modal" data-target="#ShutdownDialog">{{_('Shutdown')}}</div>
+    <div class="btn btn-default"><a id="debug" href="{{url_for('admin.download_debug')}}">{{_('Download Debug Package')}}</a></div>
+    <div class="btn btn-default"><a id="logfile" href="{{url_for('admin.view_logfile')}}">{{_('View Logs')}}</a></div>
+  </div>
+  <div class="row form-group">
+    <div class="btn btn-default" id="restart_database" data-toggle="modal" data-target="#StatusDialog">{{_('Reconnect Calibre Database')}}</div>
+    <div class="btn btn-default" id="clear_cover_thumbnail_cache" data-toggle="modal" data-target="#ClearCacheDialog">{{_('Clear Cover Thumbnail Cache')}}</div>
+  </div>
+  <div class="row form-group">
+    <div class="btn btn-default" id="admin_restart" data-toggle="modal" data-target="#RestartDialog">{{_('Restart')}}</div>
+    <div class="btn btn-default" id="admin_stop" data-toggle="modal" data-target="#ShutdownDialog">{{_('Shutdown')}}</div>
   </div>
 
   <div class="row">
@@ -226,4 +229,21 @@
     </div>
   </div>
 </div>
+<div id="ClearCacheDialog" class="modal fade" role="dialog">
+  <div class="modal-dialog modal-sm">
+    <!-- Modal content-->
+    <div class="modal-content">
+      <div class="modal-header bg-info"></div>
+      <div class="modal-body text-center">
+        <p>{{_('Are you sure you want to clear the cover thumbnail cache?')}}</p>
+        <div id="spinner3" class="spinner" style="display:none;">
+          <img id="img-spinner3" src="{{ url_for('static', filename='css/libs/images/loading-icon.gif') }}"/>
+        </div>
+        <p></p>
+        <button type="button" class="btn btn-default" id="clear_cache" >{{_('OK')}}</button>
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
 {% endblock %}
diff --git a/cps/templates/author.html b/cps/templates/author.html
index 24ce876a..3cf3fb4b 100644
--- a/cps/templates/author.html
+++ b/cps/templates/author.html
@@ -36,7 +36,7 @@
     <div id="books" class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
         <a href="{{ url_for('web.show_book', book_id=entry.id) }}">
-            {{ book_cover_image(entry.id, entry.title) }}
+            {{ book_cover_image(entry, entry.title) }}
         </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html
index 878c14a8..3f0ed154 100644
--- a/cps/templates/book_cover.html
+++ b/cps/templates/book_cover.html
@@ -1,8 +1,8 @@
-{% macro book_cover_image(book_id, book_title) -%}
+{% macro book_cover_image(book, book_title) -%}
     <img
-        srcset="{{ url_for('web.get_cover', book_id=book_id, resolution=1) }} 1x,
-                {{ url_for('web.get_cover', book_id=book_id, resolution=2) }} 2x"
-        src="{{ url_for('web.get_cover', book_id=book_id) }}"
+        srcset="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id(1)) }} 1x,
+                {{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id(2)) }} 2x"
+        src="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id) }}"
         alt="{{ book_title }}"
     />
 {%- endmacro %}
diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html
index 881fa8ff..369b8d05 100644
--- a/cps/templates/book_edit.html
+++ b/cps/templates/book_edit.html
@@ -4,8 +4,7 @@
 {% if book %}
   <div class="col-sm-3 col-lg-3 col-xs-12">
     <div class="cover">
-        {{ book_cover_image(book.id, book.title) }}
-<!--        <img src="{{ url_for('web.get_cover', book_id=book.id, edit=1|uuidfilter)  }}" alt="{{ book.title }}"/>-->
+        {{ book_cover_image(book, book.title) }}
     </div>
 {% if g.user.role_delete_books() %}
     <div class="text-center">
diff --git a/cps/templates/detail.html b/cps/templates/detail.html
index d3615563..cdf6ab2b 100644
--- a/cps/templates/detail.html
+++ b/cps/templates/detail.html
@@ -4,8 +4,7 @@
   <div class="row">
     <div class="col-sm-3 col-lg-3 col-xs-5">
       <div class="cover">
-          {{ book_cover_image(entry.id, entry.title) }}
-<!--          <img src="{{ url_for('web.get_cover', book_id=entry.id, edit=1|uuidfilter) }}" alt="{{ entry.title }}" />-->
+          {{ book_cover_image(entry, entry.title) }}
       </div>
     </div>
     <div class="col-sm-9 col-lg-9 book-meta">
diff --git a/cps/templates/discover.html b/cps/templates/discover.html
index 33bafbbe..5d4666f6 100644
--- a/cps/templates/discover.html
+++ b/cps/templates/discover.html
@@ -9,7 +9,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry.id, entry.title) }}
+              {{ book_cover_image(entry, entry.title) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/grid.html b/cps/templates/grid.html
index bc3ca4a2..67594b4e 100644
--- a/cps/templates/grid.html
+++ b/cps/templates/grid.html
@@ -29,7 +29,7 @@
               <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
                   <div class="cover">
                       <a href="{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].series[0].id )}}">
-                          {{ book_cover_image(entry[0].id, entry[0].name) }}
+                          {{ book_cover_image(entry[0], entry[0].name) }}
                           <span class="badge">{{entry.count}}</span>
                       </a>
                   </div>
diff --git a/cps/templates/index.html b/cps/templates/index.html
index c536884f..531a535c 100644
--- a/cps/templates/index.html
+++ b/cps/templates/index.html
@@ -9,7 +9,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books_rand">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry.id, entry.title) }}
+              {{ book_cover_image(entry, entry.title) }}
           </a>
       </div>
       <div class="meta">
@@ -83,7 +83,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry.id, entry.title) }}
+              {{ book_cover_image(entry, entry.title) }}
           </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/search.html b/cps/templates/search.html
index 56b12154..8e0cf668 100644
--- a/cps/templates/search.html
+++ b/cps/templates/search.html
@@ -44,7 +44,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
            <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-               {{ book_cover_image(entry.id, entry.title) }}
+               {{ book_cover_image(entry, entry.title) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html
index 7a678ea6..bebc0b1f 100644
--- a/cps/templates/shelf.html
+++ b/cps/templates/shelf.html
@@ -31,7 +31,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
             <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-                {{ book_cover_image(entry.id, entry.title) }}
+                {{ book_cover_image(entry, entry.title) }}
             </a>
       </div>
       <div class="meta">
diff --git a/cps/web.py b/cps/web.py
index 27a5849b..16c14fcf 100644
--- a/cps/web.py
+++ b/cps/web.py
@@ -50,7 +50,7 @@ from . import babel, db, ub, config, get_locale, app
 from . import calibre_db, shelf
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
 from .helper import check_valid_domain, render_task_status, \
-    get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
+    get_cc_columns, get_book_cover, get_cached_book_cover, get_download_link, send_mail, generate_random_password, \
     send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
@@ -1177,6 +1177,12 @@ def get_cover(book_id, resolution=1):
     return get_book_cover(book_id, resolution)
 
 
+@web.route("/cached-cover/<string:cache_id>")
+@login_required_if_no_ano
+def get_cached_cover(cache_id):
+    return get_cached_book_cover(cache_id)
+
+
 @web.route("/robots.txt")
 def get_robots():
     return send_from_directory(constants.STATIC_DIR, "robots.txt")

From 242a2767a1e374a938de2f1b3fdb7cb2175c5fd1 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Thu, 24 Dec 2020 02:35:32 -0600
Subject: [PATCH 6/9] Added thumbnail urls to book cover srcsets with cache
 busting ids

---
 cps/db.py                     |  3 +-
 cps/helper.py                 | 38 +++++++++++++++
 cps/jinjia.py                 | 16 +++++++
 cps/schedule.py               | 10 ++--
 cps/tasks/thumbnail.py        | 61 ++++++++++++++++++++----
 cps/templates/author.html     |  2 +-
 cps/templates/book_cover.html | 19 +++++---
 cps/templates/book_edit.html  |  2 +-
 cps/templates/detail.html     |  2 +-
 cps/templates/discover.html   |  2 +-
 cps/templates/grid.html       |  2 +-
 cps/templates/index.html      |  4 +-
 cps/templates/search.html     |  2 +-
 cps/templates/shelf.html      |  2 +-
 cps/web.py                    | 89 +++++++++++++++++++++++++----------
 15 files changed, 199 insertions(+), 55 deletions(-)

diff --git a/cps/db.py b/cps/db.py
index 2e428f72..8b8db10a 100644
--- a/cps/db.py
+++ b/cps/db.py
@@ -609,7 +609,8 @@ class CalibreDB():
             randm = self.session.query(Books) \
                 .filter(self.common_filters(allow_show_archived)) \
                 .order_by(func.random()) \
-                .limit(self.config.config_random_books)
+                .limit(self.config.config_random_books) \
+                .all()
         else:
             randm = false()
         off = int(int(pagesize) * (page - 1))
diff --git a/cps/helper.py b/cps/helper.py
index 0b0c675f..c33a69e1 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -533,6 +533,21 @@ def delete_book(book, calibrepath, book_format):
         return delete_book_file(book, calibrepath, book_format)
 
 
+def get_thumbnails_for_books(books):
+    books_with_covers = list(filter(lambda b: b.has_cover, books))
+    book_ids = list(map(lambda b: b.id, books_with_covers))
+    return ub.session\
+        .query(ub.Thumbnail)\
+        .filter(ub.Thumbnail.book_id.in_(book_ids))\
+        .filter(ub.Thumbnail.expiration > datetime.utcnow())\
+        .all()
+
+
+def get_thumbnails_for_book_series(series):
+    books = list(map(lambda s: s[0], series))
+    return get_thumbnails_for_books(books)
+
+
 def get_cover_on_failure(use_generic_cover):
     if use_generic_cover:
         return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
@@ -558,6 +573,29 @@ def get_cached_book_cover(cache_id):
     return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
 
 
+def get_cached_book_cover_thumbnail(cache_id):
+    parts = cache_id.split('_')
+    thumbnail_uuid = parts[0] if len(parts) else None
+    thumbnail = None
+    if thumbnail_uuid:
+        thumbnail = ub.session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.uuid == thumbnail_uuid)\
+            .first()
+
+    if thumbnail and thumbnail.expiration > datetime.utcnow():
+        cache = fs.FileSystem()
+        if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS):
+            return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename)
+
+    elif thumbnail:
+        book = calibre_db.get_book(thumbnail.book_id)
+        return get_book_cover_internal(book, use_generic_cover_on_failure=True)
+
+    else:
+        return get_cover_on_failure(True)
+
+
 def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None):
     if book and book.has_cover:
 
diff --git a/cps/jinjia.py b/cps/jinjia.py
index bf81c059..b2479adc 100644
--- a/cps/jinjia.py
+++ b/cps/jinjia.py
@@ -139,3 +139,19 @@ def book_cover_cache_id(book, resolution=None):
     timestamp = int(book.last_modified.timestamp() * 1000)
     cache_bust = str(book.uuid) + '_' + str(timestamp)
     return cache_bust if not resolution else cache_bust + '_' + str(resolution)
+
+
+@jinjia.app_template_filter('get_book_thumbnails')
+def get_book_thumbnails(book_id, thumbnails=None):
+    return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list()
+
+
+@jinjia.app_template_filter('get_book_thumbnail_srcset')
+def get_book_thumbnail_srcset(thumbnails):
+    srcset = list()
+    for thumbnail in thumbnails:
+        timestamp = int(thumbnail.generated_at.timestamp() * 1000)
+        cache_id = str(thumbnail.uuid) + '_' + str(timestamp)
+        url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id)
+        srcset.append(url + ' ' + str(thumbnail.resolution) + 'x')
+    return ', '.join(srcset)
diff --git a/cps/schedule.py b/cps/schedule.py
index f349a231..7ee43410 100644
--- a/cps/schedule.py
+++ b/cps/schedule.py
@@ -20,17 +20,17 @@ from __future__ import division, print_function, unicode_literals
 
 from .services.background_scheduler import BackgroundScheduler
 from .tasks.database import TaskReconnectDatabase
-from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails
+from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails
 
 
 def register_jobs():
     scheduler = BackgroundScheduler()
 
     # Generate 100 book cover thumbnails every 5 minutes
-    scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5)
+    scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5')
 
-    # Cleanup book cover cache every day at 4am
-    scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4)
+    # Cleanup book cover cache every 6 hours
+    scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6')
 
     # Reconnect metadata.db every 4 hours
-    scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4)
+    scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4')
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index b541bc70..e9df170e 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -59,6 +59,10 @@ class TaskGenerateCoverThumbnails(CalibreTask):
             books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
 
             count = len(books_without_thumbnails)
+            if count == 0:
+                # Do not display this task on the frontend if there are no covers to update
+                self.self_cleanup = True
+
             for i, book in enumerate(books_without_thumbnails):
                 for resolution in self.resolutions:
                     expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution(
@@ -71,6 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask):
                     else:
                         self.create_book_thumbnail(book, resolution)
 
+                self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count))
                 self.progress = (1.0 / count) * i
 
         self._handleSuccess()
@@ -181,12 +186,12 @@ class TaskGenerateCoverThumbnails(CalibreTask):
 
     @property
     def name(self):
-        return "GenerateCoverThumbnails"
+        return "ThumbnailsGenerate"
 
 
-class TaskCleanupCoverThumbnailCache(CalibreTask):
-    def __init__(self, task_message=u'Cleaning up cover thumbnail cache'):
-        super(TaskCleanupCoverThumbnailCache, self).__init__(task_message)
+class TaskSyncCoverThumbnailCache(CalibreTask):
+    def __init__(self, task_message=u'Syncing cover thumbnail cache'):
+        super(TaskSyncCoverThumbnailCache, self).__init__(task_message)
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
         self.calibre_db = db.CalibreDB(expire_on_commit=False)
@@ -199,14 +204,23 @@ class TaskCleanupCoverThumbnailCache(CalibreTask):
         # This case will happen if a user deletes the cache dir or cached files
         if self.app_db_session:
             self.expire_missing_thumbnails(cached_thumbnail_files)
-            self.progress = 0.33
+            self.progress = 0.25
 
         # Delete thumbnails in the database if the book has been removed
         # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem
         if self.app_db_session and self.calibre_db:
             book_ids = self.get_book_ids()
             self.delete_thumbnails_for_missing_books(book_ids)
-            self.progress = 0.66
+            self.progress = 0.50
+
+        # Expire thumbnails in the database if their corresponding book has been updated since they were generated
+        # This case will happen if the book was updated externally
+        if self.app_db_session and self.cache:
+            books = self.get_books_updated_in_the_last_day()
+            book_ids = list(map(lambda b: b.id, books))
+            thumbnails = self.get_thumbnails_for_updated_books(book_ids)
+            self.expire_thumbnails_for_updated_book(books, thumbnails)
+            self.progress = 0.75
 
         # Delete extraneous cached thumbnail files
         # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally
@@ -261,9 +275,40 @@ class TaskCleanupCoverThumbnailCache(CalibreTask):
         for file in extraneous_files:
             self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS)
 
+    def get_books_updated_in_the_last_day(self):
+        return self.calibre_db.session\
+            .query(db.Books)\
+            .filter(db.Books.has_cover == 1)\
+            .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\
+            .all()
+
+    def get_thumbnails_for_updated_books(self, book_ids):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.book_id.in_(book_ids))\
+            .all()
+
+    def expire_thumbnails_for_updated_book(self, books, thumbnails):
+        thumbnail_ids = list()
+        for book in books:
+            for thumbnail in thumbnails:
+                if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified:
+                    thumbnail_ids.append(thumbnail.id)
+
+        try:
+            self.app_db_session\
+                .query(ub.Thumbnail)\
+                .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \
+                .update({"expiration": datetime.utcnow()}, synchronize_session=False)
+            self.app_db_session.commit()
+        except Exception as ex:
+            self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex))
+            self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex))
+            self.app_db_session.rollback()
+
     @property
     def name(self):
-        return "CleanupCoverThumbnailCache"
+        return "ThumbnailsSync"
 
 
 class TaskClearCoverThumbnailCache(CalibreTask):
@@ -318,4 +363,4 @@ class TaskClearCoverThumbnailCache(CalibreTask):
 
     @property
     def name(self):
-        return "ClearCoverThumbnailCache"
+        return "ThumbnailsClear"
diff --git a/cps/templates/author.html b/cps/templates/author.html
index 3cf3fb4b..e5acdd2f 100644
--- a/cps/templates/author.html
+++ b/cps/templates/author.html
@@ -36,7 +36,7 @@
     <div id="books" class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
         <a href="{{ url_for('web.show_book', book_id=entry.id) }}">
-            {{ book_cover_image(entry, entry.title) }}
+            {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
         </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html
index 3f0ed154..c5281797 100644
--- a/cps/templates/book_cover.html
+++ b/cps/templates/book_cover.html
@@ -1,8 +1,13 @@
-{% macro book_cover_image(book, book_title) -%}
-    <img
-        srcset="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id(1)) }} 1x,
-                {{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id(2)) }} 2x"
-        src="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id) }}"
-        alt="{{ book_title }}"
-    />
+{% macro book_cover_image(book, thumbnails) -%}
+    {%- set book_title = book.title if book.title else book.name -%}
+    {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %}
+    {%- if srcset|length -%}
+        <img
+            srcset="{{ srcset }}"
+            src="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id) }}"
+            alt="{{ book_title }}"
+        />
+    {%- else -%}
+        <img src="{{ url_for('web.get_cached_cover', cache_id=book|book_cover_cache_id) }}" alt="{{ book_title }}" />
+    {%- endif -%}
 {%- endmacro %}
diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html
index 369b8d05..c538d5ca 100644
--- a/cps/templates/book_edit.html
+++ b/cps/templates/book_edit.html
@@ -4,7 +4,7 @@
 {% if book %}
   <div class="col-sm-3 col-lg-3 col-xs-12">
     <div class="cover">
-        {{ book_cover_image(book, book.title) }}
+        {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }}
     </div>
 {% if g.user.role_delete_books() %}
     <div class="text-center">
diff --git a/cps/templates/detail.html b/cps/templates/detail.html
index cdf6ab2b..671186c7 100644
--- a/cps/templates/detail.html
+++ b/cps/templates/detail.html
@@ -4,7 +4,7 @@
   <div class="row">
     <div class="col-sm-3 col-lg-3 col-xs-5">
       <div class="cover">
-          {{ book_cover_image(entry, entry.title) }}
+          {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
       </div>
     </div>
     <div class="col-sm-9 col-lg-9 book-meta">
diff --git a/cps/templates/discover.html b/cps/templates/discover.html
index 5d4666f6..c5c12db2 100644
--- a/cps/templates/discover.html
+++ b/cps/templates/discover.html
@@ -9,7 +9,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry, entry.title) }}
+              {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/grid.html b/cps/templates/grid.html
index 67594b4e..0f669d51 100644
--- a/cps/templates/grid.html
+++ b/cps/templates/grid.html
@@ -29,7 +29,7 @@
               <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
                   <div class="cover">
                       <a href="{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].series[0].id )}}">
-                          {{ book_cover_image(entry[0], entry[0].name) }}
+                          {{ book_cover_image(entry[0], entry[0].id|get_book_thumbnails(thumbnails)) }}
                           <span class="badge">{{entry.count}}</span>
                       </a>
                   </div>
diff --git a/cps/templates/index.html b/cps/templates/index.html
index 531a535c..2be15ba7 100644
--- a/cps/templates/index.html
+++ b/cps/templates/index.html
@@ -9,7 +9,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books_rand">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry, entry.title) }}
+              {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
           </a>
       </div>
       <div class="meta">
@@ -83,7 +83,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book" id="books">
       <div class="cover">
           <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-              {{ book_cover_image(entry, entry.title) }}
+              {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
           </a>
       </div>
       <div class="meta">
diff --git a/cps/templates/search.html b/cps/templates/search.html
index 8e0cf668..a5871afb 100644
--- a/cps/templates/search.html
+++ b/cps/templates/search.html
@@ -44,7 +44,7 @@
       <div class="cover">
         {% if entry.has_cover is defined %}
            <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-               {{ book_cover_image(entry, entry.title) }}
+               {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
           </a>
         {% endif %}
       </div>
diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html
index bebc0b1f..cb55c50c 100644
--- a/cps/templates/shelf.html
+++ b/cps/templates/shelf.html
@@ -31,7 +31,7 @@
     <div class="col-sm-3 col-lg-2 col-xs-6 book">
       <div class="cover">
             <a href="{{ url_for('web.show_book', book_id=entry.id) }}" data-toggle="modal" data-target="#bookDetailsModal" data-remote="false">
-                {{ book_cover_image(entry, entry.title) }}
+                {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
             </a>
       </div>
       <div class="meta">
diff --git a/cps/web.py b/cps/web.py
index 16c14fcf..c400b96e 100644
--- a/cps/web.py
+++ b/cps/web.py
@@ -49,9 +49,10 @@ from . import constants, logger, isoLanguages, services
 from . import babel, db, ub, config, get_locale, app
 from . import calibre_db, shelf
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import check_valid_domain, render_task_status, \
-    get_cc_columns, get_book_cover, get_cached_book_cover, get_download_link, send_mail, generate_random_password, \
-    send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
+from .helper import check_valid_domain, render_task_status, get_cc_columns, get_book_cover, get_cached_book_cover, \
+    get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, get_download_link, \
+    send_mail, generate_random_password, send_registration_mail, check_send_to_kindle, check_read_formats, \
+    tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
 from .usermanagement import login_required_if_no_ano
@@ -386,16 +387,18 @@ def render_books_list(data, sort, book_id, page):
                                                                     db.Books,
                                                                     db.Books.ratings.any(db.Ratings.rating > 9),
                                                                     order)
+            thumbnails = get_thumbnails_for_books(entries + random)
             return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                         id=book_id, title=_(u"Top Rated Books"), page="rated")
+                                         id=book_id, title=_(u"Top Rated Books"), page="rated", thumbnails=thumbnails)
         else:
             abort(404)
     elif data == "discover":
         if current_user.check_visibility(constants.SIDEBAR_RANDOM):
             entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)])
             pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page)
+            thumbnails = get_thumbnails_for_books(entries)
             return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id,
-                                         title=_(u"Discover (Random Books)"), page="discover")
+                                         title=_(u"Discover (Random Books)"), page="discover", thumbnails=thumbnails)
         else:
             abort(404)
     elif data == "unread":
@@ -433,8 +436,9 @@ def render_books_list(data, sort, book_id, page):
     else:
         website = data or "newest"
         entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order)
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                     title=_(u"Books"), page=website)
+                                     title=_(u"Books"), page=website, thumbnails=thumbnails)
 
 
 def render_hot_books(page):
@@ -458,8 +462,9 @@ def render_hot_books(page):
                 ub.delete_download(book.Downloads.book_id)
         numBooks = entries.__len__()
         pagination = Pagination(page, config.config_books_per_page, numBooks)
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                     title=_(u"Hot Books (Most Downloaded)"), page="hot")
+                                     title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -490,12 +495,14 @@ def render_downloaded_books(page, order):
                              .filter(db.Books.id == book.id).first():
                 ub.delete_download(book.id)
 
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html',
                                      random=random,
                                      entries=entries,
                                      pagination=pagination,
                                      title=_(u"Downloaded books by %(user)s",user=current_user.nickname),
-                                     page="download")
+                                     page="download",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -521,9 +528,10 @@ def render_author_books(page, author_id, order):
         author_info = services.goodreads_support.get_author_info(author_name)
         other_books = services.goodreads_support.get_other_books(author_info, entries)
 
+    thumbnails = get_thumbnails_for_books(entries)
     return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id,
                                  title=_(u"Author: %(name)s", name=author_name), author=author_info,
-                                 other_books=other_books, page="author")
+                                 other_books=other_books, page="author", thumbnails=thumbnails)
 
 
 def render_publisher_books(page, book_id, order):
@@ -535,8 +543,10 @@ def render_publisher_books(page, book_id, order):
                                                                 [db.Series.name, order[0], db.Books.series_index],
                                                                 db.books_series_link,
                                                                 db.Series)
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher")
+                                     title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -548,8 +558,10 @@ def render_series_books(page, book_id, order):
                                                                 db.Books,
                                                                 db.Books.series.any(db.Series.id == book_id),
                                                                 [order[0]])
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Series: %(serie)s", serie=name.name), page="series")
+                                     title=_(u"Series: %(serie)s", serie=name.name), page="series",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -561,8 +573,10 @@ def render_ratings_books(page, book_id, order):
                                                             db.Books.ratings.any(db.Ratings.id == book_id),
                                                             [order[0]])
     if name and name.rating <= 10:
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings")
+                                     title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -574,8 +588,10 @@ def render_formats_books(page, book_id, order):
                                                                 db.Books,
                                                                 db.Books.data.any(db.Data.format == book_id.upper()),
                                                                 [order[0]])
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"File format: %(format)s", format=name.format), page="formats")
+                                     title=_(u"File format: %(format)s", format=name.format), page="formats",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -588,8 +604,10 @@ def render_category_books(page, book_id, order):
                                                                 db.Books.tags.any(db.Tags.id == book_id),
                                                                 [order[0], db.Series.name, db.Books.series_index],
                                                                 db.books_series_link, db.Series)
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Category: %(name)s", name=name.name), page="category")
+                                     title=_(u"Category: %(name)s", name=name.name), page="category",
+                                     thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -607,8 +625,9 @@ def render_language_books(page, name, order):
                                                             db.Books,
                                                             db.Books.languages.any(db.Languages.lang_code == name),
                                                             [order[0]])
+    thumbnails = get_thumbnails_for_books(entries + random)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
-                                 title=_(u"Language: %(name)s", name=lang_name), page="language")
+                                 title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails)
 
 
 def render_read_books(page, are_read, as_xml=False, order=None):
@@ -652,8 +671,10 @@ def render_read_books(page, are_read, as_xml=False, order=None):
         else:
             name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')'
             pagename = "unread"
+
+        thumbnails = get_thumbnails_for_books(entries + random)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                     title=name, page=pagename)
+                                     title=name, page=pagename, thumbnails=thumbnails)
 
 
 def render_archived_books(page, order):
@@ -676,8 +697,9 @@ def render_archived_books(page, order):
 
     name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
     pagename = "archived"
+    thumbnails = get_thumbnails_for_books(entries + random)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                 title=name, page=pagename)
+                                 title=name, page=pagename, thumbnails=thumbnails)
 
 
 def render_prepare_search_form(cc):
@@ -710,6 +732,7 @@ def render_prepare_search_form(cc):
 
 def render_search_results(term, offset=None, order=None, limit=None):
     entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
+    thumbnails = get_thumbnails_for_books(entries)
     return render_title_template('search.html',
                                  searchterm=term,
                                  pagination=pagination,
@@ -718,7 +741,8 @@ def render_search_results(term, offset=None, order=None, limit=None):
                                  entries=entries,
                                  result_count=result_count,
                                  title=_(u"Search"),
-                                 page="search")
+                                 page="search",
+                                 thumbnails=thumbnails)
 
 
 # ################################### View Books list ##################################################################
@@ -748,6 +772,7 @@ def books_table():
     return render_title_template('book_table.html', title=_(u"Books List"), page="book_table",
                                  visiblility=visibility)
 
+
 @web.route("/ajax/listbooks")
 @login_required
 def list_books():
@@ -780,6 +805,7 @@ def list_books():
     response.headers["Content-Type"] = "application/json; charset=utf-8"
     return response
 
+
 @web.route("/ajax/table_settings", methods=['POST'])
 @login_required
 def update_table_settings():
@@ -834,6 +860,7 @@ def publisher_list():
         charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
             .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all()
+
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher")
     else:
@@ -865,8 +892,10 @@ def series_list():
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
 
+            thumbnails = get_thumbnails_for_book_series(entries)
             return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist,
-                                         title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view")
+                                         title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view",
+                                         thumbnails=thumbnails)
     else:
         abort(404)
 
@@ -1150,13 +1179,16 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
     else:
         offset = 0
         limit_all = result_count
+
+    thumbnails = get_thumbnails_for_books(entries)
     return render_title_template('search.html',
                                  adv_searchterm=searchterm,
                                  pagination=pagination,
                                  entries=q[offset:limit_all],
                                  result_count=result_count,
-                                 title=_(u"Advanced Search"), page="advsearch")
-
+                                 title=_(u"Advanced Search"),
+                                 page="advsearch",
+                                 thumbnails=thumbnails)
 
 
 @web.route("/advsearch", methods=['GET'])
@@ -1171,10 +1203,9 @@ def advanced_search_form():
 
 
 @web.route("/cover/<int:book_id>")
-@web.route("/cover/<int:book_id>/<int:resolution>")
 @login_required_if_no_ano
-def get_cover(book_id, resolution=1):
-    return get_book_cover(book_id, resolution)
+def get_cover(book_id):
+    return get_book_cover(book_id)
 
 
 @web.route("/cached-cover/<string:cache_id>")
@@ -1183,6 +1214,12 @@ def get_cached_cover(cache_id):
     return get_cached_book_cover(cache_id)
 
 
+@web.route("/cached-cover-thumbnail/<string:cache_id>")
+@login_required_if_no_ano
+def get_cached_cover_thumbnail(cache_id):
+    return get_cached_book_cover_thumbnail(cache_id)
+
+
 @web.route("/robots.txt")
 def get_robots():
     return send_from_directory(constants.STATIC_DIR, "robots.txt")
@@ -1591,6 +1628,7 @@ def show_book(book_id):
             if media_format.format.lower() in constants.EXTENSIONS_AUDIO:
                 audioentries.append(media_format.format.lower())
 
+        thumbnails = get_thumbnails_for_books([entries])
         return render_title_template('detail.html',
                                      entry=entries,
                                      audioentries=audioentries,
@@ -1602,7 +1640,8 @@ def show_book(book_id):
                                      is_archived=is_archived,
                                      kindle_list=kindle_list,
                                      reader_list=reader_list,
-                                     page="book")
+                                     page="book",
+                                     thumbnails=thumbnails)
     else:
         log.debug(u"Error opening eBook. File does not exist or file is not accessible")
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")

From eef21759cd56c28560ee2007726fab18c8595df5 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Thu, 24 Dec 2020 03:00:26 -0600
Subject: [PATCH 7/9] Fix generate thumbnail task messages, don't load
 thumbnails when the cache file has been deleted

---
 cps/helper.py          | 4 ++++
 cps/tasks/thumbnail.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cps/helper.py b/cps/helper.py
index c33a69e1..add1b067 100644
--- a/cps/helper.py
+++ b/cps/helper.py
@@ -536,9 +536,13 @@ def delete_book(book, calibrepath, book_format):
 def get_thumbnails_for_books(books):
     books_with_covers = list(filter(lambda b: b.has_cover, books))
     book_ids = list(map(lambda b: b.id, books_with_covers))
+    cache = fs.FileSystem()
+    thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS)
+
     return ub.session\
         .query(ub.Thumbnail)\
         .filter(ub.Thumbnail.book_id.in_(book_ids))\
+        .filter(ub.Thumbnail.filename.in_(thumbnail_files))\
         .filter(ub.Thumbnail.expiration > datetime.utcnow())\
         .all()
 
diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py
index e9df170e..70ddc06b 100644
--- a/cps/tasks/thumbnail.py
+++ b/cps/tasks/thumbnail.py
@@ -75,7 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask):
                     else:
                         self.create_book_thumbnail(book, resolution)
 
-                self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count))
+                self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count)
                 self.progress = (1.0 / count) * i
 
         self._handleSuccess()

From 8cc06683df25684793d1d6154a74a3b11408d980 Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Mon, 4 Jan 2021 12:28:05 -0600
Subject: [PATCH 8/9] only python3 supported now

---
 cps/tasks/database.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/cps/tasks/database.py b/cps/tasks/database.py
index 11f0186d..62208c0c 100644
--- a/cps/tasks/database.py
+++ b/cps/tasks/database.py
@@ -20,11 +20,7 @@ from __future__ import division, print_function, unicode_literals
 
 from cps import config, logger
 from cps.services.worker import CalibreTask
-
-try:
-    from urllib.request import urlopen
-except ImportError as e:
-    from urllib2 import urlopen
+from urllib.request import urlopen
 
 
 class TaskReconnectDatabase(CalibreTask):

From 2c8d055ca4e9b92842615516834bb2472348b15e Mon Sep 17 00:00:00 2001
From: mmonkey <mmonkey@gmail.com>
Date: Mon, 4 Jan 2021 12:36:40 -0600
Subject: [PATCH 9/9] support python2.7 for the mean time

---
 cps/tasks/database.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cps/tasks/database.py b/cps/tasks/database.py
index 62208c0c..11f0186d 100644
--- a/cps/tasks/database.py
+++ b/cps/tasks/database.py
@@ -20,7 +20,11 @@ from __future__ import division, print_function, unicode_literals
 
 from cps import config, logger
 from cps.services.worker import CalibreTask
-from urllib.request import urlopen
+
+try:
+    from urllib.request import urlopen
+except ImportError as e:
+    from urllib2 import urlopen
 
 
 class TaskReconnectDatabase(CalibreTask):