Kobo driver: Add an option to directly update metadata in the Kobo

device database, instead of waiting for the Kobo to update the database
after disconnecting. (Preferences->Plugins->Cusotmize the Kobo device plugin)

Merge branch 'master' of https://github.com/davidfor/calibre

src/calibre/devices/kobo/books.py

@@ -8,6 +8,7 @@ from calibre.constants import preferred_encoding, DEBUG
 from calibre import isbytestring, force_unicode
 from calibre.utils.icu import sort_key
 
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.devices.usbms.books import Book as Book_
 from calibre.devices.usbms.books import CollectionsBookList
 from calibre.utils.config_base import prefs
@@ -22,10 +23,13 @@ class Book(Book_):
         from calibre.utils.date import parse_date
 #         debug_print('Book::__init__ - title=', title)
         show_debug = title is not None and title.lower().find("xxxxx") >= 0
+        if other is not None:
+            other.title = title
+            other.published_date = date
         if show_debug:
             debug_print("Book::__init__ - title=", title, 'authors=', authors)
             debug_print("Book::__init__ - other=", other)
-        Book_.__init__(self, prefix, lpath, size, other)
+        super(Book, self).__init__(prefix, lpath, size, other)
 
         if title is not None and len(title) > 0:
             self.title = title
@@ -58,12 +62,14 @@ class Book(Book_):
                             except:
                                 self.datetime = time.gmtime()
 
+        self.kobo_metadata = Metadata(title, self.authors)
         self.contentID          = None
         self.current_shelves    = []
         self.kobo_collections   = []
-        self.kobo_series        = None
-        self.kobo_series_number = None
         self.can_put_on_shelves = True
+        self.kobo_series        = None
+        self.kobo_series_number = None  # Kobo stores the series number as string. And it can have a leading "#".
+        self.kobo_subtitle      = None
 
         if thumbnail_name is not None:
             self.thumbnail = ImageWrapper(thumbnail_name)
@@ -72,6 +78,38 @@ class Book(Book_):
             debug_print("Book::__init__ end - self=", self)
             debug_print("Book::__init__ end - title=", title, 'authors=', authors)
 
+    @property
+    def is_sideloaded(self):
+        # If we don't have a content Id, we don't know what type it is.
+        return self.contentID and self.contentID.startswith("file")
+
+    @property
+    def is_purchased_kepub(self):
+        return self.contentID and not self.contentID.startswith("file")
+
+    def __unicode__(self):
+        '''
+        A string representation of this object, suitable for printing to
+        console
+        '''
+        ans = [u"Kobo metadata:"]
+
+        def fmt(x, y):
+            ans.append(u'%-20s: %s'%(unicode(x), unicode(y)))
+
+        if self.contentID:
+            fmt('Content ID', self.contentID)
+        if self.kobo_series:
+            fmt('Kobo Series', self.kobo_series + ' #%s'%self.kobo_series_number)
+        if self.kobo_subtitle:
+            fmt('Subtitle', self.kobo_subtitle)
+        if self.mime:
+            fmt('MimeType', self.mime)
+
+        ans = u'\n'.join(ans) + u"\n" + self.kobo_metadata.__unicode__()
+
+        return super(Book,self).__unicode__() + u"\n" + ans
+
 
 class ImageWrapper(object):
 

src/calibre/devices/kobo/driver.py

@@ -17,9 +17,14 @@ Extended to support Touch firmware 2.0.0 and later and newer devices by David Fo
 import os, time, shutil, re
 
 from contextlib import closing
+from datetime import datetime
+from calibre import strftime
+from calibre.utils.date import parse_date
 from calibre.devices.usbms.books import BookList
 from calibre.devices.usbms.books import CollectionsBookList
 from calibre.devices.kobo.books import KTCollectionsBookList
+from calibre.ebooks.metadata import authors_to_string
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.devices.kobo.books import Book
 from calibre.devices.kobo.books import ImageWrapper
 from calibre.devices.mime import mime_type_ext
@@ -73,7 +78,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge and David Forrester'
-    version = (2, 3, 3)
+    version = (2, 4, 0)
 
     dbversion = 0
     fwversion = (0,0,0)
@@ -164,18 +169,24 @@ class KOBO(USBMS):
     def device_database_path(self):
         return self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite')
 
-    def device_database_connection(self):
+    def device_database_connection(self, use_row_factory=False):
         import apsw
         db_connection = apsw.Connection(self.device_database_path())
 
+        if use_row_factory:
+            db_connection.setrowtrace(self.row_factory)
+
         return db_connection
 
+    def row_factory(self, cursor, row):
+        return {k[0]: row[i] for i, k in enumerate(cursor.getdescription())}
+
     def get_database_version(self, connection):
         cursor = connection.cursor()
         cursor.execute('SELECT version FROM dbversion')
         try:
             result = cursor.next()
-            dbversion = result[0]
+            dbversion = result['version']
         except StopIteration:
             dbversion = 0
 
@@ -323,7 +334,7 @@ class KOBO(USBMS):
                 traceback.print_exc()
             return changed
 
-        with closing(self.device_database_connection()) as connection:
+        with closing(self.device_database_connection(use_row_factory=True)) as connection:
 
             self.dbversion = self.get_database_version(connection)
             debug_print("Database Version: ", self.dbversion)
@@ -370,20 +381,23 @@ class KOBO(USBMS):
                 cursor.execute(query)
 
             changed = False
-            for i, row in enumerate(cursor):
+            for row in cursor:
                 #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
-                if not hasattr(row[3], 'startswith') or row[3].startswith("file:///usr/local/Kobo/help/"):
+                if not hasattr(row['ContentID'], 'startswith') or row['ContentID'].startswith("file:///usr/local/Kobo/help/"):
                     # These are internal to the Kobo device and do not exist
                     continue
-                path = self.path_from_contentid(row[3], row[5], row[4], oncard)
+                path = self.path_from_contentid(row['ContentID'], row['ContentType'], row['MimeType'], oncard)
                 mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip'
                 # debug_print("mime:", mime)
-
+                if oncard != 'carda' and oncard != 'cardb' and not row['ContentID'].startswith("file:///mnt/sd/"):
-                if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
+                    prefix = self._main_prefix
-                    changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9], row[10])
+                elif oncard == 'carda' and row['ContentID'].startswith("file:///mnt/sd/"):
-                    # print "shortbook: " + path
+                    prefix = self._card_a_prefix
-                elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"):
+                changed = update_booklist(self._main_prefix, path,
-                    changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9], row[10])
+                                          row['Title'], row['Attribution'], mime, row['DateCreated'], row['ContentType'],
+                                          row['ImageId'], row['ReadStatus'], row['MimeType'], row['___ExpirationStatus'],
+                                          row['FavouritesIndex'], row['Accessibility']
+                                          )
 
                 if changed:
                     need_sync = True
@@ -769,21 +783,28 @@ class KOBO(USBMS):
             cursor.close()
 
     def set_readstatus(self, connection, ContentID, ReadStatus):
+        debug_print("Kobo::set_readstatus - ContentID=%s, ReadStatus=%d" % (ContentID, ReadStatus))
         cursor = connection.cursor()
         t = (ContentID,)
         cursor.execute('select DateLastRead, ReadStatus  from Content where BookID is Null and ContentID = ?', t)
         try:
             result = cursor.next()
-            datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00'
+            datelastread = result['DateLastRead']
-            current_ReadStatus = result[1]
+            current_ReadStatus = result['ReadStatus']
         except StopIteration:
-            datelastread = '1970-01-01T00:00:00'
+            datelastread = None
             current_ReadStatus = 0
 
         if not ReadStatus == current_ReadStatus:
+            if ReadStatus == 0:
+                datelastread = None
+            else:
+                datelastread = 'CURRENT_TIMESTAMP' if datelastread is None else datelastread
+
             t = (ReadStatus, datelastread, ContentID,)
 
             try:
+                debug_print("Kobo::set_readstatus - Making change - ContentID=%s, ReadStatus=%d, DateLastRead=%s" % (ContentID, ReadStatus, datelastread))
                 cursor.execute('update content set ReadStatus=?,FirstTimeReading=\'false\',DateLastRead=? where BookID is Null and ContentID = ?', t)
             except:
                 debug_print('    Database Exception: Unable to update ReadStatus')
@@ -1526,7 +1547,6 @@ class KOBOTOUCH(KOBO):
 
     def books(self, oncard=None, end_session=True):
         debug_print("KoboTouch:books - oncard='%s'"%oncard)
-        from calibre.ebooks.metadata.meta import path_to_ext
         self.debugging_title = self.get_debugging_title()
 
         dummy_bl = self.booklist_class(None, None, None)
@@ -1577,15 +1597,19 @@ class KOBOTOUCH(KOBO):
         for idx,b in enumerate(bl):
             bl_cache[b.lpath] = idx
 
-        def update_booklist(prefix, path, title, authors, mime, date, ContentID, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex,
+        def update_booklist(prefix, path, ContentID, ContentType, MimeType, ImageID,
-                            accessibility, isdownloaded, series, seriesnumber, userid, bookshelves):
+                            title, authors, DateCreated, Description, Publisher, series, seriesnumber,
+                            ISBN, Language, Subtitle,
+                            readstatus, expired, favouritesindex, accessibility, isdownloaded,
+                            userid, bookshelves
+                            ):
             show_debug = self.is_debugging_title(title)
 #            show_debug = authors == 'L. Frank Baum'
             if show_debug:
                 debug_print("KoboTouch:update_booklist - title='%s'"%title, "ContentType=%s"%ContentType, "isdownloaded=", isdownloaded)
                 debug_print(
-                    "         prefix=%s, mime=%s, date=%s, readstatus=%d, MimeType=%s, expired=%d, favouritesindex=%d, accessibility=%d, isdownloaded=%s"%
+                    "         prefix=%s, DateCreated=%s, readstatus=%d, MimeType=%s, expired=%d, favouritesindex=%d, accessibility=%d, isdownloaded=%s"%
-                (prefix, mime, date, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded,))
+                (prefix, DateCreated, readstatus, MimeType, expired, favouritesindex, accessibility, isdownloaded,))
             changed = False
             try:
                 lpath = path.partition(self.normalize_path(prefix))[2]
@@ -1658,6 +1682,23 @@ class KOBOTOUCH(KOBO):
                 path = self.normalize_path(path)
                 # print "Normalized FileName: " + path
 
+                # Collect the Kobo metadata
+                kobo_metadata = Metadata(title, [a.strip() for a in authors.split("&")])
+                kobo_metadata.series       = series
+                kobo_metadata.series_index = seriesnumber
+                kobo_metadata.comments     = Description
+                kobo_metadata.publisher    = Publisher
+                kobo_metadata.language     = Language
+                kobo_metadata.isbn         = ISBN
+                if DateCreated is not None:
+                    try:
+                        kobo_metadata.pubdate     = parse_date(DateCreated, assume_utc=True)
+                    except:
+                        try:
+                            kobo_metadata.pubdate = datetime.strptime(DateCreated, "%Y-%m-%dT%H:%M:%S.%fZ")
+                        except:
+                            debug_print("KoboTouch:update_booklist - Cannot convert date - DateCreated='%s'"%DateCreated)
+
                 idx = bl_cache.get(lpath, None)
                 if idx is not None:  # and not (accessibility == 1 and isdownloaded == 'false'):
                     if show_debug:
@@ -1690,9 +1731,12 @@ class KOBOTOUCH(KOBO):
                     if show_debug:
                         debug_print("KoboTouch:update_booklist - ContentID='%s'"%ContentID)
                     bl[idx].contentID           = ContentID
+                    bl[idx].kobo_metadata       = kobo_metadata
                     bl[idx].kobo_series         = series
                     bl[idx].kobo_series_number  = seriesnumber
+                    bl[idx].kobo_subtitle       = Subtitle
                     bl[idx].can_put_on_shelves  = allow_shelves
+                    bl[idx].mime                = MimeType
 
                     if lpath in playlist_map:
                         bl[idx].device_collections  = playlist_map.get(lpath,[])
@@ -1710,19 +1754,19 @@ class KOBOTOUCH(KOBO):
                         debug_print('KoboTouch:update_booklist - idx is none')
                     try:
                         if os.path.exists(self.normalize_path(os.path.join(prefix, lpath))):
-                            book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
+                            book = self.book_from_path(prefix, lpath, title, authors, MimeType, DateCreated, ContentType, ImageID)
                         else:
                             if isdownloaded == 'true':  # A recommendation or preview is OK to not have a file
                                 debug_print("    Strange:  The file: ", prefix, lpath, " does not exist!")
                                 title = "FILE MISSING: " + title
-                            book =  self.book_class(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=0)
+                            book =  self.book_class(prefix, lpath, title, authors, MimeType, DateCreated, ContentType, ImageID, size=0)
                             if show_debug:
                                 debug_print('KoboTouch:update_booklist - book file does not exist. ContentID="%s"'%ContentID)
 
                     except Exception as e:
                         debug_print("KoboTouch:update_booklist - exception creating book: '%s'"%str(e))
                         debug_print("        prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors,
-                                    "mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID)
+                                    "MimeType: ", MimeType, "DateCreated: ", DateCreated, "ContentType: ", ContentType, "ImageID: ", ImageID)
                         raise
 
                     if show_debug:
@@ -1740,8 +1784,10 @@ class KOBOTOUCH(KOBO):
                     book.current_shelves    = bookshelves
                     book.kobo_collections   = kobo_collections
                     book.contentID          = ContentID
+                    book.kobo_metadata      = kobo_metadata
                     book.kobo_series        = series
                     book.kobo_series_number = seriesnumber
+                    book.kobo_subtitle      = Subtitle
                     book.can_put_on_shelves = allow_shelves
 #                    debug_print('KoboTouch:update_booklist - title=', title, 'book.device_collections', book.device_collections)
 
@@ -1770,7 +1816,7 @@ class KOBOTOUCH(KOBO):
             values = (ContentID, )
             cursor.execute(query, values)
             for i, row in enumerate(cursor):
-                bookshelves.append(row[0])
+                bookshelves.append(row['ShelfName'])
 
             cursor.close()
 #            debug_print("KoboTouch:get_bookshelvesforbook - count bookshelves=" + unicode(count_bookshelves))
@@ -1778,30 +1824,28 @@ class KOBOTOUCH(KOBO):
 
         self.debug_index = 0
 
-        with closing(self.device_database_connection()) as connection:
+        with closing(self.device_database_connection(use_row_factory=True)) as connection:
             debug_print("KoboTouch:books - reading device database")
 
             self.dbversion = self.get_database_version(connection)
             debug_print("Database Version: ", self.dbversion)
 
-            cursor = connection.cursor()
-
             self.bookshelvelist = self.get_bookshelflist(connection)
             debug_print("KoboTouch:books - shelf list:", self.bookshelvelist)
 
-            columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageID, ReadStatus'
+            columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageId, ReadStatus, Description, Publisher, Language '
             if self.dbversion >= 16:
                 columns += ', ___ExpirationStatus, FavouritesIndex, Accessibility'
             else:
                 columns += ', -1 as ___ExpirationStatus, -1 as FavouritesIndex, -1 as Accessibility'
             if self.dbversion >= 33:
-                columns += ', IsDownloaded'
+                columns += ', IsDownloaded, ISBN'
             else:
-                columns += ', "1" as IsDownloaded'
+                columns += ', "1" as IsDownloaded, null AS ISBN'
             if self.supports_series():
-                columns += ", Series, SeriesNumber, ___UserID, ExternalId"
+                columns += ", Series, SeriesNumber, ___UserID, ExternalId, Subtitle"
             else:
-                columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId'
+                columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId, null as Subtitle'
 
             where_clause = ''
             if self.supports_kobo_archive() or self.supports_overdrive():
@@ -1855,6 +1899,8 @@ class KOBOTOUCH(KOBO):
 
             query = 'SELECT ' + columns + ' FROM content ' + where_clause + card_condition
             debug_print("KoboTouch:books - query=", query)
+
+            cursor = connection.cursor()
             try:
                 cursor.execute(query)
             except Exception as e:
@@ -1862,38 +1908,43 @@ class KOBOTOUCH(KOBO):
                 if not (any_in(err, '___ExpirationStatus', 'FavouritesIndex', 'Accessibility', 'IsDownloaded', 'Series', 'ExternalId')):
                     raise
                 query= ('SELECT Title, Attribution, DateCreated, ContentID, MimeType, ContentType, '
-                        'ImageID, ReadStatus, -1 AS ___ExpirationStatus, "-1" AS '
+                        'ImageId, ReadStatus, -1 AS ___ExpirationStatus, "-1" AS FavouritesIndex, '
-                        'FavouritesIndex, -1 AS Accessibility, 1 AS IsDownloaded, NULL AS Series, NULL AS SeriesNumber '
+                        'null AS ISBN, Language '
+                        '-1 AS Accessibility, 1 AS IsDownloaded, NULL AS Series, NULL AS SeriesNumber, null as Subtitle '
                         'FROM content '
                         'WHERE BookID IS NULL'
                         )
                 cursor.execute(query)
 
             changed = False
-            for i, row in enumerate(cursor):
+            i = 0
-                #  self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
+            for row in cursor:
-                show_debug = self.is_debugging_title(row[0])
+                i += 1
+#                 self.report_progress((i) / float(books_on_device), _('Getting list of books on device...'))
+                show_debug = self.is_debugging_title(row['Title'])
                 if show_debug:
                     debug_print("KoboTouch:books - looping on database - row=%d" % i)
-                    debug_print("KoboTouch:books - title='%s'"%row[0], "authors=", row[1])
+                    debug_print("KoboTouch:books - title='%s'"%row['Title'], "authors=", row['Attribution'])
                     debug_print("KoboTouch:books - row=", row)
-                if not hasattr(row[3], 'startswith') or row[3].lower().startswith(
+                if not hasattr(row['ContentID'], 'startswith') or row['ContentID'].lower().startswith(
-                        "file:///usr/local/kobo/help/") or row[3].lower().startswith("/usr/local/kobo/help/"):
+                        "file:///usr/local/kobo/help/") or row['ContentID'].lower().startswith("/usr/local/kobo/help/"):
                     # These are internal to the Kobo device and do not exist
                     continue
-                externalId = None if row[15] and len(row[15]) == 0 else row[15]
+                externalId = None if row['ExternalId'] and len(row['ExternalId']) == 0 else row['ExternalId']
-                path = self.path_from_contentid(row[3], row[5], row[4], oncard, externalId)
+                path = self.path_from_contentid(row['ContentID'], row['ContentType'], row['MimeType'], oncard, externalId)
-                mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/x-kobo-epub+zip'
-                # debug_print("mime:", mime)
                 if show_debug:
-                    debug_print("KoboTouch:books - path='%s'"%path, "  ContentID='%s'"%row[3], " externalId=%s" % externalId)
+                    debug_print("KoboTouch:books - path='%s'"%path, "  ContentID='%s'"%row['ContentID'], " externalId=%s" % externalId)
 
-                bookshelves = get_bookshelvesforbook(connection, row[3])
+                bookshelves = get_bookshelvesforbook(connection, row['ContentID'])
 
                 prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix
-                changed = update_booklist(prefix, path, row[0], row[1], mime, row[2], row[3], row[5],
+                changed = update_booklist(prefix, path, row['ContentID'], row['ContentType'], row['MimeType'], row['ImageId'],
-                                          row[6], row[7], row[4], row[8], int(row[9]), row[10], row[11],
+                                          row['Title'], row['Attribution'], row['DateCreated'], row['Description'], row['Publisher'],
-                                          row[12], row[13], row[14], bookshelves)
+                                          row['Series'], row['SeriesNumber'], row['ISBN'], row['Language'], row['Subtitle'],
+                                          row['ReadStatus'], row['___ExpirationStatus'],
+                                          int(row['FavouritesIndex']), row['Accessibility'], row['IsDownloaded'],
+                                          row['___UserID'], bookshelves
+                                          )
 
                 if changed:
                     need_sync = True
@@ -2281,6 +2332,10 @@ class KOBOTOUCH(KOBO):
             ContentType = 901
         return ContentType
 
+    def set_plugboards(self, plugboards, pb_func):
+        self.plugboards = plugboards
+        self.plugboard_func = pb_func
+
     def update_device_database_collections(self, booklists, collections_attributes, oncard):
         debug_print("KoboTouch:update_device_database_collections - oncard='%s'"%oncard)
         if self.modify_database_check("update_device_database_collections") is False:
@@ -2313,6 +2368,8 @@ class KOBOTOUCH(KOBO):
         create_collections       = self.create_collections
         delete_empty_collections = self.delete_empty_collections
         update_series_details    = self.update_series_details
+        update_core_metadata     = self.update_core_metadata
+        update_purchased_kepubs  = self.update_purchased_kepubs
         debugging_title          = self.get_debugging_title()
         debug_print("KoboTouch:update_device_database_collections - set_debugging_title to '%s'" % debugging_title)
         booklists.set_debugging_title(debugging_title)
@@ -2328,7 +2385,7 @@ class KOBOTOUCH(KOBO):
         # the last book from the collection the list of books is empty
         # and the removal of the last book would not occur
 
-        with closing(self.device_database_connection()) as connection:
+        with closing(self.device_database_connection(use_row_factory=True)) as connection:
 
             if self.manage_collections:
                 if collections:
@@ -2422,19 +2479,32 @@ class KOBOTOUCH(KOBO):
                         self.reset_favouritesindex(connection, oncard)
 
             # Set the series info and cleanup the bookshelves only if the firmware supports them and the user has set the options.
-            if (self.supports_bookshelves and self.manage_collections or self.supports_series()) and (bookshelf_attribute or update_series_details):
+            if (self.supports_bookshelves and self.manage_collections or self.supports_series()) and (
+                    bookshelf_attribute or update_series_details or update_core_metadata):
                 debug_print("KoboTouch:update_device_database_collections - managing bookshelves and series.")
 
                 self.series_set        = 0
+                self.core_metadata_set = 0
                 books_in_library       = 0
                 for book in booklists:
-                    if book.application_id is not None:
+                    # debug_print("KoboTouch:update_device_database_collections - book.title=%s, book.contentID=%s" % (book.title, book.contentID))
+                    if book.application_id is not None and book.contentID is not None:
                         books_in_library += 1
                         show_debug = self.is_debugging_title(book.title)
                         if show_debug:
                             debug_print("KoboTouch:update_device_database_collections - book.title=%s" % book.title)
-                        if update_series_details:
+                            debug_print(
-                            self.set_series(connection, book)
+                                "KoboTouch:update_device_database_collections - contentId=%s,"
+                                "update_core_metadata=%s,update_purchased_kepubs=%s, book.is_sideloaded=%s" % (
+                                book.contentID, update_core_metadata, update_purchased_kepubs, book.is_sideloaded))
+                        if update_core_metadata and (update_purchased_kepubs or book.is_sideloaded):
+                            if show_debug:
+                                debug_print("KoboTouch:update_device_database_collections - calling set_core_metadata")
+                            self.set_core_metadata(connection, book)
+                        elif update_series_details:
+                            if show_debug:
+                                debug_print("KoboTouch:update_device_database_collections - calling set_core_metadata - series only")
+                            self.set_core_metadata(connection, book, series_only=True)
                         if self.manage_collections and bookshelf_attribute:
                             if show_debug:
                                 debug_print("KoboTouch:update_device_database_collections - about to remove a book from shelves book.title=%s" % book.title)
@@ -2444,6 +2514,8 @@ class KOBOTOUCH(KOBO):
                     debug_print("KoboTouch:update_device_database_collections - about to clear empty bookshelves")
                     self.delete_empty_bookshelves(connection)
                 debug_print("KoboTouch:update_device_database_collections - Number of series set=%d Number of books=%d" % (self.series_set, books_in_library))
+                debug_print("KoboTouch:update_device_database_collections - Number of core metadata set=%d Number of books=%d" % (
+                    self.core_metadata_set, books_in_library))
 
                 self.dump_bookshelves(connection)
 
@@ -2752,8 +2824,8 @@ class KOBOTOUCH(KOBO):
         cursor = connection.cursor()
         cursor.execute(query)
 #        count_bookshelves = 0
-        for i, row in enumerate(cursor):
+        for row in cursor:
-            bookshelves.append(row[0])
+            bookshelves.append(row['Name'])
 #            count_bookshelves = i + 1
 
         cursor.close()
@@ -2790,7 +2862,7 @@ class KOBOTOUCH(KOBO):
             if show_debug:
                 debug_print('        Did not find a record - adding')
             cursor.execute(addquery, add_values)
-        elif result[0] == 'true':
+        elif result['_IsDeleted'] == 'true':
             if show_debug:
                 debug_print('        Found a record - updating - result=', result)
             cursor.execute(updatequery, update_values)
@@ -2839,8 +2911,9 @@ class KOBOTOUCH(KOBO):
             if show_debug:
                 debug_print('        Did not find a record - adding shelf "%s"' % bookshelf_name)
             cursor.execute(addquery, add_values)
-        elif result[2] == 'true':
+        elif result['_IsDeleted'] == 'true':
-            debug_print('KoboTouch:check_for_bookshelf - Shelf "%s" is deleted - undeleting. result[2]="%s"' % (bookshelf_name, unicode(result[2])))
+            debug_print("KoboTouch:check_for_bookshelf - Shelf '%s' is deleted - undeleting. result['_IsDeleted']='%s'" % (
+                bookshelf_name, unicode(result['_IsDeleted'])))
             cursor.execute(updatequery, test_values)
 
         cursor.close()
@@ -2879,6 +2952,7 @@ class KOBOTOUCH(KOBO):
 
         debug_print("KoboTouch:remove_from_bookshelf - end")
 
+    # No longer used, but keep for a little bit.
     def set_series(self, connection, book):
         show_debug = self.is_debugging_title(book.title)
         if show_debug:
@@ -2921,6 +2995,125 @@ class KOBOTOUCH(KOBO):
         if show_debug:
             debug_print("KoboTouch:set_series - end")
 
+    def set_core_metadata(self, connection, book, series_only=False):
+        # debug_print('KoboTouch:set_core_metadata book="%s"' % book.title)
+        show_debug = self.is_debugging_title(book.title)
+        if show_debug:
+            debug_print('KoboTouch:set_core_metadata book="%s", series_only="%s"' % (book, series_only))
+
+        plugboard = None
+        if self.plugboard_func and not series_only:
+            if book.contentID.endswith('.kepub.epub') or os.path.splitext(book.contentID)[1] == "":
+                extension = 'kepub'
+            else:
+                extension = os.path.splitext(book.contentID)[1][1:]
+            plugboard = self.plugboard_func(self.__class__.__name__, extension, self.plugboards)
+
+            # If the book is a kepub, and there is no kepub plugboard, use the epub plugboard if it exists.
+            if not plugboard and extension == 'kepub':
+                plugboard = self.plugboard_func(self.__class__.__name__, 'epub', self.plugboards)
+
+        if plugboard is not None:
+            newmi = book.deepcopy_metadata()
+            newmi.template_to_attribute(book, plugboard)
+        else:
+            newmi = book
+
+        update_query  = 'UPDATE content SET '
+        update_values = []
+        set_clause    = ''
+        changes_found = False
+        kobo_metadata = book.kobo_metadata
+
+        series_changed = not (newmi.series == kobo_metadata.series)
+        series_number_changed = False
+        if kobo_metadata.series_index:
+            try:
+                kobo_series_number = float(book.kobo_series_number)
+            except:
+                kobo_series_number = None
+            series_number_changed = not (kobo_series_number == newmi.series_index)
+
+        if series_changed or series_number_changed:
+            if newmi.series:
+                new_series = newmi.series
+                new_series_number = "%g" % newmi.series_index
+            else:
+                new_series = None
+                new_series_number = None
+
+            update_values.append(new_series)
+            set_clause += ', Series = ? '
+            update_values.append(new_series_number)
+            set_clause += ', SeriesNumber = ? '
+
+        if not series_only:
+            if not (newmi.title == kobo_metadata.title):
+                update_values.append(newmi.title)
+                set_clause += ', Title = ? '
+
+            if not (authors_to_string(newmi.authors) == authors_to_string(kobo_metadata.authors)):
+                update_values.append(authors_to_string(newmi.authors))
+                set_clause += ', Attribution = ? '
+
+            if not (newmi.publisher == kobo_metadata.publisher):
+                update_values.append(newmi.publisher)
+                set_clause += ', Publisher = ? '
+
+            if not (newmi.pubdate == kobo_metadata.pubdate):
+                pubdate_string = strftime(self.TIMESTAMP_STRING, newmi.pubdate) if newmi.pubdate else None
+                update_values.append(pubdate_string)
+                set_clause += ', DateCreated = ? '
+
+            if not (newmi.comments == kobo_metadata.comments):
+                update_values.append(newmi.comments)
+                set_clause += ', Description = ? '
+
+            if not (newmi.isbn == kobo_metadata.isbn):
+                update_values.append(newmi.isbn)
+                set_clause += ', ISBN = ? '
+
+            if not (newmi.language == kobo_metadata.language):
+                update_values.append(newmi.language)
+                set_clause += ', Language = ? '
+
+            if self.update_subtitle:
+                if self.subtitle_template is None or self.subtitle_template == '':
+                    new_subtitle = None
+                else:
+                    pb = [(self.subtitle_template, 'subtitle')]
+                    book.template_to_attribute(book, pb)
+                    new_subtitle = book.subtitle
+                if (new_subtitle and (book.kobo_subtitle is None or not book.subtitle == book.kobo_subtitle)) or \
+                    (new_subtitle is None and book.kobo_subtitle is not None):
+                    update_values.append(new_subtitle)
+                    set_clause += ', Subtitle = ? '
+
+        if len(set_clause) > 0:
+            update_query += set_clause[1:]
+            changes_found = True
+            if show_debug:
+                debug_print('KoboTouch:set_core_metadata set_clause="%s"' % set_clause)
+                debug_print('KoboTouch:set_core_metadata update_values="%s"' % update_values)
+        if changes_found:
+            update_query += 'WHERE ContentID = ? AND BookID IS NULL'
+            update_values.append(book.contentID)
+            cursor = connection.cursor()
+            try:
+                if show_debug:
+                    debug_print('KoboTouch:set_core_metadata - about to set - parameters:', update_values)
+                    debug_print('KoboTouch:set_core_metadata - about to set - update_query:', update_query)
+                cursor.execute(update_query, update_values)
+                self.core_metadata_set += 1
+            except:
+                debug_print('    Database Exception:  Unable to set the core metadata')
+                raise
+            finally:
+                cursor.close()
+
+        if show_debug:
+            debug_print("KoboTouch:set_core_metadata - end")
+
     @classmethod
     def config_widget(cls):
         # TODO: Cleanup the following
@@ -2975,7 +3168,11 @@ class KOBOTOUCH(KOBO):
         c.add_opt('show_recommendations', default=False)
 
         c.add_opt('update_series', default=True)
+        c.add_opt('update_core_metadata', default=False)
+        c.add_opt('update_purchased_kepubs', default=False)
         c.add_opt('update_device_metadata', default=True)
+        c.add_opt('update_subtitle', default=False)
+        c.add_opt('subtitle_template', default=None)
 
         c.add_opt('modify_css', default=False)
         c.add_opt('override_kobo_replace_existing', default=True)  # Overriding the replace behaviour is how the driver has always worked.
@@ -3166,6 +3363,27 @@ class KOBOTOUCH(KOBO):
     def update_series_details(self):
         return self.update_device_metadata and self.get_pref('update_series') and self.supports_series()
 
+    @property
+    def update_subtitle(self):
+        # Subtitle was added to the database at the same time as the series support.
+        return self.update_device_metadata and self.supports_series() and self.subtitle_template is not None
+
+    @property
+    def subtitle_template(self):
+        subtitle_template = self.get_pref('subtitle_template')
+        if subtitle_template is not None:
+            subtitle_template = subtitle_template.strip()
+        subtitle_template = subtitle_template.strip() if subtitle_template is not None else None
+        return subtitle_template
+
+    @property
+    def update_core_metadata(self):
+        return self.update_device_metadata and self.get_pref('update_core_metadata')
+
+    @property
+    def update_purchased_kepubs(self):
+        return self.update_device_metadata and self.get_pref('update_purchased_kepubs')
+
     @classmethod
     def get_debugging_title(cls):
         debugging_title = cls.get_pref('debugging_title')

src/calibre/devices/kobo/kobotouch_config.py

@@ -9,11 +9,13 @@ __docformat__ = 'restructuredtext en'
 
 import textwrap
 
-from PyQt5.Qt import (QLabel, QGridLayout, QLineEdit, QVBoxLayout,
+from PyQt5.Qt import (QWidget, QLabel, QGridLayout, QLineEdit, QVBoxLayout,
-                      QDialog, QDialogButtonBox, QCheckBox)
+                      QDialog, QDialogButtonBox, QCheckBox, QPushButton)
 
 from calibre.gui2.device_drivers.tabbed_device_config import TabbedDeviceConfig, DeviceConfigTab, DeviceOptionsGroupBox
 from calibre.devices.usbms.driver import debug_print
+from calibre.gui2 import error_dialog
+from calibre.gui2.dialogs.template_dialog import TemplateDialog
 
 
 def wrap_msg(msg):
@@ -63,12 +65,9 @@ class KOBOTOUCHConfig(TabbedDeviceConfig):
         return self._device()
 
     def validate(self):
-        if hasattr(self, 'formats'):
+        validated = super(KOBOTOUCHConfig, self).validate()
-            if not self.formats.validate():
+        validated &= self.tab2.validate()
-                return False
+        return validated
-            if not self.template.validate():
-                return False
-        return True
 
     @property
     def book_uploads_options(self):
@@ -113,6 +112,11 @@ class KOBOTOUCHConfig(TabbedDeviceConfig):
         p['show_archived_books'] = self.show_archived_books
 
         p['update_series'] = self.update_series
+        p['update_core_metadata'] = self.update_core_metadata
+        p['update_purchased_kepubs'] = self.update_purchased_kepubs
+        p['subtitle_template'] = self.subtitle_template
+        p['update_subtitle'] = self.update_subtitle
+
         p['modify_css'] = self.modify_css
         p['override_kobo_replace_existing'] = self.override_kobo_replace_existing
 
@@ -142,6 +146,8 @@ class Tab1Config(DeviceConfigTab):  # {{{
         self.book_uploads_options = BookUploadsGroupBox(self, device)
         self.l.addWidget(self.book_uploads_options)
         self.addDeviceWidget(self.book_uploads_options)
+
+        self.l.addStretch()
 # }}}
 
 
@@ -164,6 +170,12 @@ class Tab2Config(DeviceConfigTab):  # {{{
         self.advanced_options = AdvancedGroupBox(self, device)
         self.l.addWidget(self.advanced_options)
         self.addDeviceWidget(self.advanced_options)
+
+        self.l.addStretch()
+
+    def validate(self):
+        return self.metadata_options.validate()
+
 # }}}
 
 
@@ -199,7 +211,6 @@ class BookUploadsGroupBox(DeviceOptionsGroupBox):
 
         self.options_layout.addWidget(self.modify_css_checkbox, 0, 0, 1, 2)
         self.options_layout.addWidget(self.override_kobo_replace_existing_checkbox, 1, 0, 1, 2)
-        self.options_layout.setRowStretch(2, 1)
 
     @property
     def modify_css(self):
@@ -255,7 +266,6 @@ class CollectionsGroupBox(DeviceOptionsGroupBox):
         self.options_layout.addWidget(self.delete_empty_collections_checkbox, 3, 0, 1, 2)
         self.options_layout.addWidget(self.ignore_collections_names_label,    4, 0, 1, 1)
         self.options_layout.addWidget(self.ignore_collections_names_edit,     4, 1, 1, 1)
-        self.options_layout.setRowStretch(4, 1)
 
     @property
     def manage_collections(self):
@@ -306,7 +316,6 @@ class CoversGroupBox(DeviceOptionsGroupBox):
 
         self.options_layout.addWidget(self.keep_cover_aspect_checkbox,    0, 0, 1, 1)
         self.options_layout.addWidget(self.upload_grayscale_checkbox,     1, 0, 1, 1)
-        self.options_layout.setRowStretch(2, 1)
 
     @property
     def upload_covers(self):
@@ -357,7 +366,6 @@ class DeviceListGroupBox(DeviceOptionsGroupBox):
         self.options_layout.addWidget(self.show_recommendations_checkbox, 0, 0, 1, 1)
         self.options_layout.addWidget(self.show_archived_books_checkbox,  1, 0, 1, 1)
         self.options_layout.addWidget(self.show_previews_checkbox,        2, 0, 1, 1)
-        self.options_layout.setRowStretch(3, 1)
 
     @property
     def show_recommendations(self):
@@ -411,7 +419,6 @@ class AdvancedGroupBox(DeviceOptionsGroupBox):
         self.options_layout.addWidget(self.support_newer_firmware_checkbox,   0, 0, 1, 2)
         self.options_layout.addWidget(self.debugging_title_label,             1, 0, 1, 1)
         self.options_layout.addWidget(self.debugging_title_edit,              1, 1, 1, 1)
-        self.options_layout.setRowStretch(2, 2)
 
     @property
     def support_newer_firmware(self):
@@ -445,17 +452,136 @@ class MetadataGroupBox(DeviceOptionsGroupBox):
                                'Enable if you wish to set series information.'),
                              device.get_pref('update_series')
                              )
-        self.options_layout.addWidget(self.update_series_checkbox, 0, 0, 1, 1)
+        self.update_core_metadata_checkbox = create_checkbox(
-        self.options_layout.setRowStretch(1, 1)
+                             _("Update metadata on Book Details pages"),
+                             _('This will update the metadata in the device database when the device is connected. '
+                               'The metadata updated is displayed on the device in the library and the book details page. '
+                               'This is the Title, Authors, Comments/Synopsis, Series name and number, Publisher and Published Date, ISBN and Language. '
+                               'If a metadata plugboard exists for the device and book format, this will be used to set the metadata.'
+                               ),
+                             device.get_pref('update_core_metadata')
+                             )
+
+        self.update_purchased_kepubs_checkbox = create_checkbox(
+                             _("Update purchased books"),
+                             _('Update books purchased from Kobo and downloaded to the device.'
+                               ),
+                             device.get_pref('update_purchased_kepubs')
+                             )
+        self.update_subtitle_checkbox = create_checkbox(
+                             _("Subtitle"),
+                             _('Update the subtitle on the device using a template.'),
+                             device.get_pref('update_subtitle')
+                             )
+        self.subtitle_template_edit = TemplateConfig(
+                            device.get_pref('subtitle_template'),
+                            tooltip=_("Enter a template to use to set the subtitle. "
+                                      "If the template is empty, the subtitle will be cleared."
+                                      )
+                            )
+
+        self.options_layout.addWidget(self.update_series_checkbox, 0, 0, 1, 2)
+        self.options_layout.addWidget(self.update_core_metadata_checkbox, 1, 0, 1, 2)
+        self.options_layout.addWidget(self.update_subtitle_checkbox, 2, 0, 1, 1)
+        self.options_layout.addWidget(self.subtitle_template_edit, 2, 1, 1, 1)
+        self.options_layout.addWidget(self.update_purchased_kepubs_checkbox, 3, 0, 1, 2)
+
+        self.update_core_metadata_checkbox.clicked.connect(self.update_core_metadata_checkbox_clicked)
+        self.update_subtitle_checkbox.clicked.connect(self.update_subtitle_checkbox_clicked)
+        self.update_core_metadata_checkbox_clicked(device.get_pref('update_core_metadata'))
+        self.update_subtitle_checkbox_clicked(device.get_pref('update_subtitle'))
+
+    def update_core_metadata_checkbox_clicked(self, checked):
+        self.update_series_checkbox.setEnabled(not checked)
+        self.subtitle_template_edit.setEnabled(checked)
+        self.update_subtitle_checkbox.setEnabled(checked)
+        self.update_subtitle_checkbox_clicked(self.update_subtitle)
+        self.update_purchased_kepubs_checkbox.setEnabled(checked)
+
+    def update_subtitle_checkbox_clicked(self, checked):
+        self.subtitle_template_edit.setEnabled(checked and self.update_core_metadata)
+
+    def edit_template(self):
+        t = TemplateDialog(self, self.template)
+        t.setWindowTitle(_('Edit template'))
+        if t.exec_():
+            self.t.setText(t.rule[1])
+
+    def validate(self):
+        if self.update_subtitle and not self.subtitle_template_edit.validate():
+            return False
+        return True
 
     @property
     def update_series(self):
         return self.update_series_checkbox.isChecked()
 
+    @property
+    def update_core_metadata(self):
+        return self.update_core_metadata_checkbox.isChecked()
+
+    @property
+    def update_purchased_kepubs(self):
+        return self.update_purchased_kepubs_checkbox.isChecked()
+
     @property
     def update_device_metadata(self):
         return self.isChecked()
 
+    @property
+    def subtitle_template(self):
+        return self.subtitle_template_edit.template
+
+    @property
+    def update_subtitle(self):
+        return self.update_subtitle_checkbox.isChecked()
+
+
+class TemplateConfig(QWidget):  # {{{
+
+    def __init__(self, val, tooltip=None):
+        QWidget.__init__(self)
+        self.t = t = QLineEdit(self)
+        t.setText(val or '')
+        t.setCursorPosition(0)
+        self.setMinimumWidth(300)
+        self.l = l = QGridLayout(self)
+        self.setLayout(l)
+        l.addWidget(t, 1, 0, 1, 1)
+        b = self.b = QPushButton(_('&Template editor'))
+        l.addWidget(b, 1, 1, 1, 1)
+        b.clicked.connect(self.edit_template)
+        self.setToolTip(tooltip)
+
+    @property
+    def template(self):
+        return unicode(self.t.text()).strip()
+
+    @template.setter
+    def template(self, template):
+        self.t.setText(template)
+
+    def edit_template(self):
+        t = TemplateDialog(self, self.template)
+        t.setWindowTitle(_('Edit template'))
+        if t.exec_():
+            self.t.setText(t.rule[1])
+
+    def validate(self):
+        from calibre.utils.formatter import validation_formatter
+
+        tmpl = self.template
+        try:
+            validation_formatter.validate(tmpl)
+            return True
+        except Exception as err:
+            error_dialog(self, _('Invalid template'),
+                    '<p>'+_('The template "%s" is invalid:')%tmpl +
+                    '<br>'+unicode(err), show=True)
+
+            return False
+# }}}
+
 
 if __name__ == '__main__':
     from calibre.gui2 import Application