Fix #7046 (Removing a publisher results in an error). Add tweak to control how articles in title sort are handled. Add a new format type 'device_db' to plugboards to control the metadata displayed in book lists on SONY devices.

resources/default_tweaks.py

@@ -83,6 +83,16 @@ title_series_sorting = 'library_order'
 # strictly_alphabetic, it would remain "The Client".
 save_template_title_series_sorting = 'library_order'
 
+# Set the list of words that are to be considered 'articles' when computing the
+# title sort strings. The list is a regular expression, with the articles
+# separated by 'or' bars. Comparisons are case insensitive, and that cannot be
+# changed. Changes to this tweak won't have an effect until the book is modified
+# in some way. If you enter an invalid pattern, it is silently ignored.
+# To disable use the expression: '^$'
+# Default: '^(A|The|An)\s+'
+title_sort_articles=r'^(A|The|An)\s+'
+
+
 # Specify a folder that calibre should connect to at startup using
 # connect_to_folder. This must be a full path to the folder. If the folder does
 # not exist when calibre starts, it is ignored. If there are '\' characters in

src/calibre/devices/prs505/driver.py

@@ -14,6 +14,7 @@ from calibre.devices.prs505 import CACHE_XML
 from calibre.devices.prs505.sony_cache import XMLCache
 from calibre import __appname__
 from calibre.devices.usbms.books import CollectionsBookList
+from calibre.utils.config import tweaks
 
 class PRS505(USBMS):
 
@@ -63,6 +64,8 @@ class PRS505(USBMS):
                     'series, tags, authors'
     EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags'])
 
+    plugboard = None
+
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
@@ -150,7 +153,7 @@ class PRS505(USBMS):
         else:
             collections = []
         debug_print('PRS505: collection fields:', collections)
-        c.update(blists, collections)
+        c.update(blists, collections, self.plugboard)
         c.write()
 
         USBMS.sync_booklists(self, booklists, end_session=end_session)
@@ -163,3 +166,9 @@ class PRS505(USBMS):
         c.write()
         debug_print('PRS505: finished rebuild_collections')
 
+    def use_plugboard_ext(self):
+        return 'device_db'
+
+    def set_plugboard(self, pb):
+        debug_print('PRS505: use plugboard', pb)
+        self.plugboard = pb

src/calibre/devices/prs505/sony_cache.py

@@ -325,12 +325,6 @@ class XMLCache(object):
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
-                title = record.get('title', None)
-                if title is not None and title != book.title:
-                    debug_print('Renaming title', book.title, 'to', title)
-                    book.title = title
-                    # Don't set the author, because the reader strips all but
-                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -350,7 +344,7 @@ class XMLCache(object):
     # }}}
 
     # Update XML from JSON {{{
-    def update(self, booklists, collections_attributes):
+    def update(self, booklists, collections_attributes, plugboard):
         debug_print('Starting update', collections_attributes)
         use_tz_var = False
         for i, booklist in booklists.items():
@@ -365,8 +359,13 @@ class XMLCache(object):
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
+                if plugboard is not None:
+                    newmi = book.deepcopy()
+                    newmi.template_to_attribute(book, plugboard)
+                else:
+                    newmi = book
                 (gtz_count, ltz_count, use_tz_var) = \
-                    self.update_text_record(record, book, path, i,
+                    self.update_text_record(record, newmi, path, i,
                                             gtz_count, ltz_count, use_tz_var)
                 # Ensure the collections in the XML database are recorded for
                 # this book

src/calibre/ebooks/metadata/__init__.py

@@ -44,7 +44,15 @@ def author_to_author_sort(author):
 def authors_to_sort_string(authors):
     return ' & '.join(map(author_to_author_sort, authors))
 
-_title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE)
+try:
+    _title_pat = re.compile(tweaks.get('title_sort_articles',
+                                       r'^(A|The|An)\s+'), re.IGNORECASE)
+except:
+    print 'Error in title sort pattern'
+    import traceback
+    traceback.print_exc()
+    _title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE)
+
 _ignore_starts = u'\'"'+u''.join(unichr(x) for x in range(0x2018, 0x201e)+[0x2032, 0x2033])
 
 def title_sort(title):

src/calibre/ebooks/metadata/book/base.py

@@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en'
 import copy, traceback
 
 from calibre import prints
+from calibre.constants import DEBUG
 from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
 from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
 from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
@@ -50,6 +51,8 @@ class SafeFormat(TemplateFormatter):
                 return ''
             return v
         except:
+            if DEBUG:
+                traceback.print_exc()
             return key
 
 composite_formatter = SafeFormat()
@@ -320,8 +323,8 @@ class Metadata(object):
                 else:
                     self.set(dest, val)
             except:
-                traceback.print_exc()
+                if DEBUG:
-                pass
+                    traceback.print_exc()
 
     # Old Metadata API {{{
     def print_all_attributes(self):

src/calibre/gui2/device.py

@@ -310,7 +310,13 @@ class DeviceManager(Thread): # {{{
         self.device.sync_booklists(booklists, end_session=False)
         return self.device.card_prefix(end_session=False), self.device.free_space()
 
-    def sync_booklists(self, done, booklists):
+    def sync_booklists(self, done, booklists, plugboards):
+        if hasattr(self.connected_device, 'use_plugboard_ext') and \
+                callable(self.connected_device.use_plugboard_ext):
+            ext = self.connected_device.use_plugboard_ext()
+            if ext is not None:
+                self.connected_device.set_plugboard(
+                                        self.find_plugboard(ext, plugboards))
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
@@ -319,28 +325,31 @@ class DeviceManager(Thread): # {{{
                                args=[booklist, on_card],
                         description=_('Send collections to device'))
 
+    def find_plugboard(self, ext, plugboards):
+        dev_name = self.connected_device.__class__.__name__
+        cpb = None
+        if ext in plugboards:
+            cpb = plugboards[ext]
+        elif plugboard_any_format_value in plugboards:
+            cpb = plugboards[plugboard_any_format_value]
+        if cpb is not None:
+            if dev_name in cpb:
+                cpb = cpb[dev_name]
+            elif plugboard_any_device_value in cpb:
+                cpb = cpb[plugboard_any_device_value]
+            else:
+                cpb = None
+        if DEBUG:
+            prints('Device using plugboard', ext, dev_name, cpb)
+        return cpb
+
     def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
         '''Upload books to device: '''
         if metadata and files and len(metadata) == len(files):
             for f, mi in zip(files, metadata):
                 if isinstance(f, unicode):
                     ext = f.rpartition('.')[-1].lower()
-                    dev_name = self.connected_device.__class__.__name__
+                    cpb = self.find_plugboard(ext, plugboards)
-                    cpb = None
-                    if ext in plugboards:
-                        cpb = plugboards[ext]
-                    elif plugboard_any_format_value in plugboards:
-                        cpb = plugboards[plugboard_any_format_value]
-                    if cpb is not None:
-                        if dev_name in cpb:
-                            cpb = cpb[dev_name]
-                        elif plugboard_any_device_value in cpb:
-                            cpb = cpb[plugboard_any_device_value]
-                        else:
-                            cpb = None
-
-                    if DEBUG:
-                        prints('Using plugboard', ext, dev_name, cpb)
                     if ext:
                         try:
                             if DEBUG:
@@ -1247,8 +1256,9 @@ class DeviceMixin(object): # {{{
         '''
         Upload metadata to device.
         '''
+        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
         self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
-                                           self.booklists())
+                                           self.booklists(), plugboards)
 
     def metadata_synced(self, job):
         '''
@@ -1502,8 +1512,10 @@ class DeviceMixin(object): # {{{
 
         if update_metadata:
             if self.device_manager.is_device_connected:
+                plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                 self.device_manager.sync_booklists(
-                                    Dispatcher(self.metadata_synced), booklists)
+                                    Dispatcher(self.metadata_synced), booklists,
+                                    plugboards)
         return update_metadata
     # }}}
 

src/calibre/gui2/library/delegates.py

@@ -153,7 +153,7 @@ class TextDelegate(QStyledItemDelegate): # {{{
             complete_items = [i[1] for i in self.auto_complete_function()]
             completer = QCompleter(complete_items, self)
             completer.setCaseSensitivity(Qt.CaseInsensitive)
-            completer.setCompletionMode(QCompleter.InlineCompletion)
+            completer.setCompletionMode(QCompleter.PopupCompletion)
             editor.setCompleter(completer)
         return editor
 #}}}

src/calibre/gui2/preferences/plugboard.py

@@ -39,6 +39,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         ConfigWidgetBase.initialize(self)
 
+        if self.gui.device_manager.connected_device is not None:
+            self.device_label.setText(_('Device currently connected: ') +
+                    self.gui.device_manager.connected_device.__class__.__name__)
+        else:
+            self.device_label.setText(_('Device currently connected: None'))
+
         self.devices = ['']
         for device in device_plugins():
             n = device.__class__.__name__
@@ -54,6 +60,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         for w in metadata_writers():
             for f in w.file_types:
                 self.formats.append(f)
+        self.formats.append('device_db')
         self.formats.sort()
         self.formats.insert(1, plugboard_any_format_value)
         self.new_format.addItems(self.formats)

src/calibre/gui2/preferences/plugboard.ui

@@ -40,7 +40,18 @@ One possible use for a plugboard is to alter the title to contain series informa
      </property>
     </widget>
    </item>
-   <item row="2" column="0">
+   <item row="2" column="0" colspan="2">
+    <widget class="QLabel" name="device_label">
+    </widget>
+   </item>
+   <item row="3" column="0" colspan="2">
+    <widget class="Line" name="line">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0">
     <layout class="QGridLayout" name="gridLayout_2">
      <item row="0" column="1">
       <widget class="QLabel" name="label_6">
@@ -123,7 +134,7 @@ One possible use for a plugboard is to alter the title to contain series informa
      </item>
     </layout>
    </item>
-   <item row="2" column="1">
+   <item row="4" column="1">
     <layout class="QGridLayout" name="fields_layout">
      <item row="0" column="0">
       <widget class="QLabel" name="label_2">

src/calibre/library/database2.py

@@ -1593,7 +1593,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.commit()
 
     def delete_publisher_using_id(self, old_id):
-        self.dirty_books_referencing('publisher', id, commit=False)
+        self.dirty_books_referencing('publisher', old_id, commit=False)
         self.conn.execute('''DELETE FROM books_publishers_link
                              WHERE publisher=?''', (old_id,))
         self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))

src/calibre/library/save_to_disk.py

@@ -8,6 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import os, traceback, cStringIO, re
 
+from calibre import prints
+from calibre.constants import DEBUG
 from calibre.utils.config import Config, StringConfig, tweaks
 from calibre.utils.formatter import TemplateFormatter
 from calibre.utils.filenames import shorten_components_to, supports_long_names, \
@@ -118,8 +120,8 @@ class SafeFormat(TemplateFormatter):
             try:
                 b = self.book.get_user_metadata(key, False)
             except:
-                prints('save_to_disk get value exception')
+                if DEBUG:
-                traceback.print_exc()
+                    traceback.print_exc()
                 b = None
 
             if b is not None and b['datatype'] == 'composite':
@@ -129,13 +131,13 @@ class SafeFormat(TemplateFormatter):
                 self.composite_values[key] = \
                     self.vformat(b['display']['composite_template'], [], kwargs)
                 return self.composite_values[key]
-            if kwargs[key]:
+            if key in kwargs:
-                return self.sanitize(kwargs[key])
+                return kwargs[key].replace('/', '_').replace('\\', '_')
             return ''
         except:
-            print('save_to_disk general exception')
+            if DEBUG:
-            traceback.print_exc()
+                traceback.print_exc()
-            return ''
+            return key
 
 safe_formatter = SafeFormat()
 
@@ -182,8 +184,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
             elif custom_metadata[key]['datatype'] == 'bool':
                 format_args[key] = _('yes') if format_args[key] else _('no')
 
-    components = safe_formatter.safe_format(template, format_args, '', mi,
+    components = safe_formatter.safe_format(template, format_args,
-                                            sanitize=sanitize_func)
+                                            'G_C-EXCEPTION!', mi)
     components = [x.strip() for x in components.split('/') if x.strip()]
     components = [sanitize_func(x) for x in components if x]
     if not components:
@@ -267,7 +269,8 @@ def save_book_to_disk(id, db, root, opts, length):
                 cpb = cpb[dev_name]
             else:
                 cpb = None
-        #prints('Using plugboard:', fmt, cpb)
+        if DEBUG:
+            prints('Save-to-disk using plugboard:', fmt, cpb)
         data = db.format(id, fmt, index_is_id=True)
         if data is None:
             continue
@@ -285,7 +288,8 @@ def save_book_to_disk(id, db, root, opts, length):
                     newmi = mi
                 set_metadata(stream, newmi, fmt)
             except:
-                traceback.print_exc()
+                if DEBUG:
+                    traceback.print_exc()
             stream.seek(0)
             data = stream.read()
         fmt_path = base_path+'.'+str(fmt)

src/calibre/manual/template_lang.rst

@@ -165,7 +165,9 @@ For tags, the result cut apart whereever |app| finds a comma. For example, if th
 
 The same thing happens for authors, but using a different character for the cut, a `&` (ampersand) instead of a comma. For example, if the template produces the value ``Blogs, Joe&Posts, Susan``, then the book will end up with two authors, ``Blogs, Joe`` and ``Posts, Susan``. If the template produces the value ``Blogs, Joe;Posts, Susan``, then the book will have one author with a rather strange name.
 
-Plugboards affect only the metadata written into the book. They do not affect calibre's metadata or the metadata used in ``save to disk`` and ``send to device`` templates. Plugboards also do not affect what is written into a Sony's database, so cannot be used for altering the metadata shown on a Sony's menu.
+Plugboards affect the metadata written into the book when it is saved to disk or written to the device. Plugboards do not affect the metadata used by ``save to disk`` and ``send to device`` to create the file names. Instead, file names are constructed using the templates entered on the appropriate preferences window. 
+
+Plugboards also do not affect the metadata that is written into a Sony's database, because this metadata does not come from the book. As such, you cannot use plugboards for altering the metadata shown on a Sony's menu.
 
 Helpful Tips
 ------------

src/calibre/utils/formatter.py

@@ -4,7 +4,9 @@ Created on 23 Sep 2010
 @author: charles
 '''
 
-import re, string
+import re, string, traceback
+
+from calibre.constants import DEBUG
 
 class TemplateFormatter(string.Formatter):
     '''
@@ -19,7 +21,6 @@ class TemplateFormatter(string.Formatter):
         string.Formatter.__init__(self)
         self.book = None
         self.kwargs = None
-        self.sanitize = None
 
     def _lookup(self, val, field_if_set, field_not_set):
         if val:
@@ -99,8 +100,8 @@ class TemplateFormatter(string.Formatter):
                 return fmt, '', ''
             return matches.groups()
         except:
-            import traceback
+            if DEBUG:
-            traceback.print_exc()
+                traceback.print_exc()
             return fmt, '', ''
 
     def format_field(self, val, fmt):
@@ -139,14 +140,15 @@ class TemplateFormatter(string.Formatter):
         ans = string.Formatter.vformat(self, fmt, args, kwargs)
         return self.compress_spaces.sub(' ', ans).strip()
 
-    def safe_format(self, fmt, kwargs, error_value, book, sanitize=None):
+    def safe_format(self, fmt, kwargs, error_value, book):
         self.kwargs = kwargs
         self.book = book
-        self.sanitize = sanitize
         self.composite_values = {}
         try:
             ans = self.vformat(fmt, [], kwargs).strip()
         except:
+            if DEBUG:
+                traceback.print_exc()
             ans = error_value
         return ans