From 8324d42162a2a8e56c430bc28987088a22d3074e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 08:45:24 +0100 Subject: [PATCH 1/8] Slight changes to the templates FAQ to (I hope) reduce confusion --- src/calibre/manual/template_lang.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index bfffe1140a..e2ab4a0072 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -165,7 +165,9 @@ For tags, the result cut apart whereever |app| finds a comma. For example, if th The same thing happens for authors, but using a different character for the cut, a `&` (ampersand) instead of a comma. For example, if the template produces the value ``Blogs, Joe&Posts, Susan``, then the book will end up with two authors, ``Blogs, Joe`` and ``Posts, Susan``. If the template produces the value ``Blogs, Joe;Posts, Susan``, then the book will have one author with a rather strange name. -Plugboards affect only the metadata written into the book. They do not affect calibre's metadata or the metadata used in ``save to disk`` and ``send to device`` templates. Plugboards also do not affect what is written into a Sony's database, so cannot be used for altering the metadata shown on a Sony's menu. +Plugboards affect the metadata written into the book when it is saved to disk or written to the device. Plugboards do not affect the metadata used by ``save to disk`` and ``send to device`` to create the file names. Instead, file names are constructed using the templates entered on the appropriate preferences window. + +Plugboards also do not affect the metadata that is written into a Sony's database, because this metadata does not come from the book. As such, you cannot use plugboards for altering the metadata shown on a Sony's menu. Helpful Tips ------------ From 50aae924fc3c9e060ccdc6ada455bd5366225ceb Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 09:21:08 +0100 Subject: [PATCH 2/8] Make template processing more robust (I hope) --- src/calibre/ebooks/metadata/book/base.py | 7 +++++-- src/calibre/gui2/device.py | 2 +- src/calibre/library/save_to_disk.py | 26 ++++++++++++++---------- src/calibre/utils/formatter.py | 14 +++++++------ 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 17611875f8..b8288e210c 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en' import copy, traceback from calibre import prints +from calibre.constants import DEBUG from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS @@ -50,6 +51,8 @@ class SafeFormat(TemplateFormatter): return '' return v except: + if DEBUG: + traceback.print_exc() return key composite_formatter = SafeFormat() @@ -320,8 +323,8 @@ class Metadata(object): else: self.set(dest, val) except: - traceback.print_exc() - pass + if DEBUG: + traceback.print_exc() # Old Metadata API {{{ def print_all_attributes(self): diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 254c62e48c..d81fad3da9 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -340,7 +340,7 @@ class DeviceManager(Thread): # {{{ cpb = None if DEBUG: - prints('Using plugboard', ext, dev_name, cpb) + prints('Device using plugboard', ext, dev_name, cpb) if ext: try: if DEBUG: diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 6c86db3420..f3c384cb15 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -8,6 +8,8 @@ __docformat__ = 'restructuredtext en' import os, traceback, cStringIO, re +from calibre import prints +from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ @@ -118,8 +120,8 @@ class SafeFormat(TemplateFormatter): try: b = self.book.get_user_metadata(key, False) except: - print 'save_to_disk get value exception' - traceback.print_exc() + if DEBUG: + traceback.print_exc() b = None if b is not None and b['datatype'] == 'composite': @@ -129,13 +131,13 @@ class SafeFormat(TemplateFormatter): self.composite_values[key] = \ self.vformat(b['display']['composite_template'], [], kwargs) return self.composite_values[key] - if kwargs[key]: - return self.sanitize(kwargs[key]) + if key in kwargs: + return kwargs[key].replace('/', '_').replace('\\', '_') return '' except: - print 'save_to_disk general exception' - traceback.print_exc() - return '' + if DEBUG: + traceback.print_exc() + return key safe_formatter = SafeFormat() @@ -182,8 +184,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, elif custom_metadata[key]['datatype'] == 'bool': format_args[key] = _('yes') if format_args[key] else _('no') - components = safe_formatter.safe_format(template, format_args, '', mi, - sanitize=sanitize_func) + components = safe_formatter.safe_format(template, format_args, + 'G_C-EXCEPTION!', mi) components = [x.strip() for x in components.split('/') if x.strip()] components = [sanitize_func(x) for x in components if x] if not components: @@ -267,7 +269,8 @@ def save_book_to_disk(id, db, root, opts, length): cpb = cpb[dev_name] else: cpb = None - #prints('Using plugboard:', fmt, cpb) + if DEBUG: + prints('Save-to-disk using plugboard:', fmt, cpb) data = db.format(id, fmt, index_is_id=True) if data is None: continue @@ -285,7 +288,8 @@ def save_book_to_disk(id, db, root, opts, length): newmi = mi set_metadata(stream, newmi, fmt) except: - traceback.print_exc() + if DEBUG: + traceback.print_exc() stream.seek(0) data = stream.read() fmt_path = base_path+'.'+str(fmt) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index cba21705a9..fdfd7b77c3 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -4,7 +4,9 @@ Created on 23 Sep 2010 @author: charles ''' -import re, string +import re, string, traceback + +from calibre.constants import DEBUG class TemplateFormatter(string.Formatter): ''' @@ -19,7 +21,6 @@ class TemplateFormatter(string.Formatter): string.Formatter.__init__(self) self.book = None self.kwargs = None - self.sanitize = None def _lookup(self, val, field_if_set, field_not_set): if val: @@ -99,8 +100,8 @@ class TemplateFormatter(string.Formatter): return fmt, '', '' return matches.groups() except: - import traceback - traceback.print_exc() + if DEBUG: + traceback.print_exc() return fmt, '', '' def format_field(self, val, fmt): @@ -139,14 +140,15 @@ class TemplateFormatter(string.Formatter): ans = string.Formatter.vformat(self, fmt, args, kwargs) return self.compress_spaces.sub(' ', ans).strip() - def safe_format(self, fmt, kwargs, error_value, book, sanitize=None): + def safe_format(self, fmt, kwargs, error_value, book): self.kwargs = kwargs self.book = book - self.sanitize = sanitize self.composite_values = {} try: ans = self.vformat(fmt, [], kwargs).strip() except: + if DEBUG: + traceback.print_exc() ans = error_value return ans From ce8edcb045ed3c59c58e2d03b1ea95aee9e76518 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 10:15:38 +0100 Subject: [PATCH 3/8] Add the current device classname to the plugboard UI. --- src/calibre/gui2/preferences/plugboard.py | 6 ++++++ src/calibre/gui2/preferences/plugboard.ui | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 59ef4cb246..5bc05a5874 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -39,6 +39,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ConfigWidgetBase.initialize(self) + if self.gui.device_manager.connected_device is not None: + self.device_label.setText(_('Device currently connected: ') + + self.gui.device_manager.connected_device.__class__.__name__) + else: + self.device_label.setText(_('Device currently connected: None')) + self.devices = [''] for device in device_plugins(): n = device.__class__.__name__ diff --git a/src/calibre/gui2/preferences/plugboard.ui b/src/calibre/gui2/preferences/plugboard.ui index b73c396481..8249584678 100644 --- a/src/calibre/gui2/preferences/plugboard.ui +++ b/src/calibre/gui2/preferences/plugboard.ui @@ -40,7 +40,18 @@ One possible use for a plugboard is to alter the title to contain series informa - + + + + + + + + Qt::Horizontal + + + + @@ -123,7 +134,7 @@ One possible use for a plugboard is to alter the title to contain series informa - + From bf58ecaa46c922cd961c81ac89c08e39678a8148 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 10:27:34 +0100 Subject: [PATCH 4/8] Fix #7046 - exception when deleting a publisher --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index e679780b46..529fd3fdc4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1593,7 +1593,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() def delete_publisher_using_id(self, old_id): - self.dirty_books_referencing('publisher', id, commit=False) + self.dirty_books_referencing('publisher', old_id, commit=False) self.conn.execute('''DELETE FROM books_publishers_link WHERE publisher=?''', (old_id,)) self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,)) From ac38d10b4b20905e418da09becb0a959e08a4964 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 10:46:35 +0100 Subject: [PATCH 5/8] Fix #7037 - on library view, publisher field overwritten by last publisher if d-clicked and then not changed. --- src/calibre/gui2/library/delegates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index 1156d1ace9..f9ba612bc9 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -153,7 +153,7 @@ class TextDelegate(QStyledItemDelegate): # {{{ complete_items = [i[1] for i in self.auto_complete_function()] completer = QCompleter(complete_items, self) completer.setCaseSensitivity(Qt.CaseInsensitive) - completer.setCompletionMode(QCompleter.InlineCompletion) + completer.setCompletionMode(QCompleter.PopupCompletion) editor.setCompleter(completer) return editor #}}} From 60083e325c59a134586722d57ee515ae0b49d576 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 12:57:48 +0100 Subject: [PATCH 6/8] Secret feature: sony driver can use plugboard when writing metadata to its DB. --- resources/default_tweaks.py | 10 +++++ src/calibre/devices/prs505/driver.py | 13 +++++- src/calibre/devices/prs505/sony_cache.py | 15 ++++--- src/calibre/gui2/device.py | 50 +++++++++++++++--------- 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 095eba0c3d..3b60e3410e 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -171,3 +171,13 @@ content_server_wont_display = [''] # level sorts, and if you are seeing a slowdown, reduce the value of this tweak. maximum_resort_levels = 5 + +# Tell the Sony driver to apply the plugboard specified by the given format +# before writing to its database. This can be used to change the title, etc, +# in the DB, and therefore what the Sony displays. +# Example: +# sony_db_use_plugboard_format='epub' +# Apply the epub plugboard before writing to the Sony DB. +# Default: '' +sony_db_use_plugboard_format='' + diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 7952660c21..e69851253a 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -14,6 +14,7 @@ from calibre.devices.prs505 import CACHE_XML from calibre.devices.prs505.sony_cache import XMLCache from calibre import __appname__ from calibre.devices.usbms.books import CollectionsBookList +from calibre.utils.config import tweaks class PRS505(USBMS): @@ -63,6 +64,8 @@ class PRS505(USBMS): 'series, tags, authors' EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags']) + plugboard = None + def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id @@ -150,7 +153,7 @@ class PRS505(USBMS): else: collections = [] debug_print('PRS505: collection fields:', collections) - c.update(blists, collections) + c.update(blists, collections, self.plugboard) c.write() USBMS.sync_booklists(self, booklists, end_session=end_session) @@ -163,3 +166,11 @@ class PRS505(USBMS): c.write() debug_print('PRS505: finished rebuild_collections') + def use_plugboard_ext(self): + ext = tweaks.get('sony_db_use_plugboard_format', None) + return ext + + def set_plugboard(self, pb): + if tweaks.get('sony_db_use_plugboard_format', None): + debug_print('PRS505: use plugboard', pb) + self.plugboard = pb \ No newline at end of file diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index 879f86d66a..5247e051f1 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -325,12 +325,6 @@ class XMLCache(object): for book in bl: record = lpath_map.get(book.lpath, None) if record is not None: - title = record.get('title', None) - if title is not None and title != book.title: - debug_print('Renaming title', book.title, 'to', title) - book.title = title - # Don't set the author, because the reader strips all but - # the first author. for thumbnail in record.xpath( 'descendant::*[local-name()="thumbnail"]'): for img in thumbnail.xpath( @@ -350,7 +344,7 @@ class XMLCache(object): # }}} # Update XML from JSON {{{ - def update(self, booklists, collections_attributes): + def update(self, booklists, collections_attributes, plugboard): debug_print('Starting update', collections_attributes) use_tz_var = False for i, booklist in booklists.items(): @@ -365,8 +359,13 @@ class XMLCache(object): record = lpath_map.get(book.lpath, None) if record is None: record = self.create_text_record(root, i, book.lpath) + if plugboard is not None: + newmi = book.deepcopy() + newmi.template_to_attribute(book, plugboard) + else: + newmi = book (gtz_count, ltz_count, use_tz_var) = \ - self.update_text_record(record, book, path, i, + self.update_text_record(record, newmi, path, i, gtz_count, ltz_count, use_tz_var) # Ensure the collections in the XML database are recorded for # this book diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index c8a2325a0a..d06b77b4e2 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -310,7 +310,13 @@ class DeviceManager(Thread): # {{{ self.device.sync_booklists(booklists, end_session=False) return self.device.card_prefix(end_session=False), self.device.free_space() - def sync_booklists(self, done, booklists): + def sync_booklists(self, done, booklists, plugboards): + if hasattr(self.connected_device, 'use_plugboard_ext') and \ + callable(self.connected_device.use_plugboard_ext): + ext = self.connected_device.use_plugboard_ext() + if ext is not None: + self.connected_device.set_plugboard( + self.find_plugboard(ext, plugboards)) return self.create_job(self._sync_booklists, done, args=[booklists], description=_('Send metadata to device')) @@ -319,28 +325,31 @@ class DeviceManager(Thread): # {{{ args=[booklist, on_card], description=_('Send collections to device')) + def find_plugboard(self, ext, plugboards): + dev_name = self.connected_device.__class__.__name__ + cpb = None + if ext in plugboards: + cpb = plugboards[ext] + elif plugboard_any_format_value in plugboards: + cpb = plugboards[plugboard_any_format_value] + if cpb is not None: + if dev_name in cpb: + cpb = cpb[dev_name] + elif plugboard_any_device_value in cpb: + cpb = cpb[plugboard_any_device_value] + else: + cpb = None + if DEBUG: + prints('Device using plugboard', ext, dev_name, cpb) + return cpb + def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None): '''Upload books to device: ''' if metadata and files and len(metadata) == len(files): for f, mi in zip(files, metadata): if isinstance(f, unicode): ext = f.rpartition('.')[-1].lower() - dev_name = self.connected_device.__class__.__name__ - cpb = None - if ext in plugboards: - cpb = plugboards[ext] - elif plugboard_any_format_value in plugboards: - cpb = plugboards[plugboard_any_format_value] - if cpb is not None: - if dev_name in cpb: - cpb = cpb[dev_name] - elif plugboard_any_device_value in cpb: - cpb = cpb[plugboard_any_device_value] - else: - cpb = None - - if DEBUG: - prints('Device using plugboard', ext, dev_name, cpb) + cpb = self.find_plugboard(ext, plugboards) if ext: try: if DEBUG: @@ -1247,8 +1256,9 @@ class DeviceMixin(object): # {{{ ''' Upload metadata to device. ''' + plugboards = self.library_view.model().db.prefs.get('plugboards', {}) self.device_manager.sync_booklists(Dispatcher(self.metadata_synced), - self.booklists()) + self.booklists(), plugboards) def metadata_synced(self, job): ''' @@ -1502,8 +1512,10 @@ class DeviceMixin(object): # {{{ if update_metadata: if self.device_manager.is_device_connected: + plugboards = self.library_view.model().db.prefs.get('plugboards', {}) self.device_manager.sync_booklists( - Dispatcher(self.metadata_synced), booklists) + Dispatcher(self.metadata_synced), booklists, + plugboards) return update_metadata # }}} From 6f45344847a544542c54e8ddcf8fb03dd567af6c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 14:54:06 +0100 Subject: [PATCH 7/8] Add tweak to change articles in title sort pattern --- resources/default_tweaks.py | 9 +++++++++ src/calibre/ebooks/metadata/__init__.py | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 3b60e3410e..17ed0eeea7 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -83,6 +83,15 @@ title_series_sorting = 'library_order' # strictly_alphabetic, it would remain "The Client". save_template_title_series_sorting = 'library_order' +# Set the list of words that are to be considered 'articles' when computing the +# title sort strings. The list is a regular expression, with the articles +# separated by 'or' bars. Comparisons are case insensitive, and that cannot be +# changed. Changes to this tweak won't have an effect until the book is modified +# in some way. If you enter an invalid pattern, it is silently ignored. +# Default: '^(A|The|An)\s+' +title_sort_articles='^(A|The|An)\s+' + + # Specify a folder that calibre should connect to at startup using # connect_to_folder. This must be a full path to the folder. If the folder does # not exist when calibre starts, it is ignored. If there are '\' characters in diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 429ba06c6e..01e5190640 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -44,7 +44,15 @@ def author_to_author_sort(author): def authors_to_sort_string(authors): return ' & '.join(map(author_to_author_sort, authors)) -_title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE) +try: + _title_pat = re.compile(tweaks.get('title_sort_articles', + r'^(A|The|An)\s+'), re.IGNORECASE) +except: + print 'Error in title sort pattern' + import traceback + traceback.print_exc() + _title_pat = re.compile('^(A|The|An)\s+', re.IGNORECASE) + _ignore_starts = u'\'"'+u''.join(unichr(x) for x in range(0x2018, 0x201e)+[0x2032, 0x2033]) def title_sort(title): From 0d163a2a7d090a2b252a8630fdc094ec81d05901 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 3 Oct 2010 15:29:38 +0100 Subject: [PATCH 8/8] Use 'device_db' for device database updates. Change the sony driver as appropriate. Remove the tweak. --- resources/default_tweaks.py | 10 ---------- src/calibre/devices/prs505/driver.py | 8 +++----- src/calibre/gui2/preferences/plugboard.py | 1 + 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 17ed0eeea7..98cdccf164 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -180,13 +180,3 @@ content_server_wont_display = [''] # level sorts, and if you are seeing a slowdown, reduce the value of this tweak. maximum_resort_levels = 5 - -# Tell the Sony driver to apply the plugboard specified by the given format -# before writing to its database. This can be used to change the title, etc, -# in the DB, and therefore what the Sony displays. -# Example: -# sony_db_use_plugboard_format='epub' -# Apply the epub plugboard before writing to the Sony DB. -# Default: '' -sony_db_use_plugboard_format='' - diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index e69851253a..ca8c09245c 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -167,10 +167,8 @@ class PRS505(USBMS): debug_print('PRS505: finished rebuild_collections') def use_plugboard_ext(self): - ext = tweaks.get('sony_db_use_plugboard_format', None) - return ext + return 'device_db' def set_plugboard(self, pb): - if tweaks.get('sony_db_use_plugboard_format', None): - debug_print('PRS505: use plugboard', pb) - self.plugboard = pb \ No newline at end of file + debug_print('PRS505: use plugboard', pb) + self.plugboard = pb \ No newline at end of file diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 5bc05a5874..4781921073 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -60,6 +60,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): for w in metadata_writers(): for f in w.file_types: self.formats.append(f) + self.formats.append('device_db') self.formats.sort() self.formats.insert(1, plugboard_any_format_value) self.new_format.addItems(self.formats)