From 1a782eb0ffa9ca7bfc063902b35b06f6e8399271 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 23 Sep 2010 20:36:52 +0100 Subject: [PATCH 01/13] - Some cleanups on templates. - Make save_to_disk templates sanitize all fields except composites --- src/calibre/ebooks/metadata/book/base.py | 9 ++- src/calibre/library/save_to_disk.py | 12 ++-- src/calibre/utils/formatter.py | 72 ++++++++++++------------ 3 files changed, 51 insertions(+), 42 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 9e1085df25..929dc01aec 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -34,9 +34,10 @@ NULL_VALUES = { field_metadata = FieldMetadata() class SafeFormat(TemplateFormatter): - def get_value(self, key, args, mi): + + def get_value(self, key, args, kwargs): try: - ign, v = mi.format_field(key.lower(), series_with_index=False) + ign, v = self.book.format_field(key.lower(), series_with_index=False) if v is None: return '' if v == '': @@ -100,7 +101,9 @@ class Metadata(object): cf['#value#'] = 'RECURSIVE_COMPOSITE FIELD ' + field cf['#value#'] = composite_formatter.safe_format( cf['display']['composite_template'], - self, _('TEMPLATE ERROR')).strip() + self, + _('TEMPLATE ERROR'), + self).strip() return d['#value#'] raise AttributeError( diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 11922b7154..2d0a3d1277 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -108,8 +108,12 @@ class SafeFormat(TemplateFormatter): ''' def get_value(self, key, args, kwargs): try: - if kwargs[key.lower()]: - return kwargs[key.lower()] + b = self.book.get_user_metadata(key, False) + key = key.lower() + if b is not None and b['datatype'] == 'composite': + return self.vformat(b['display']['composite_template'], [], kwargs) + if kwargs[key]: + return self.sanitize(kwargs[key.lower()]) return '' except: return '' @@ -159,9 +163,9 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, elif custom_metadata[key]['datatype'] == 'bool': format_args[key] = _('yes') if format_args[key] else _('no') - components = safe_formatter.safe_format(template, format_args, '') + components = safe_formatter.safe_format(template, format_args, '', mi, + sanitize=sanitize_func) components = [x.strip() for x in components.split('/') if x.strip()] - components = [sanitize_func(x) for x in components if x] if not components: components = [str(id)] components = [x.encode(filesystem_encoding, 'replace') if isinstance(x, diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index f9ef4e0846..95870d9c61 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -6,48 +6,48 @@ Created on 23 Sep 2010 import re, string -def _lookup(val, mi, field_if_set, field_not_set): - if hasattr(mi, 'format_field'): - if val: - return mi.format_field(field_if_set.strip())[1] - else: - return mi.format_field(field_not_set.strip())[1] - else: - if val: - return mi.get(field_if_set.strip(), '') - else: - return mi.get(field_not_set.strip(), '') - -def _ifempty(val, mi, value_if_empty): - if val: - return val - else: - return value_if_empty - -def _shorten(val, mi, leading, center_string, trailing): - l = int(leading) - t = int(trailing) - if len(val) > l + len(center_string) + t: - return val[0:l] + center_string + val[-t:] - else: - return val - class TemplateFormatter(string.Formatter): ''' Provides a format function that substitutes '' for any missing value ''' + def __init__(self): + string.Formatter.__init__(self) + self.book = None + self.kwargs = None + self.sanitize = None + + def _lookup(self, val, field_if_set, field_not_set): + if val: + return self.vformat('{'+field_if_set.strip()+'}', [], self.kwargs) + else: + return self.vformat('{'+field_not_set.strip()+'}', [], self.kwargs) + + def _ifempty(self, val, value_if_empty): + if val: + return val + else: + return value_if_empty + + def _shorten(self, val, leading, center_string, trailing): + l = int(leading) + t = int(trailing) + if len(val) > l + len(center_string) + t: + return val[0:l] + center_string + val[-t:] + else: + return val + functions = { - 'uppercase' : (0, lambda x: x.upper()), - 'lowercase' : (0, lambda x: x.lower()), - 'titlecase' : (0, lambda x: x.title()), - 'capitalize' : (0, lambda x: x.capitalize()), + 'uppercase' : (0, lambda s,x: x.upper()), + 'lowercase' : (0, lambda s,x: x.lower()), + 'titlecase' : (0, lambda s,x: x.title()), + 'capitalize' : (0, lambda s,x: x.capitalize()), 'ifempty' : (1, _ifempty), 'lookup' : (2, _lookup), 'shorten' : (3, _shorten), } - def get_value(self, key, args, mi): + def get_value(self, key, args): raise Exception('get_value must be implemented in the subclass') format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') @@ -75,9 +75,9 @@ class TemplateFormatter(string.Formatter): (func[0] > 0 and func[0] != len(args)): raise Exception ('Incorrect number of arguments for function '+ fmt[0:p]) if func[0] == 0: - val = func[1](val, self.mi) + val = func[1](self, val) else: - val = func[1](val, self.mi, *args) + val = func[1](self, val, *args) else: val = string.Formatter.format_field(self, val, fmt) if not val: @@ -87,11 +87,13 @@ class TemplateFormatter(string.Formatter): compress_spaces = re.compile(r'\s+') def vformat(self, fmt, args, kwargs): - self.mi = kwargs ans = string.Formatter.vformat(self, fmt, args, kwargs) return self.compress_spaces.sub(' ', ans).strip() - def safe_format(self, fmt, kwargs, error_value): + def safe_format(self, fmt, kwargs, error_value, book, sanitize=None): + self.kwargs = kwargs + self.book = book + self.sanitize = sanitize try: ans = self.vformat(fmt, [], kwargs).strip() except: From 992e5c3c087c1e28cb1e5f1aa61ead0e4556de18 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 08:21:10 +0100 Subject: [PATCH 02/13] Repair damage during conflict resolution --- src/calibre/utils/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index f1c2a2cb4d..a98f0e7f45 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -50,7 +50,7 @@ class TemplateFormatter(string.Formatter): format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') compress_spaces = re.compile(r'\s+') - def get_value(self, key, args): + def get_value(self, key, args, kwargs): raise Exception('get_value must be implemented in the subclass') From 12768864a59be5ddf477c490072fd682b1f5942a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 08:54:06 +0100 Subject: [PATCH 03/13] 1) fix exception in set_metadata related to composite custom columns 2) make ondevice work with add_books_from_device --- src/calibre/gui2/actions/add.py | 2 +- src/calibre/gui2/device.py | 16 +++++++++------- src/calibre/gui2/library/models.py | 3 +++ src/calibre/library/custom_columns.py | 2 ++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index aa20b8bc16..e0a7b5647e 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -232,7 +232,7 @@ class AddAction(InterfaceAction): # metadata for this book to the device. This sets the uuid to the # correct value. Note that set_books_in_library might sync_booklists self.gui.set_books_in_library(booklists=[model.db], reset=True) - model.reset() + self.gui.refresh_ondevice() def add_books_from_device(self, view): rows = view.selectionModel().selectedRows() diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a7e55c4619..58c5e5d9ad 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -721,14 +721,16 @@ class DeviceMixin(object): # {{{ self.device_manager.device.__class__.get_gui_name()+\ _(' detected.'), 3000) self.device_connected = device_kind - self.refresh_ondevice_info (device_connected = True, reset_only = True) + self.library_view.set_device_connected(self.device_connected) + self.refresh_ondevice (reset_only = True) else: self.device_connected = None self.status_bar.device_disconnected() if self.current_view() != self.library_view: self.book_details.reset_info() self.location_manager.update_devices() - self.refresh_ondevice_info(device_connected=False) + self.library_view.set_device_connected(self.device_connected) + self.refresh_ondevice() def info_read(self, job): ''' @@ -760,9 +762,9 @@ class DeviceMixin(object): # {{{ self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA) self.sync_news() self.sync_catalogs() - self.refresh_ondevice_info(device_connected = True) + self.refresh_ondevice() - def refresh_ondevice_info(self, device_connected, reset_only = False): + def refresh_ondevice(self, reset_only = False): ''' Force the library view to refresh, taking into consideration new device books information @@ -770,7 +772,7 @@ class DeviceMixin(object): # {{{ self.book_on_device(None, reset=True) if reset_only: return - self.library_view.set_device_connected(device_connected) + self.library_view.model().refresh_ondevice() # }}} @@ -803,7 +805,7 @@ class DeviceMixin(object): # {{{ self.book_on_device(None, reset=True) # We need to reset the ondevice flags in the library. Use a big hammer, # so we don't need to worry about whether some succeeded or not. - self.refresh_ondevice_info(device_connected=True, reset_only=False) + self.refresh_ondevice(reset_only=False) def dispatch_sync_event(self, dest, delete, specific): rows = self.library_view.selectionModel().selectedRows() @@ -1300,7 +1302,7 @@ class DeviceMixin(object): # {{{ if not self.set_books_in_library(self.booklists(), reset=True): self.upload_booklists() self.book_on_device(None, reset=True) - self.refresh_ondevice_info(device_connected = True) + self.refresh_ondevice() view = self.card_a_view if on_card == 'carda' else \ self.card_b_view if on_card == 'cardb' else self.memory_view diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 9d9de358c8..640a588d29 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -120,6 +120,9 @@ class BooksModel(QAbstractTableModel): # {{{ def set_device_connected(self, is_connected): self.device_connected = is_connected + self.refresh_ondevice() + + def refresh_ondevice(self): self.db.refresh_ondevice() self.refresh() # does a resort() self.research() diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 2d8634659b..97c8565177 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -427,6 +427,8 @@ class CustomColumns(object): data = self.custom_column_label_map[label] if num is not None: data = self.custom_column_num_map[num] + if data['datatype'] == 'composite': + return None if not data['editable']: raise ValueError('Column %r is not editable'%data['label']) table, lt = self.custom_table_names(data['num']) From 97e2c838d0e6e4920ce4e1f4d71688979f57b68d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 10:50:50 +0100 Subject: [PATCH 04/13] 1) Fix of json codec. 2) make dump_metadata set get_cover=False --- src/calibre/ebooks/metadata/book/json_codec.py | 3 ++- src/calibre/library/database2.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index 2550089473..c02d4e953d 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -75,7 +75,8 @@ class JsonCodec(object): self.field_metadata = FieldMetadata() def encode_to_file(self, file, booklist): - json.dump(self.encode_booklist_metadata(booklist), file, indent=2, encoding='utf-8') + file.write(json.dumps(self.encode_booklist_metadata(booklist), + indent=2, encoding='utf-8')) def encode_booklist_metadata(self, booklist): result = [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 6a0d442927..773a4bdc9f 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -561,7 +561,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for book_id in book_ids: if not self.data.has_id(book_id): continue - mi = self.get_metadata(book_id, index_is_id=True, get_cover=True) + mi = self.get_metadata(book_id, index_is_id=True, get_cover=False) # Always set cover to cover.jpg. Even if cover doesn't exist, # no harm done. This way no need to call dirtied when # cover is set/removed From 8b9b64a8e6bdab03f62c98a4f2c35ec73957cca7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 11:34:52 +0100 Subject: [PATCH 05/13] 1) add two tweaks controlling what custom fields the content server displays 2) add & cleanup some field_metadata methods --- resources/default_tweaks.py | 18 ++++++++++++++++++ src/calibre/gui2/library/models.py | 2 +- src/calibre/library/database2.py | 6 ++++++ src/calibre/library/field_metadata.py | 2 +- src/calibre/library/server/__init__.py | 12 +++++++++++- src/calibre/library/server/mobile.py | 3 ++- src/calibre/library/server/opds.py | 3 ++- src/calibre/library/server/xml.py | 3 ++- 8 files changed, 43 insertions(+), 6 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 04b861605e..095eba0c3d 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -145,6 +145,24 @@ add_new_book_tags_when_importing_books = False # Set the maximum number of tags to show per book in the content server max_content_server_tags_shown=5 +# Set custom metadata fields that the content server will or will not display. +# content_server_will_display is a list of custom fields to be displayed. +# content_server_wont_display is a list of custom fields not to be displayed. +# wont_display has priority over will_display. +# The special value '*' means all custom fields. +# Defaults: +# content_server_will_display = ['*'] +# content_server_wont_display = [''] +# Examples: +# To display only the custom fields #mytags and #genre: +# content_server_will_display = ['#mytags', '#genre'] +# content_server_wont_display = [''] +# To display all fields except #mycomments: +# content_server_will_display = ['*'] +# content_server_wont_display['#mycomments'] +content_server_will_display = ['*'] +content_server_wont_display = [''] + # Set the maximum number of sort 'levels' that calibre will use to resort the # library after certain operations such as searches or device insertion. Each diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 640a588d29..af1b42bf33 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -132,7 +132,7 @@ class BooksModel(QAbstractTableModel): # {{{ def set_database(self, db): self.db = db - self.custom_columns = self.db.field_metadata.get_custom_field_metadata() + self.custom_columns = self.db.field_metadata.custom_field_metadata() self.column_map = list(self.orig_headers.keys()) + \ list(self.custom_columns) def col_idx(name): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 773a4bdc9f..c7c4926b14 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -554,6 +554,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def search_term_to_field_key(self, term): return self.field_metadata.search_term_to_key(term) + def custom_field_metadata(self): + return self.field_metadata.custom_field_metadata() + + def all_metadata(self): + return self.field_metadata.all_metadata() + def metadata_for_field(self, key): return self.field_metadata[key] diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index bac423f46d..d608dca49d 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -411,7 +411,7 @@ class FieldMetadata(dict): l[k] = self._tb_cats[k] return l - def get_custom_field_metadata(self): + def custom_field_metadata(self): l = {} for k in self._tb_cats: if self._tb_cats[k]['is_custom']: diff --git a/src/calibre/library/server/__init__.py b/src/calibre/library/server/__init__.py index 5050dfaa99..7cdea9f602 100644 --- a/src/calibre/library/server/__init__.py +++ b/src/calibre/library/server/__init__.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import os -from calibre.utils.config import Config, StringConfig, config_dir +from calibre.utils.config import Config, StringConfig, config_dir, tweaks listen_on = '0.0.0.0' @@ -46,6 +46,16 @@ def server_config(defaults=None): 'to disable grouping.')) return c +def custom_fields_to_display(db): + ckeys = db.custom_field_keys() + yes_fields = set(tweaks['content_server_will_display']) + no_fields = set(tweaks['content_server_wont_display']) + if '*' in yes_fields: + yes_fields = set(ckeys) + if '*' in no_fields: + no_fields = set(ckeys) + return frozenset(yes_fields - no_fields) + def main(): from calibre.library.server.main import main return main() diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index c0a3c122cd..071c7b1077 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -13,6 +13,7 @@ from lxml import html from lxml.html.builder import HTML, HEAD, TITLE, LINK, DIV, IMG, BODY, \ OPTION, SELECT, INPUT, FORM, SPAN, TABLE, TR, TD, A, HR +from calibre.library.server import custom_fields_to_display from calibre.library.server.utils import strftime, format_tag_string from calibre.ebooks.metadata import fmt_sidx from calibre.constants import __appname__ @@ -197,7 +198,7 @@ class MobileServer(object): self.sort(items, sort, (order.lower().strip() == 'ascending')) CFM = self.db.field_metadata - CKEYS = [key for key in sorted(CFM.get_custom_fields(), + CKEYS = [key for key in sorted(custom_fields_to_display(self.db), cmp=lambda x,y: cmp(CFM[x]['name'].lower(), CFM[y]['name'].lower()))] # This method uses its own book dict, not the Metadata dict. The loop diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index d495f58fa1..0e6917c504 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -17,6 +17,7 @@ import routes from calibre.constants import __appname__ from calibre.ebooks.metadata import fmt_sidx from calibre.library.comments import comments_to_html +from calibre.library.server import custom_fields_to_display from calibre.library.server.utils import format_tag_string from calibre import guess_type from calibre.utils.ordered_dict import OrderedDict @@ -277,7 +278,7 @@ class AcquisitionFeed(NavFeed): db): NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url) CFM = db.field_metadata - CKEYS = [key for key in sorted(CFM.get_custom_fields(), + CKEYS = [key for key in sorted(custom_fields_to_display(db), cmp=lambda x,y: cmp(CFM[x]['name'].lower(), CFM[y]['name'].lower()))] for item in items: diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index 45ffdc2737..12fcc217f0 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -11,6 +11,7 @@ import cherrypy from lxml.builder import ElementMaker from lxml import etree +from calibre.library.server import custom_fields_to_display from calibre.library.server.utils import strftime, format_tag_string from calibre.ebooks.metadata import fmt_sidx from calibre.constants import preferred_encoding @@ -94,7 +95,7 @@ class XMLServer(object): c = kwargs.pop('comments') CFM = self.db.field_metadata - CKEYS = [key for key in sorted(CFM.get_custom_fields(), + CKEYS = [key for key in sorted(custom_fields_to_display(self.db), cmp=lambda x,y: cmp(CFM[x]['name'].lower(), CFM[y]['name'].lower()))] custcols = [] From 67c7555fd0eb22802892ec716ec5564f3d423bc4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 11:36:26 +0100 Subject: [PATCH 06/13] Fix content server gui.js bug where it put '...' on the end of a list even if the list was exactly the right size. --- resources/content_server/gui.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/content_server/gui.js b/resources/content_server/gui.js index bd0743a854..86cd04289b 100644 --- a/resources/content_server/gui.js +++ b/resources/content_server/gui.js @@ -63,8 +63,9 @@ function render_book(book) { if (tags) { t = tags.split(':&:', 2); m = parseInt(t[0]); + tall = t[1].split(','); t = t[1].split(',', m); - if (t.length == m) t[m] = '...' + if (tall.length > m) t[m] = '...' title += 'Tags=[{0}] '.format(t.join(',')); } custcols = book.attr("custcols").split(',') From ad69ef985a04b5485550f83661ee0b56723605f1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 12:27:39 +0100 Subject: [PATCH 07/13] Add a 'test' function to templates. Analogous to lookup, but inserts plain text instead of a template. --- src/calibre/utils/formatter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index a98f0e7f45..5c5893576c 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -23,6 +23,12 @@ class TemplateFormatter(string.Formatter): else: return self.vformat('{'+field_not_set.strip()+'}', [], self.kwargs) + def _test(self, val, value_if_set, value_not_set): + if val: + return value_if_set + else: + return value_not_set + def _ifempty(self, val, value_if_empty): if val: return val @@ -45,6 +51,7 @@ class TemplateFormatter(string.Formatter): 'ifempty' : (1, _ifempty), 'lookup' : (2, _lookup), 'shorten' : (3, _shorten), + 'test' : (2, _lookup), } format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') From b2a6ed3af48a47c5cc7bb3372a3b084a004b7fcf Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 12:43:54 +0100 Subject: [PATCH 08/13] 1) fix bulk edit to not display a tab if library has only composite columns 2) fix a reference to get_custom_field_metadata that I somehow missed. --- src/calibre/gui2/dialogs/metadata_bulk.py | 3 ++- src/calibre/gui2/preferences/columns.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index a9e45087fd..1e3576e333 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -167,7 +167,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.tag_editor_button.clicked.connect(self.tag_editor) self.autonumber_series.stateChanged[int].connect(self.auto_number_changed) - if len(db.custom_column_label_map) == 0: + if len([k for k in db.custom_field_metadata().values() + if k['datatype'] != 'composite']) == 0: self.central_widget.removeTab(1) else: self.create_custom_column_editors() diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index c1b9230f42..03a50e6f3a 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -21,7 +21,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): self.gui = gui db = self.gui.library_view.model().db - self.custcols = copy.deepcopy(db.field_metadata.get_custom_field_metadata()) + self.custcols = copy.deepcopy(db.field_metadata.custom_field_metadata()) self.column_up.clicked.connect(self.up_column) self.column_down.clicked.connect(self.down_column) From 529756238340e0dd66c3be0cfc1f5f7a8a178cfa Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 12:57:49 +0100 Subject: [PATCH 09/13] Refactor code to clean interfaces and remove overly complex loop in bulk edit --- src/calibre/gui2/dialogs/metadata_bulk.py | 3 +-- src/calibre/library/database2.py | 8 ++++---- src/calibre/library/field_metadata.py | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 1e3576e333..b14390e001 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -167,8 +167,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): self.tag_editor_button.clicked.connect(self.tag_editor) self.autonumber_series.stateChanged[int].connect(self.auto_number_changed) - if len([k for k in db.custom_field_metadata().values() - if k['datatype'] != 'composite']) == 0: + if len(db.custom_field_keys(include_composites=False)) == 0: self.central_widget.removeTab(1) else: self.create_custom_column_editors() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index c7c4926b14..22175d3910 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -539,8 +539,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def standard_field_keys(self): return self.field_metadata.standard_field_keys() - def custom_field_keys(self): - return self.field_metadata.custom_field_keys() + def custom_field_keys(self, include_composites=True): + return self.field_metadata.custom_field_keys(include_composites) def all_field_keys(self): return self.field_metadata.all_field_keys() @@ -554,8 +554,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def search_term_to_field_key(self, term): return self.field_metadata.search_term_to_key(term) - def custom_field_metadata(self): - return self.field_metadata.custom_field_metadata() + def custom_field_metadata(self, include_composites=True): + return self.field_metadata.custom_field_metadata(include_composites) def all_metadata(self): return self.field_metadata.all_metadata() diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index d608dca49d..37393d0d2c 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -358,10 +358,14 @@ class FieldMetadata(dict): if self._tb_cats[k]['kind']=='field' and not self._tb_cats[k]['is_custom']] - def custom_field_keys(self): - return [k for k in self._tb_cats.keys() - if self._tb_cats[k]['kind']=='field' and - self._tb_cats[k]['is_custom']] + def custom_field_keys(self, include_composites=True): + res = [] + for k in self._tb_cats.keys(): + fm = self._tb_cats[k] + if fm['kind']=='field' and fm['is_custom'] and \ + (fm['datatype'] != 'composite' or include_composites): + res.append(k) + return res def all_field_keys(self): return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field'] @@ -402,20 +406,16 @@ class FieldMetadata(dict): return self.custom_label_to_key_map[label] raise ValueError('Unknown key [%s]'%(label)) - def get_custom_fields(self): - return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']] - def all_metadata(self): l = {} for k in self._tb_cats: l[k] = self._tb_cats[k] return l - def custom_field_metadata(self): + def custom_field_metadata(self, include_composites=True): l = {} - for k in self._tb_cats: - if self._tb_cats[k]['is_custom']: - l[k] = self._tb_cats[k] + for k in self.custom_field_keys(include_composites): + l[k] = self._tb_cats[k] return l def add_custom_field(self, label, table, column, datatype, colnum, name, From 25905a349c745108abd6290d5835b109359a72de Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 13:20:26 +0100 Subject: [PATCH 10/13] Test the 'test' function. Add 're' function and test it. --- src/calibre/utils/formatter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 5c5893576c..c6bcaa1c3e 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -43,6 +43,9 @@ class TemplateFormatter(string.Formatter): else: return val + def _re(self, val, pattern, replacement): + return re.sub(pattern, replacement, val) + functions = { 'uppercase' : (0, lambda s,x: x.upper()), 'lowercase' : (0, lambda s,x: x.lower()), @@ -50,8 +53,9 @@ class TemplateFormatter(string.Formatter): 'capitalize' : (0, lambda s,x: x.capitalize()), 'ifempty' : (1, _ifempty), 'lookup' : (2, _lookup), + 're' : (2, _re), 'shorten' : (3, _shorten), - 'test' : (2, _lookup), + 'test' : (2, _test), } format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') From 211bb81113865b1cb38c2f8697b33694a4bb38fe Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 14:43:07 +0100 Subject: [PATCH 11/13] Put back the sanitize after split on slashes. --- src/calibre/library/save_to_disk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index a58686f709..e479d27121 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -166,6 +166,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, components = safe_formatter.safe_format(template, format_args, '', mi, sanitize=sanitize_func) components = [x.strip() for x in components.split('/') if x.strip()] + components = [sanitize_func(x) for x in components if x] if not components: components = [str(id)] components = [x.encode(filesystem_encoding, 'replace') if isinstance(x, From 93c8836cb6622579323d95f86dfb6fa03dda78cb Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 15:18:03 +0100 Subject: [PATCH 12/13] Changes to template faq --- src/calibre/manual/template_lang.rst | 85 +++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 14 deletions(-) diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index 59e5c1da4c..6d87a90c93 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -7,9 +7,9 @@ The |app| template language ======================================================= The |app| template language is used in various places. It is used to control the folder structure and file name when saving files from the |app| library to the disk or eBook reader. -It is used to define "virtual" columns that contain data from other columns and so on. +It is also used to define "virtual" columns that contain data from other columns and so on. -In essence, the template language is very simple. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is:: +The basi template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is:: {author_sort}/{title}/{title} - {authors} @@ -17,7 +17,9 @@ For the book "The Foundation" by "Isaac Asimov" it will become:: Asimov, Isaac/The Foundation/The Foundation - Isaac Asimov -You can use all the various metadata fields available in calibre in a template, including the custom columns you have created yourself. To find out the template name for a column sinply hover your mouse over the column header. Names for custom fields (columns you have created yourself) are always prefixed by an #. For series type fields, there is always an additional field named ``series_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index. In addition to the column based fields, you also can use:: +You can use all the various metadata fields available in calibre in a template, including any custom columns you have created yourself. To find out the template name for a column simply hover your mouse over the column header. Names for custom fields (columns you have created yourself) always have a # as the first character. For series type custom fields, there is always an additional field named ``#seriesname_index`` that becomes the series index for that series. So if you have a custom series field named #myseries, there will also be a field named #myseries_index. + +In addition to the column based fields, you also can use:: {formats} - A list of formats available in the calibre library for a book {isbn} - The ISBN number of the book @@ -26,7 +28,7 @@ If a particular book does not have a particular piece of metadata, the field in {author_sort}/{series}/{title} {series_index} -will become:: +If a book has a series, the template will produce:: {Asimov, Isaac}/Foundation/Second Foundation - 3 @@ -40,35 +42,90 @@ and if a book does not have a series:: Advanced formatting ---------------------- -You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted. +You can do more than just simple substitution with the templates. You can also conditionally include text and control how the substituted data is formatted. + +First, conditionally including text. There are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax. -Regarding conditionally including text: there are cases where you might want to have text appear in the output only if a field is not empty. A common case is series and series_index, where you want either nothing or the two values with a hyphen between them. Calibre handles this case using a special field syntax. For example, assume you want to use the template {series} - {series_index} - {title} -Unfortunately, if the book has no series, the answer will be '- - title'. Many people would rather it be simply 'title', without the hyphens. To do this, use the extended syntax {some_text|field|other_text}. When you use this syntax, if field has the value SERIES then the result will be some_textSERIESother_text. If field has no value, then the result will be the empty string (nothing). Using this syntax, we can solve the above series problem with the template:: +If the book has no series, the answer will be '- - title'. Many people would rather the result be simply 'title', without the hyphens. To do this, use the extended syntax `{field:|prefix_text|suffix_text}`. When you use this syntax, if field has the value SERIES then the result will be prefix_textSERIESsuffix_text. If field has no value, then the result will be the empty string (nothing). The prefix and suffix can contain blanks. - {series}{ - |series_index| - }{title} +Using this syntax, we can solve the above series problem with the template: -The hyphens will be included only if the book has a series index. Note: you must either use no | characters or both of them. Using one, such as in {field| - }, is not allowed. It is OK to not provide any text for one side or the other, such as in {\|series\| - }. Using {\|title\|} is the same as using {title}. + {series}{series_index:| - | - }{title} -Now to formatting. Suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick:: +The hyphens will be included only if the book has a series index. + +Notes: you must include the : character if you want to use a prefix or a suffix. You must either use no | characters or both of them; using one, as in `{field:| - }`, is not allowed. It is OK not to provide any text for one side or the other, such as in `{series:|| - }`. Using `{title:||}` is the same as using `{title}`. + +Second: formatting. Suppose you wanted to ensure that the series_index is always formatted as three digits with leading zeros. This would do the trick:: {series_index:0>3s} - Three digits with leading zeros -If instead of leading zeros you want leading spaces, use:: +If instead of leading zeros you want leading spaces, use: - {series_index:>3s} - Thre digits with leading spaces + {series_index:>3s} - Three digits with leading spaces -For trailing zeros, use:: +For trailing zeros, use: {series_index:0<3s} - Three digits with trailing zeros -If you want only the first two letters of the data to be rendered, use:: +If you want only the first two letters of the data, use:: {author_sort:.2} - Only the first two letter of the author sort name The |app| template language comes from python and for more details on the syntax of these advanced formatting operations, look at the `Python documentation `_. +Advanced features +------------------ + +Using templates in custom columns +---------------------------------- + +There are sometimes cases where you want to display metadata that |app| does not normally display, or to display data in a way different from how |app| normally does. For example, you might want to display the ISBN, a field that |app| does not display. You can use custom columns for this. To do so, you create a column with the type 'column built from other columns' (hereafter called composite columns), enter a template, and |app| will display in the column the result of evaluating that template. To display the isbn, create the column and enter `{isbn}` into the template box. To display a column containing the values of two series custom columns separated by a comma, use `{#series1:||,}{#series2}`. + +Composite columns can use any template option, including formatting. + +You cannot change the data contained in a composite column. If you edit a composite column by double-clicking on any item, you will open the template for editing, not the underlying data. Editing the template on the GUI is a quick way of testing and changing composite columns. + +Using functions in templates +----------------------------- + +Suppose you want to display the value of a field in upper case, when that field is normally in title case. You can do this (and many more things) using the functions available for templates. For example, to display the title in upper case, use `{title:uppercase()}`. To display it in title case, use `{title:titlecase()}`. + +Function references replace the formatting specification, going after the : and before the first `|` or the closing `}`. Functions must always end with `()`. Some functions take extra values (arguments), and these go inside the `()`. + +The syntax for using functions is `{field:function(arguments)}`, or `{field:function(arguments)|prefix|suffix}`. Argument values cannot contain a comma, because it is used to separate arguments. Functions return the value of the field used in the template, suitably modified. + +The functions available are: + +* `lowercase()` -- return value of the field in lower case. +* `uppercase()` -- return the value of the field in upper case. +* `titlecase()` -- return the value of the field in title case. +* `capitalize()` -- return the value as capitalized. +* `ifempty(text)` -- if the field is not empty, return the value of the field. Otherwise return `text`. +* `test(text if not empty, text if empty)` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`. +* `shorten(left chars, middle text, right chars)` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use `{title:shorten(9,-,5)}, the result will be `Ancient E-nhoe`. If the field's length is less than `left chars` + `right chars` + the length of `middle text`, then the field will be used intact. For example, the title `The Dome` would not be changed. +* `lookup(field if not empty, field if empty)` -- like test, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later). +* `re(pattern, replacement)` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions. + +Special notes for save/send templates +------------------------------------- + +Special processing is applied when a template is used in a `save to disk` or `send to device` template. The values of the fields are cleaned, replacing characters that are special to file systems with underscores, including slashes. This means that field text cannot be used to create folders. However, slashes are not changed in prefix or suffix strings, so slashes in these strings will cause folders to be created. Because of this, you can create variable-depth folder structure. + +For example, assume we want the folder structure `series/series_index - title`, with the caveat that if series does not exist, then the title should be in the top folder. The template to do this is + + {series:||/}{series_index:|| - }{title} + +The slash and the hyphen appear only if series is not empty. + +The lookup function lets us do even fancier processing. For example, assume we want the following: if a book has a series, then we want the folder structure `series/series index - title.fmt`. If the book does not have a series, then we want the folder structure `genre/author_sort/title.fmt`. If the book has no genre, use 'Unknown'. We want two completely different paths, depending on the value of series. + +To accomplish this, we: +1. Create a composite field (call it AA) containing `{series:||}/{series_index} - {title'}`. If the series is not empty, then this template will produce `series/series_index - title`. +2. Create a composite field (call it BB) containing `{#genre:ifempty(Unknown)}/{author_sort}/{title}`. This template produces `genre/author_sort/title`, where an empty genre is replaced wuth `Unknown`. +3. Set the save template to `{series:lookup(AA,BB)}`. This template chooses composite field AA if series is not empty, and composite field BB if series is empty. We therefore have two completely different save paths, depending on whether or not `series` is empty. From 02e9160f3752c179391bfddbb1b3febb9cb3a517 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 24 Sep 2010 15:23:35 +0100 Subject: [PATCH 13/13] Fix typo --- src/calibre/manual/template_lang.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index 6d87a90c93..2c49cfe308 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -9,7 +9,7 @@ The |app| template language The |app| template language is used in various places. It is used to control the folder structure and file name when saving files from the |app| library to the disk or eBook reader. It is also used to define "virtual" columns that contain data from other columns and so on. -The basi template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is:: +The basic template language is very simple, but has very powerful advanced features. The basic idea is that a template consists of names in curly brackets that are then replaced by the corresponding metadata from the book being processed. So, for example, the default template used for saving books to device in |app| is:: {author_sort}/{title}/{title} - {authors}