From 405e4c3f085cb5363042bdd8d8246f9a07ba0bc9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 May 2010 18:55:14 -0600 Subject: [PATCH 01/18] Add support for storage card to Kobo driver --- src/calibre/devices/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 9d58bbcae6..c7e0356f32 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -45,7 +45,7 @@ class KOBO(USBMS): BCD = [0x0110] VENDOR_NAME = 'KOBO_INC' - WINDOWS_MAIN_MEM = '.KOBOEREADER' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '.KOBOEREADER' EBOOK_DIR_MAIN = '' SUPPORTS_SUB_DIRS = True From cb09c7ef4670947fed27d8c68321e1acc34a013a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 09:57:04 +0100 Subject: [PATCH 02/18] commit before changing tag_browser_categories to field_metadata, just in case... --- src/calibre/gui2/dialogs/tag_categories.py | 2 +- src/calibre/library/caches.py | 114 ++++++++--------- src/calibre/library/custom_columns.py | 6 +- src/calibre/library/database2.py | 19 +-- src/calibre/library/field_metadata.py | 142 +++++++++++---------- 5 files changed, 137 insertions(+), 146 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index fdec767d4d..50a0dccfb3 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -49,7 +49,7 @@ class TagCategories(QDialog, Ui_TagCategories): cc_map = self.db.custom_column_label_map for cc in cc_map: if cc_map[cc]['datatype'] == 'text': - self.category_labels.append(db.tag_browser_categories.get_search_label(cc)) + self.category_labels.append(db.tag_browser_categories.label_to_key(cc)) category_icons.append(cc_icon) category_values.append(lambda col=cc: self.db.all_custom(label=col)) category_names.append(cc_map[cc]['name']) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 36698533c5..20cfce02bd 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -157,7 +157,7 @@ class ResultCache(SearchQueryParser): self.first_sort = True self.search_restriction = '' self.tag_browser_categories = tag_browser_categories - self.all_search_locations = tag_browser_categories.get_search_labels() + self.all_search_locations = tag_browser_categories.get_search_keys() SearchQueryParser.__init__(self, self.all_search_locations) self.build_date_relop_dict() self.build_numeric_relop_dict() @@ -249,10 +249,10 @@ class ResultCache(SearchQueryParser): query = query[p:] if relop is None: (p, relop) = self.date_search_relops['='] - if location in self.custom_column_label_map: - loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']] - else: - loc = self.FIELD_MAP[{'date':'timestamp', 'pubdate':'pubdate'}[location]] + + if location == 'date': + location = 'timestamp' + loc = self.tag_browser_categories[location]['rec_index'] if query == _('today'): qd = now() @@ -310,22 +310,18 @@ class ResultCache(SearchQueryParser): query = query[p:] if relop is None: (p, relop) = self.numeric_search_relops['='] - if location in self.custom_column_label_map: - loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']] - dt = self.custom_column_label_map[location]['datatype'] - if dt == 'int': - cast = (lambda x: int (x)) - adjust = lambda x: x - elif dt == 'rating': - cast = (lambda x: int (x)) - adjust = lambda x: x/2 - elif dt == 'float': - cast = lambda x : float (x) - adjust = lambda x: x - else: - loc = self.FIELD_MAP['rating'] + + loc = self.tag_browser_categories[location]['rec_index'] + dt = self.tag_browser_categories[location]['datatype'] + if dt == 'int': + cast = (lambda x: int (x)) + adjust = lambda x: x + elif dt == 'rating': cast = (lambda x: int (x)) adjust = lambda x: x/2 + elif dt == 'float': + cast = lambda x : float (x) + adjust = lambda x: x try: q = cast(query) @@ -347,21 +343,22 @@ class ResultCache(SearchQueryParser): matches = set([]) if query and query.strip(): location = location.lower().strip() + if location in ('tag', 'author', 'format', 'comment'): + location += 's' ### take care of dates special case if (location in ('pubdate', 'date')) or \ - ((location in self.custom_column_label_map) and \ - self.custom_column_label_map[location]['datatype'] == 'datetime'): + (location in self.tag_browser_categories and \ + self.tag_browser_categories[location]['datatype'] == 'datetime'): return self.get_dates_matches(location, query.lower()) - ### take care of numerics special case - if location == 'rating' or \ - (location in self.custom_column_label_map and - self.custom_column_label_map[location]['datatype'] in - ('rating', 'int', 'float')): + ### take care of numbers special case + if location in self.tag_browser_categories and \ + self.tag_browser_categories[location]['datatype'] in \ + ('rating', 'int', 'float'): return self.get_numeric_matches(location, query.lower()) - ### everything else + ### everything else, or 'all' matches matchkind = CONTAINS_MATCH if (len(query) > 1): if query.startswith('\\'): @@ -372,57 +369,48 @@ class ResultCache(SearchQueryParser): elif query.startswith('~'): matchkind = REGEXP_MATCH query = query[1:] - if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D + if matchkind != REGEXP_MATCH: + # leave case in regexps because it can be significant e.g. \S \W \D query = query.lower() if not isinstance(query, unicode): query = query.decode('utf-8') - if location in ('tag', 'author', 'format', 'comment'): - location += 's' - MAP = {} - # Fields not used when matching against text contents. These are - # the non-text fields - EXCLUDE_FIELDS = [] - - # get the db columns for the standard searchables - for x in self.tag_browser_categories: - if len(self.tag_browser_categories[x]['search_labels']) and \ - not self.tag_browser_categories.is_custom_field(x): - MAP[x] = self.tag_browser_categories[x]['rec_index'] - if self.tag_browser_categories[x]['datatype'] != 'text': - EXCLUDE_FIELDS.append(MAP[x]) - - # add custom columns to MAP. Put the column's type into IS_CUSTOM - IS_CUSTOM = [] + db_col = {} + # fields to not check when matching against text. + exclude_fields = [] + col_datatype = [] for x in range(len(self.FIELD_MAP)): - IS_CUSTOM.append('') + col_datatype.append('') + for x in self.tag_browser_categories: + if len(self.tag_browser_categories[x]['search_keys']): + db_col[x] = self.tag_browser_categories[x]['rec_index'] + if self.tag_browser_categories[x]['datatype'] not in ['text', 'comments']: + exclude_fields.append(db_col[x]) + if self.tag_browser_categories.is_custom_field(x): + col_datatype[db_col[x]] = self.tag_browser_categories[x]['datatype'] # normal and custom ratings columns use the same code - IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating' - for x in self.tag_browser_categories.get_custom_fields(): - if self.tag_browser_categories[x]['datatype'] != "datetime": - MAP[x] = self.FIELD_MAP[self.tag_browser_categories[x]['colnum']] - IS_CUSTOM[MAP[x]] = self.tag_browser_categories[x]['datatype'] + col_datatype[self.FIELD_MAP['rating']] = 'rating' - SPLITABLE_FIELDS = [MAP['authors'], MAP['tags'], MAP['formats']] + splitable_fields = [db_col['authors'], db_col['tags'], db_col['formats']] for x in self.tag_browser_categories.get_custom_fields(): if self.tag_browser_categories[x]['is_multiple']: - SPLITABLE_FIELDS.append(MAP[x]) + splitable_fields.append(db_col[x]) try: rating_query = int(query) * 2 except: rating_query = None - location = [location] if location != 'all' else list(MAP.keys()) + location = [location] if location != 'all' else list(db_col.keys()) for i, loc in enumerate(location): - location[i] = MAP[loc] + location[i] = db_col[loc] # get the tweak here so that the string lookup and compare aren't in the loop bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes' for loc in location: - if loc == MAP['authors']: + if loc == db_col['authors']: ### DB stores authors with commas changed to bars, so change query q = query.replace(',', '|'); else: @@ -431,7 +419,7 @@ class ResultCache(SearchQueryParser): for item in self._data: if item is None: continue - if IS_CUSTOM[loc] == 'bool': # complexity caused by the two-/three-value tweak + if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak v = item[loc] if not bools_are_tristate: if v is None or not v: # item is None or set to false @@ -466,18 +454,18 @@ class ResultCache(SearchQueryParser): matches.add(item[0]) continue - if IS_CUSTOM[loc] == 'rating': # get here if 'all' query + if col_datatype[loc] == 'rating': # get here if 'all' query if rating_query and rating_query == int(item[loc]): matches.add(item[0]) continue try: # a conversion below might fail # relationals not supported in 'all' queries - if IS_CUSTOM[loc] == 'float': + if col_datatype[loc] == 'float': if float(query) == item[loc]: matches.add(item[0]) continue - if IS_CUSTOM[loc] == 'int': + if col_datatype[loc] == 'int': if int(query) == item[loc]: matches.add(item[0]) continue @@ -486,9 +474,9 @@ class ResultCache(SearchQueryParser): # no further match is possible continue - if loc not in EXCLUDE_FIELDS: - if loc in SPLITABLE_FIELDS: - if IS_CUSTOM[loc]: + if loc not in exclude_fields: + if loc in splitable_fields: + if col_datatype[loc]: vals = item[loc].split('|') else: vals = item[loc].split(',') diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 535b8cfb72..5b5fcadf59 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -144,14 +144,14 @@ class CustomColumns(object): for k in sorted(self.custom_column_label_map.keys()): v = self.custom_column_label_map[k] if v['normalized']: - searchable = True + is_category = True else: - searchable = False + is_category = False tn = 'custom_column_{0}'.format(v['num']) self.tag_browser_categories.add_custom_field(label=v['label'], table=tn, column='value', datatype=v['datatype'], is_multiple=v['is_multiple'], colnum=v['num'], - name=v['name'], searchable=searchable) + name=v['name'], is_category=is_category) def get_custom(self, idx, label=None, num=None, index_is_id=False): if label is not None: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index a69d6bab57..939874ed3b 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -656,11 +656,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') - #### First, build the standard and custom-column categories #### tb_cats = self.tag_browser_categories + + # remove all user categories from tag_browser_categories. They can + # easily come and go. We will add all the existing ones in below. + for k in tb_cats.keys(): + if tb_cats[k]['kind'] in ['user', 'search']: + del tb_cats[k] + + #### First, build the standard and custom-column categories #### for category in tb_cats.keys(): cat = tb_cats[category] - if cat['kind'] == 'not_cat': + if not cat['is_category']: continue tn = cat['table'] categories[category] = [] #reserve the position in the ordered list @@ -680,7 +687,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # icon_map is not None if get_categories is to store an icon and # possibly a tooltip in the tag structure. icon, tooltip = None, '' - label = tb_cats.get_field_label(category) + label = tb_cats.key_to_label(category) if icon_map: if not tb_cats.is_custom_field(category): if category in icon_map: @@ -737,12 +744,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): #### Now do the user-defined categories. #### user_categories = prefs['user_categories'] - # remove all user categories from tag_browser_categories. They can - # easily come and go. We will add all the existing ones in below. - for k in tb_cats.keys(): - if tb_cats[k]['kind'] in ['user', 'search']: - del tb_cats[k] - # We want to use same node in the user category as in the source # category. To do that, we need to find the original Tag node. There is # a time/space tradeoff here. By converting the tags into a map, we can diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 0134db712f..f8f3695473 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -4,7 +4,6 @@ Created on 25 May 2010 @author: charles ''' -from UserDict import DictMixin from calibre.utils.ordered_dict import OrderedDict class TagsIcons(dict): @@ -22,7 +21,7 @@ class TagsIcons(dict): raise ValueError('Missing category icon [%s]'%a) self[a] = icon_dict[a] -class FieldMetadata(dict, DictMixin): +class FieldMetadata(dict): # kind == standard: is tag category. May be a search label. Is db col # or is specially handled (e.g., news) @@ -41,86 +40,86 @@ class FieldMetadata(dict, DictMixin): ('authors', {'table':'authors', 'column':'name', 'datatype':'text', 'is_multiple':False, 'kind':'standard', 'name':_('Authors'), - 'search_labels':['authors', 'author'], - 'is_custom':False}), + 'search_keys':['authors', 'author'], + 'is_custom':False, 'is_category':True}), ('series', {'table':'series', 'column':'name', 'datatype':'text', 'is_multiple':False, 'kind':'standard', 'name':_('Series'), - 'search_labels':['series'], - 'is_custom':False}), + 'search_keys':['series'], + 'is_custom':False, 'is_category':True}), ('formats', {'table':None, 'column':None, 'datatype':'text', 'is_multiple':False, # must think what type this is! 'kind':'standard', 'name':_('Formats'), - 'search_labels':['formats', 'format'], - 'is_custom':False}), + 'search_keys':['formats', 'format'], + 'is_custom':False, 'is_category':True}), ('publisher', {'table':'publishers', 'column':'name', 'datatype':'text', 'is_multiple':False, 'kind':'standard', 'name':_('Publishers'), - 'search_labels':['publisher'], - 'is_custom':False}), + 'search_keys':['publisher'], + 'is_custom':False, 'is_category':True}), ('rating', {'table':'ratings', 'column':'rating', 'datatype':'rating', 'is_multiple':False, 'kind':'standard', 'name':_('Ratings'), - 'search_labels':['rating'], - 'is_custom':False}), + 'search_keys':['rating'], + 'is_custom':False, 'is_category':True}), ('news', {'table':'news', 'column':'name', 'datatype':None, 'is_multiple':False, 'kind':'standard', 'name':_('News'), - 'search_labels':[], - 'is_custom':False}), + 'search_keys':[], + 'is_custom':False, 'is_category':True}), ('tags', {'table':'tags', 'column':'name', 'datatype':'text', 'is_multiple':True, 'kind':'standard', 'name':_('Tags'), - 'search_labels':['tags', 'tag'], - 'is_custom':False}), + 'search_keys':['tags', 'tag'], + 'is_custom':False, 'is_category':True}), ('author_sort',{'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('comments', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['comments', 'comment'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['comments', 'comment'], 'is_custom':False, 'is_category':False}), ('cover', {'table':None, 'column':None, 'datatype':None, - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['cover'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['cover'], 'is_custom':False, 'is_category':False}), ('flags', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('id', {'table':None, 'column':None, 'datatype':'int', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('isbn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['isbn'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['isbn'], 'is_custom':False, 'is_category':False}), ('lccn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('ondevice', {'table':None, 'column':None, 'datatype':'bool', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('path', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('pubdate', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['pubdate'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['pubdate'], 'is_custom':False, 'is_category':False}), ('series_index',{'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('sort', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('size', {'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ('timestamp', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['date'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['date'], 'is_custom':False, 'is_category':False}), ('title', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['title'], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':['title'], 'is_custom':False, 'is_category':False}), ('uuid', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + 'is_multiple':False, 'kind':'standard', 'name':None, + 'search_keys':[], 'is_custom':False, 'is_category':False}), ] # search labels that are not db columns @@ -150,6 +149,12 @@ class FieldMetadata(dict, DictMixin): for key in self._tb_cats: yield key + def has_key(self, key): + return key in self._tb_cats + + def __contains__(self, key): + return self.has_key(key) + def keys(self): return self._tb_cats.keys() @@ -164,12 +169,12 @@ class FieldMetadata(dict, DictMixin): def is_custom_field(self, key): return key.startswith(self.custom_field_prefix) - def get_field_label(self, key): + def key_to_label(self, key): if 'label' not in self._tb_cats[key]: return key return self._tb_cats[key]['label'] - def get_search_label(self, label): + def label_to_key(self, label, prefer_custom=False): if 'label' in self._tb_cats: return label if self.is_custom_field(label): @@ -179,22 +184,17 @@ class FieldMetadata(dict, DictMixin): def get_custom_fields(self): return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']] - def add_custom_field(self, label, table, column, datatype, - is_multiple, colnum, name, searchable): + def add_custom_field(self, label, table, column, datatype, colnum, + name, is_multiple, is_category): fn = self.custom_field_prefix + label if fn in self._tb_cats: raise ValueError('Duplicate custom field [%s]'%(label)) - if searchable: - sl = [fn] - kind = 'standard' - else: - sl = [] - kind = 'not_cat' self._tb_cats[fn] = {'table':table, 'column':column, 'datatype':datatype, 'is_multiple':is_multiple, - 'kind':kind, 'name':name, - 'search_labels':sl, 'label':label, - 'colnum':colnum, 'is_custom':True} + 'kind':'standard', 'name':name, + 'search_keys':[fn], 'label':label, + 'colnum':colnum, 'is_custom':True, + 'is_category':is_category} def set_field_record_index(self, label, index, prefer_custom=False): if prefer_custom: @@ -214,7 +214,8 @@ class FieldMetadata(dict, DictMixin): self._tb_cats[label] = {'table':None, 'column':None, 'datatype':None, 'is_multiple':False, 'kind':'user', 'name':name, - 'search_labels':[], 'is_custom':False} + 'search_keys':[], 'is_custom':False, + 'is_category':True} def add_search_category(self, label, name): if label in self._tb_cats: @@ -222,7 +223,8 @@ class FieldMetadata(dict, DictMixin): self._tb_cats[label] = {'table':None, 'column':None, 'datatype':None, 'is_multiple':False, 'kind':'search', 'name':name, - 'search_labels':[], 'is_custom':False} + 'search_keys':[], 'is_custom':False, + 'is_category':True} # DEFAULT_LOCATIONS = frozenset([ # 'all', @@ -247,13 +249,13 @@ class FieldMetadata(dict, DictMixin): # ]) - def get_search_labels(self): - s_labels = [] + def get_search_keys(self): + s_keys = [] for v in self._tb_cats.itervalues(): - map((lambda x:s_labels.append(x)), v['search_labels']) + map((lambda x:s_keys.append(x)), v['search_keys']) for v in self.search_items: - s_labels.append(v) -# if set(s_labels) != self.DEFAULT_LOCATIONS: + s_keys.append(v) +# if set(s_keys) != self.DEFAULT_LOCATIONS: # print 'search labels and default_locations do not match:' -# print set(s_labels) ^ self.DEFAULT_LOCATIONS - return s_labels +# print set(s_keys) ^ self.DEFAULT_LOCATIONS + return s_keys From 2ea3fdf5a7ea9981f6f0ddc29d22a2b17925fc46 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 13:27:05 +0100 Subject: [PATCH 03/18] Changes to better use FieldMetadata, plus rename of the class. --- src/calibre/gui2/dialogs/tag_categories.py | 2 +- src/calibre/gui2/library/models.py | 2 +- src/calibre/gui2/tag_view.py | 4 +- src/calibre/library/caches.py | 70 ++-- src/calibre/library/custom_columns.py | 10 +- src/calibre/library/database2.py | 19 +- src/calibre/library/field_metadata.py | 421 ++++++++++++++------- src/calibre/library/schema_upgrades.py | 2 +- src/calibre/library/server/opds.py | 2 +- 9 files changed, 344 insertions(+), 188 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 50a0dccfb3..fcf517e571 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -49,7 +49,7 @@ class TagCategories(QDialog, Ui_TagCategories): cc_map = self.db.custom_column_label_map for cc in cc_map: if cc_map[cc]['datatype'] == 'text': - self.category_labels.append(db.tag_browser_categories.label_to_key(cc)) + self.category_labels.append(db.field_metadata.label_to_key(cc)) category_icons.append(cc_icon) category_values.append(lambda col=cc: self.db.all_custom(label=col)) category_names.append(cc_map[cc]['name']) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index a871ce2aa3..072f81e2d1 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -633,7 +633,7 @@ class BooksModel(QAbstractTableModel): # {{{ if role == Qt.ToolTipRole: ht = self.column_map[section] if self.is_custom_column(self.column_map[section]): - ht = self.db.tag_browser_categories.custom_field_prefix + ht + ht = self.db.field_metadata.custom_field_prefix + ht if ht == 'timestamp': # change help text because users know this field as 'date' ht = 'date' return QVariant(_('The lookup/search name is "{0}"').format(ht)) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 3882e4e174..8ecc26e30c 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -224,7 +224,7 @@ class TagsModel(QAbstractItemModel): # {{{ data = self.get_node_tree(config['sort_by_popularity']) self.root_item = TagTreeItem() for i, r in enumerate(self.row_map): - if self.db.get_tag_browser_categories()[r]['kind'] != 'user': + if self.db.field_metadata[r]['kind'] != 'user': tt = _('The lookup/search name is "{0}"').format(r) else: tt = '' @@ -248,7 +248,7 @@ class TagsModel(QAbstractItemModel): # {{{ else: data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map) - tb_categories = self.db.get_tag_browser_categories() + tb_categories = self.db.field_metadata for category in tb_categories: if category in data: # They should always be there, but ... self.row_map.append(category) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 20cfce02bd..47529f223f 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -150,14 +150,14 @@ class ResultCache(SearchQueryParser): ''' Stores sorted and filtered metadata in memory. ''' - def __init__(self, FIELD_MAP, cc_label_map, tag_browser_categories): + def __init__(self, FIELD_MAP, cc_label_map, field_metadata): self.FIELD_MAP = FIELD_MAP self.custom_column_label_map = cc_label_map self._map = self._map_filtered = self._data = [] self.first_sort = True self.search_restriction = '' - self.tag_browser_categories = tag_browser_categories - self.all_search_locations = tag_browser_categories.get_search_keys() + self.field_metadata = field_metadata + self.all_search_locations = field_metadata.get_search_terms() SearchQueryParser.__init__(self, self.all_search_locations) self.build_date_relop_dict() self.build_numeric_relop_dict() @@ -252,7 +252,7 @@ class ResultCache(SearchQueryParser): if location == 'date': location = 'timestamp' - loc = self.tag_browser_categories[location]['rec_index'] + loc = self.field_metadata[location]['rec_index'] if query == _('today'): qd = now() @@ -311,8 +311,8 @@ class ResultCache(SearchQueryParser): if relop is None: (p, relop) = self.numeric_search_relops['='] - loc = self.tag_browser_categories[location]['rec_index'] - dt = self.tag_browser_categories[location]['datatype'] + loc = self.field_metadata[location]['rec_index'] + dt = self.field_metadata[location]['datatype'] if dt == 'int': cast = (lambda x: int (x)) adjust = lambda x: x @@ -342,23 +342,21 @@ class ResultCache(SearchQueryParser): def get_matches(self, location, query): matches = set([]) if query and query.strip(): - location = location.lower().strip() - if location in ('tag', 'author', 'format', 'comment'): - location += 's' + # get metadata key associated with the search term. Eliminates + # dealing with plurals and other aliases + location = self.field_metadata.search_term_to_key(location.lower().strip()) - ### take care of dates special case - if (location in ('pubdate', 'date')) or \ - (location in self.tag_browser_categories and \ - self.tag_browser_categories[location]['datatype'] == 'datetime'): + # take care of dates special case + if location in self.field_metadata and \ + self.field_metadata[location]['datatype'] == 'datetime': return self.get_dates_matches(location, query.lower()) - ### take care of numbers special case - if location in self.tag_browser_categories and \ - self.tag_browser_categories[location]['datatype'] in \ - ('rating', 'int', 'float'): + # take care of numbers special case + if location in self.field_metadata and \ + self.field_metadata[location]['datatype'] in ('rating', 'int', 'float'): return self.get_numeric_matches(location, query.lower()) - ### everything else, or 'all' matches + # everything else, or 'all' matches matchkind = CONTAINS_MATCH if (len(query) > 1): if query.startswith('\\'): @@ -377,25 +375,18 @@ class ResultCache(SearchQueryParser): query = query.decode('utf-8') db_col = {} - # fields to not check when matching against text. - exclude_fields = [] + exclude_fields = [] # fields to not check when matching against text. col_datatype = [] + is_multiple_cols = {} for x in range(len(self.FIELD_MAP)): col_datatype.append('') - for x in self.tag_browser_categories: - if len(self.tag_browser_categories[x]['search_keys']): - db_col[x] = self.tag_browser_categories[x]['rec_index'] - if self.tag_browser_categories[x]['datatype'] not in ['text', 'comments']: + for x in self.field_metadata: + if len(self.field_metadata[x]['search_terms']): + db_col[x] = self.field_metadata[x]['rec_index'] + if self.field_metadata[x]['datatype'] not in ['text', 'comments']: exclude_fields.append(db_col[x]) - if self.tag_browser_categories.is_custom_field(x): - col_datatype[db_col[x]] = self.tag_browser_categories[x]['datatype'] - # normal and custom ratings columns use the same code - col_datatype[self.FIELD_MAP['rating']] = 'rating' - - splitable_fields = [db_col['authors'], db_col['tags'], db_col['formats']] - for x in self.tag_browser_categories.get_custom_fields(): - if self.tag_browser_categories[x]['is_multiple']: - splitable_fields.append(db_col[x]) + col_datatype[db_col[x]] = self.field_metadata[x]['datatype'] + is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple'] try: rating_query = int(query) * 2 @@ -409,7 +400,7 @@ class ResultCache(SearchQueryParser): # get the tweak here so that the string lookup and compare aren't in the loop bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes' - for loc in location: + for loc in location: # location is now an array of field indices if loc == db_col['authors']: ### DB stores authors with commas changed to bars, so change query q = query.replace(',', '|'); @@ -460,7 +451,7 @@ class ResultCache(SearchQueryParser): continue try: # a conversion below might fail - # relationals not supported in 'all' queries + # relationals are not supported in 'all' queries if col_datatype[loc] == 'float': if float(query) == item[loc]: matches.add(item[0]) @@ -474,12 +465,9 @@ class ResultCache(SearchQueryParser): # no further match is possible continue - if loc not in exclude_fields: - if loc in splitable_fields: - if col_datatype[loc]: - vals = item[loc].split('|') - else: - vals = item[loc].split(',') + if loc not in exclude_fields: # time for text matching + if is_multiple_cols[loc] is not None: + vals = item[loc].split(is_multiple_cols[loc]) else: vals = [item[loc]] ### make into list to make _match happy if _match(q, vals, matchkind): diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 5b5fcadf59..7a63f37588 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -147,11 +147,15 @@ class CustomColumns(object): is_category = True else: is_category = False + if v['is_multiple']: + is_m = '|' + else: + is_m = None tn = 'custom_column_{0}'.format(v['num']) - self.tag_browser_categories.add_custom_field(label=v['label'], + self.field_metadata.add_custom_field(label=v['label'], table=tn, column='value', datatype=v['datatype'], - is_multiple=v['is_multiple'], colnum=v['num'], - name=v['name'], is_category=is_category) + is_multiple=is_m, colnum=v['num'], name=v['name'], + is_category=is_category) def get_custom(self, idx, label=None, num=None, index_is_id=False): if label is not None: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 939874ed3b..5d4c2a783e 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -116,7 +116,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter') def __init__(self, library_path, row_factory=False): - self.tag_browser_categories = FieldMetadata() #.get_tag_browser_categories() + self.field_metadata = FieldMetadata() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -206,20 +206,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19} for k,v in self.FIELD_MAP.iteritems(): - self.tag_browser_categories.set_field_record_index(k, v, prefer_custom=False) + self.field_metadata.set_field_record_index(k, v, prefer_custom=False) base = max(self.FIELD_MAP.values()) for col in custom_cols: self.FIELD_MAP[col] = base = base+1 - self.tag_browser_categories.set_field_record_index( + self.field_metadata.set_field_record_index( self.custom_column_num_map[col]['label'], base, prefer_custom=True) self.FIELD_MAP['cover'] = base+1 - self.tag_browser_categories.set_field_record_index('cover', base+1, prefer_custom=False) + self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False) self.FIELD_MAP['ondevice'] = base+2 - self.tag_browser_categories.set_field_record_index('ondevice', base+2, prefer_custom=False) + self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -233,7 +233,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.book_on_device_func = None self.data = ResultCache(self.FIELD_MAP, self.custom_column_label_map, - self.tag_browser_categories) + self.field_metadata) self.search = self.data.search self.refresh = functools.partial(self.data.refresh, self) self.sort = self.data.sort @@ -646,9 +646,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_recipe(self, id): return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) - def get_tag_browser_categories(self): - return self.tag_browser_categories - def get_categories(self, sort_on_count=False, ids=None, icon_map=None): self.books_list_filter.change([] if not ids else ids) @@ -656,9 +653,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') - tb_cats = self.tag_browser_categories + tb_cats = self.field_metadata - # remove all user categories from tag_browser_categories. They can + # remove all user categories from field_metadata. They can # easily come and go. We will add all the existing ones in below. for k in tb_cats.keys(): if tb_cats[k]['kind'] in ['user', 'search']: diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index f8f3695473..638b7c7dd0 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -22,104 +22,252 @@ class TagsIcons(dict): self[a] = icon_dict[a] class FieldMetadata(dict): + ''' + key: the key to the dictionary is: + - for standard fields, the metadata field name. + - for custom fields, the metadata field name prefixed by '#' + This is done to create two 'namespaces' so the names don't clash - # kind == standard: is tag category. May be a search label. Is db col - # or is specially handled (e.g., news) - # kind == not_cat: Is not a tag category. May be a search label. Is db col - # kind == user: user-defined tag category - # kind == search: saved-searches category - # For 'standard', the order below is the order that the categories will - # appear in the tags pane. - # - # label is the column label. key is either the label or in the case of - # custom fields, the label prefixed with 'x'. Because of the prefixing, - # there cannot be a name clash between standard and custom fields, so key - # can be used as the metadata dictionary key. + label: the actual column label. No prefixing. - category_items_ = [ - ('authors', {'table':'authors', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Authors'), - 'search_keys':['authors', 'author'], - 'is_custom':False, 'is_category':True}), - ('series', {'table':'series', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Series'), - 'search_keys':['series'], - 'is_custom':False, 'is_category':True}), - ('formats', {'table':None, 'column':None, - 'datatype':'text', 'is_multiple':False, # must think what type this is! - 'kind':'standard', 'name':_('Formats'), - 'search_keys':['formats', 'format'], - 'is_custom':False, 'is_category':True}), - ('publisher', {'table':'publishers', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Publishers'), - 'search_keys':['publisher'], - 'is_custom':False, 'is_category':True}), - ('rating', {'table':'ratings', 'column':'rating', - 'datatype':'rating', 'is_multiple':False, - 'kind':'standard', 'name':_('Ratings'), - 'search_keys':['rating'], - 'is_custom':False, 'is_category':True}), - ('news', {'table':'news', 'column':'name', - 'datatype':None, 'is_multiple':False, - 'kind':'standard', 'name':_('News'), - 'search_keys':[], - 'is_custom':False, 'is_category':True}), - ('tags', {'table':'tags', 'column':'name', - 'datatype':'text', 'is_multiple':True, - 'kind':'standard', 'name':_('Tags'), - 'search_keys':['tags', 'tag'], - 'is_custom':False, 'is_category':True}), - ('author_sort',{'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('comments', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['comments', 'comment'], 'is_custom':False, 'is_category':False}), - ('cover', {'table':None, 'column':None, 'datatype':None, - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['cover'], 'is_custom':False, 'is_category':False}), - ('flags', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('id', {'table':None, 'column':None, 'datatype':'int', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('isbn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['isbn'], 'is_custom':False, 'is_category':False}), - ('lccn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('ondevice', {'table':None, 'column':None, 'datatype':'bool', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('path', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('pubdate', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['pubdate'], 'is_custom':False, 'is_category':False}), - ('series_index',{'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('sort', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('size', {'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), - ('timestamp', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['date'], 'is_custom':False, 'is_category':False}), - ('title', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':['title'], 'is_custom':False, 'is_category':False}), - ('uuid', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'standard', 'name':None, - 'search_keys':[], 'is_custom':False, 'is_category':False}), + datatype: the type of the information in the field. Valid values are float, + int, rating, bool, comments, datetime, text. + is_multiple: valid for the text datatype. If None, the field is to be + treated as a single term. If not None, it contains a string, and the field + is assumed to contain a list of terms separated by that string + + kind == standard: is a db field. + kind == category: standard tag category that isn't a field. see news. + kind == user: user-defined tag category. + kind == search: saved-searches category. + + is_category: is a tag browser category. If true, then: + table: name of the db table used to construct item list + column: name of the column in the connection table to join on + If these are None, then the category constructor must know how + to build the item list (e.g., formats). + The order below is the order that the categories will + appear in the tags pane. + + name: the text that is to be used when displaying the field. Column headings + in the GUI, etc. + + search_terms: the terms that can be used to identify the field when + searching. They can be thought of as aliases for metadata keys, but are only + valid when passed to search(). + + is_custom: the field has been added by the user. + + rec_index: the index of the field in the db metadata record. + + ''' + _field_metadata = [ + ('authors', {'table':'authors', + 'column':'name', + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Authors'), + 'search_terms':['authors', 'author'], + 'is_custom':False, + 'is_category':True}), + ('series', {'table':'series', + 'column':'name', + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':_('Series'), + 'search_terms':['series'], + 'is_custom':False, + 'is_category':True}), + ('formats', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Formats'), + 'search_terms':['formats', 'format'], + 'is_custom':False, + 'is_category':True}), + ('publisher', {'table':'publishers', + 'column':'name', + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':_('Publishers'), + 'search_terms':['publisher'], + 'is_custom':False, + 'is_category':True}), + ('rating', {'table':'ratings', + 'column':'rating', + 'datatype':'rating', + 'is_multiple':None, + 'kind':'field', + 'name':_('Ratings'), + 'search_terms':['rating'], + 'is_custom':False, + 'is_category':True}), + ('news', {'table':'news', + 'column':'name', + 'datatype':None, + 'is_multiple':None, + 'kind':'category', + 'name':_('News'), + 'search_terms':[], + 'is_custom':False, + 'is_category':True}), + ('tags', {'table':'tags', + 'column':'name', + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Tags'), + 'search_terms':['tags', 'tag'], + 'is_custom':False, + 'is_category':True}), + ('author_sort',{'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('comments', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['comments', 'comment'], + 'is_custom':False, 'is_category':False}), + ('cover', {'table':None, + 'column':None, + 'datatype':None, + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['cover'], + 'is_custom':False, + 'is_category':False}), + ('flags', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('id', {'table':None, + 'column':None, + 'datatype':'int', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('isbn', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['isbn'], + 'is_custom':False, + 'is_category':False}), + ('lccn', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('ondevice', {'table':None, + 'column':None, + 'datatype':'bool', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('path', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('pubdate', {'table':None, + 'column':None, + 'datatype':'datetime', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['pubdate'], + 'is_custom':False, + 'is_category':False}), + ('series_index',{'table':None, + 'column':None, + 'datatype':'float', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('sort', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('size', {'table':None, + 'column':None, + 'datatype':'float', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('timestamp', {'table':None, + 'column':None, + 'datatype':'datetime', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['date'], + 'is_custom':False, + 'is_category':False}), + ('title', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['title'], + 'is_custom':False, + 'is_category':False}), + ('uuid', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), ] # search labels that are not db columns @@ -130,10 +278,13 @@ class FieldMetadata(dict): def __init__(self): self._tb_cats = OrderedDict() - for k,v in self.category_items_: + self._search_term_map = {} + self.custom_label_to_key_map = {} + for k,v in self._field_metadata: self._tb_cats[k] = v + self._tb_cats[k]['label'] = k # saved some typing above... + self._add_search_terms_to_map(k, self._tb_cats[k]['search_terms']) self.custom_field_prefix = '#' - self.get = self._tb_cats.get def __getitem__(self, key): @@ -175,10 +326,14 @@ class FieldMetadata(dict): return self._tb_cats[key]['label'] def label_to_key(self, label, prefer_custom=False): + if prefer_custom: + if label in self.custom_label_to_key_map: + return self.custom_label_to_key_map[label] if 'label' in self._tb_cats: return label - if self.is_custom_field(label): - return self.custom_field_prefix+label + if not prefer_custom: + if label in self.custom_label_to_key_map: + return self.custom_label_to_key_map[label] raise ValueError('Unknown key [%s]'%(label)) def get_custom_fields(self): @@ -186,15 +341,35 @@ class FieldMetadata(dict): def add_custom_field(self, label, table, column, datatype, colnum, name, is_multiple, is_category): - fn = self.custom_field_prefix + label - if fn in self._tb_cats: + key = self.custom_field_prefix + label + if key in self._tb_cats: raise ValueError('Duplicate custom field [%s]'%(label)) - self._tb_cats[fn] = {'table':table, 'column':column, - 'datatype':datatype, 'is_multiple':is_multiple, - 'kind':'standard', 'name':name, - 'search_keys':[fn], 'label':label, - 'colnum':colnum, 'is_custom':True, + self._tb_cats[key] = {'table':table, 'column':column, + 'datatype':datatype, 'is_multiple':is_multiple, + 'kind':'field', 'name':name, + 'search_terms':[key], 'label':label, + 'colnum':colnum, 'is_custom':True, 'is_category':is_category} + self._add_search_terms_to_map(key, [key]) + self.custom_label_to_key_map[label] = key + + def add_user_category(self, label, name): + if label in self._tb_cats: + raise ValueError('Duplicate user field [%s]'%(label)) + self._tb_cats[label] = {'table':None, 'column':None, + 'datatype':None, 'is_multiple':None, + 'kind':'user', 'name':name, + 'search_terms':[], 'is_custom':False, + 'is_category':True} + + def add_search_category(self, label, name): + if label in self._tb_cats: + raise ValueError('Duplicate user field [%s]'%(label)) + self._tb_cats[label] = {'table':None, 'column':None, + 'datatype':None, 'is_multiple':None, + 'kind':'search', 'name':name, + 'search_terms':[], 'is_custom':False, + 'is_category':True} def set_field_record_index(self, label, index, prefer_custom=False): if prefer_custom: @@ -208,23 +383,6 @@ class FieldMetadata(dict): key = self.custom_field_prefix+label self._tb_cats[key]['rec_index'] = index # let the exception fly ... - def add_user_category(self, label, name): - if label in self._tb_cats: - raise ValueError('Duplicate user field [%s]'%(label)) - self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':False, - 'kind':'user', 'name':name, - 'search_keys':[], 'is_custom':False, - 'is_category':True} - - def add_search_category(self, label, name): - if label in self._tb_cats: - raise ValueError('Duplicate user field [%s]'%(label)) - self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':False, - 'kind':'search', 'name':name, - 'search_keys':[], 'is_custom':False, - 'is_category':True} # DEFAULT_LOCATIONS = frozenset([ # 'all', @@ -248,14 +406,23 @@ class FieldMetadata(dict): # 'title', # ]) - - def get_search_keys(self): + def get_search_terms(self): s_keys = [] for v in self._tb_cats.itervalues(): - map((lambda x:s_keys.append(x)), v['search_keys']) + map((lambda x:s_keys.append(x)), v['search_terms']) for v in self.search_items: s_keys.append(v) # if set(s_keys) != self.DEFAULT_LOCATIONS: # print 'search labels and default_locations do not match:' # print set(s_keys) ^ self.DEFAULT_LOCATIONS return s_keys + + def _add_search_terms_to_map(self, key, terms): + if terms is not None: + for t in terms: + self._search_term_map[t] = key + + def search_term_to_key(self, term): + if term in self._search_term_map: + return self._search_term_map[term] + return term diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index f1e68b3916..8d4fd5eba3 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -289,6 +289,6 @@ class SchemaUpgrade(object): '''.format(tn=table_name, cn=column_name, vcn=view_column_name)) self.conn.executescript(script) - for tn, cn in self.tag_browser_categories.items(): + for tn, cn in self.field_metadata.items(): if tn != 'news': create_tag_browser_view(tn, cn[0], cn[1]) diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index 86fae8bae8..7c727cbf52 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -331,7 +331,7 @@ class OPDSServer(object): raise cherrypy.HTTPError(404, 'Not found') categories = self.categories_cache( self.get_opds_allowed_ids_for_version(version)) - category_meta = self.db.get_tag_browser_categories() + category_meta = self.db.field_metadata cats = [ (_('Newest'), _('Date'), 'Onewest'), (_('Title'), _('Title'), 'Otitle'), From a95d5ae6d567a86c43abb5272c19e8daba9a5ea5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 10:45:07 -0600 Subject: [PATCH 04/18] Improve darknet --- resources/recipes/darknet.recipe | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/resources/recipes/darknet.recipe b/resources/recipes/darknet.recipe index 4410e0567f..c256504cf6 100644 --- a/resources/recipes/darknet.recipe +++ b/resources/recipes/darknet.recipe @@ -23,7 +23,8 @@ class darknet(BasicNewsRecipe): remove_tags = [dict(id='navi_top'), dict(id='navi_bottom'), - dict(id='logo'), + dict(id='nav'), + dict(id='top-ad'), dict(id='login_suche'), dict(id='navi_login'), dict(id='breadcrumb'), @@ -32,13 +33,14 @@ class darknet(BasicNewsRecipe): dict(name='span', attrs={'class':'rsaquo'}), dict(name='span', attrs={'class':'next'}), dict(name='span', attrs={'class':'prev'}), + dict(name='span', attrs={'class':'comments'}), dict(name='div', attrs={'class':'news_logo'}), dict(name='div', attrs={'class':'nextprev'}), + dict(name='div', attrs={'class':'tags'}), + dict(name='div', attrs={'class':'Nav'}), dict(name='p', attrs={'class':'news_option'}), dict(name='p', attrs={'class':'news_foren'})] - remove_tags_after = [dict(name='div', attrs={'class':'entrybody'})] + remove_tags_after = [dict(name='div', attrs={'class':'meta-footer'})] feeds = [ ('darknet', 'http://feedproxy.google.com/darknethackers') ] - - From 68a7cc5c8022d0589d51854cb8049aaff4dec9b4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 18:00:24 +0100 Subject: [PATCH 05/18] Use field_metadata in more places. Fixed problems with duplicate column names in preferences, viewing the library, and sorting --- src/calibre/gui2/dialogs/config/__init__.py | 10 ++--- .../dialogs/config/create_custom_column.py | 18 +++++--- .../dialogs/config/create_custom_column.ui | 2 +- src/calibre/gui2/library/delegates.py | 15 +++---- src/calibre/gui2/library/models.py | 45 ++++++++++--------- src/calibre/library/caches.py | 9 ++-- src/calibre/library/custom_columns.py | 5 ++- src/calibre/library/database2.py | 3 +- src/calibre/library/field_metadata.py | 26 +++++++---- 9 files changed, 72 insertions(+), 61 deletions(-) diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index 9d108d3807..4d0dd07746 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -371,7 +371,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): hidden_cols = state['hidden_columns'] positions = state['column_positions'] colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) - self.custcols = copy.deepcopy(self.db.custom_column_label_map) + self.custcols = copy.deepcopy(self.db.field_metadata.get_custom_field_metadata()) for col in colmap: item = QListWidgetItem(self.model.headers[col], self.columns) item.setData(Qt.UserRole, QVariant(col)) @@ -713,20 +713,20 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): must_restart = False for c in self.custcols: - if self.custcols[c]['num'] is None: + if self.custcols[c]['colnum'] is None: self.db.create_custom_column( - label=c, + label=self.custcols[c]['label'], name=self.custcols[c]['name'], datatype=self.custcols[c]['datatype'], is_multiple=self.custcols[c]['is_multiple'], display = self.custcols[c]['display']) must_restart = True elif '*deleteme' in self.custcols[c]: - self.db.delete_custom_column(label=c) + self.db.delete_custom_column(label=self.custcols[c]['label']) must_restart = True elif '*edited' in self.custcols[c]: cc = self.custcols[c] - self.db.set_custom_column_metadata(cc['num'], name=cc['name'], + self.db.set_custom_column_metadata(cc['colnum'], name=cc['name'], label=cc['label'], display = self.custcols[c]['display']) if '*must_restart' in self.custcols[c]: diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.py b/src/calibre/gui2/dialogs/config/create_custom_column.py index 9e040315c9..693f079d12 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.py +++ b/src/calibre/gui2/dialogs/config/create_custom_column.py @@ -69,13 +69,14 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): self.column_name_box.setText(c['label']) self.column_heading_box.setText(c['name']) ct = c['datatype'] if not c['is_multiple'] else '*text' - self.orig_column_number = c['num'] + self.orig_column_number = c['colnum'] self.orig_column_name = col column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), self.column_types)) self.column_type_box.setCurrentIndex(column_numbers[ct]) self.column_type_box.setEnabled(False) if ct == 'datetime': - self.date_format_box.setText(c['display'].get('date_format', '')) + if c['display'].get('date_format', None): + self.date_format_box.setText(c['display'].get('date_format', '')) self.datatype_changed() self.exec_() @@ -90,7 +91,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): def accept(self): - col = unicode(self.column_name_box.text()) + col = unicode(self.column_name_box.text()).lower() + if not col.isalnum(): + return self.simple_error('', _('The label must contain only letters and digits')) col_heading = unicode(self.column_heading_box.text()) col_type = self.column_types[self.column_type_box.currentIndex()]['datatype'] if col_type == '*text': @@ -104,14 +107,14 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): return self.simple_error('', _('No column heading was provided')) bad_col = False if col in self.parent.custcols: - if not self.editing_col or self.parent.custcols[col]['num'] != self.orig_column_number: + if not self.editing_col or self.parent.custcols[col]['colnum'] != self.orig_column_number: bad_col = True if bad_col: return self.simple_error('', _('The lookup name %s is already used')%col) bad_head = False for t in self.parent.custcols: if self.parent.custcols[t]['name'] == col_heading: - if not self.editing_col or self.parent.custcols[t]['num'] != self.orig_column_number: + if not self.editing_col or self.parent.custcols[t]['colnum'] != self.orig_column_number: bad_head = True for t in self.standard_colheads: if self.standard_colheads[t] == col_heading: @@ -129,14 +132,15 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): date_format = {'date_format': None} if not self.editing_col: - self.parent.custcols[col] = { + self.parent.db.field_metadata + self.parent.custcols[self.parent.db.field_metadata.custom_field_prefix+col] = { 'label':col, 'name':col_heading, 'datatype':col_type, 'editable':True, 'display':date_format, 'normalized':None, - 'num':None, + 'colnum':None, 'is_multiple':is_multiple, } item = QListWidgetItem(col_heading, self.parent.columns) diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.ui b/src/calibre/gui2/dialogs/config/create_custom_column.ui index 279349f28e..5cb9494845 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.ui +++ b/src/calibre/gui2/dialogs/config/create_custom_column.ui @@ -65,7 +65,7 @@ - Used for searching the column. Must be lower case and not contain spaces or colons. + Used for searching the column. Must contain only digits and lower case letters. diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index d908ed01b4..529055ecd2 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -171,7 +171,8 @@ class TagsDelegate(QStyledItemDelegate): # {{{ if not index.model().is_custom_column(col): editor = TagsLineEdit(parent, self.db.all_tags()) else: - editor = TagsLineEdit(parent, sorted(list(self.db.all_custom(label=col)))) + editor = TagsLineEdit(parent, + sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col))))) return editor else: editor = EnLineEdit(parent) @@ -209,7 +210,7 @@ class CcDateDelegate(QStyledItemDelegate): # {{{ m = index.model() # db col is not named for the field, but for the table number. To get it, # gui column -> column label -> table number -> db column - val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]] + val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']] if val is None: val = now() editor.setDate(val) @@ -243,7 +244,7 @@ class CcTextDelegate(QStyledItemDelegate): # {{{ editor.setDecimals(2) else: editor = EnLineEdit(parent) - complete_items = sorted(list(m.db.all_custom(label=col))) + complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col)))) completer = QCompleter(complete_items, self) completer.setCaseSensitivity(Qt.CaseInsensitive) completer.setCompletionMode(QCompleter.PopupCompletion) @@ -260,9 +261,7 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{ def createEditor(self, parent, option, index): m = index.model() col = m.column_map[index.column()] - # db col is not named for the field, but for the table number. To get it, - # gui column -> column label -> table number -> db column - text = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[col]['num']]] + text = m.db.data[index.row()][m.custom_columns[col]['rec_index']] editor = CommentsDialog(parent, text) d = editor.exec_() if d: @@ -297,9 +296,7 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{ def setEditorData(self, editor, index): m = index.model() - # db col is not named for the field, but for the table number. To get it, - # gui column -> column label -> table number -> db column - val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]] + val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']] if tweaks['bool_custom_columns_are_tristate'] == 'no': val = 1 if not val else 0 else: diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 072f81e2d1..5490e96169 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -111,15 +111,15 @@ class BooksModel(QAbstractTableModel): # {{{ def set_database(self, db): self.db = db - self.custom_columns = self.db.custom_column_label_map + self.custom_columns = self.db.field_metadata.get_custom_field_metadata() self.column_map = list(self.orig_headers.keys()) + \ list(self.custom_columns) def col_idx(name): if name == 'ondevice': return -1 - if name not in self.db.FIELD_MAP: + if name not in self.db.field_metadata: return 100000 - return self.db.FIELD_MAP[name] + return self.db.field_metadata[name]['rec_index'] self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) for col in self.column_map: @@ -232,11 +232,12 @@ class BooksModel(QAbstractTableModel): # {{{ return self.about_to_be_sorted.emit(self.db.id) ascending = order == Qt.AscendingOrder - self.db.sort(self.column_map[col], ascending) + label = self.column_map[col] + self.db.sort(label, ascending) if reset: self.clear_caches() self.reset() - self.sorted_on = (self.column_map[col], order) + self.sorted_on = (label, order) self.sort_history.insert(0, self.sorted_on) self.sorting_done.emit(self.db.index) @@ -551,36 +552,36 @@ class BooksModel(QAbstractTableModel): # {{{ self.dc = { 'title' : functools.partial(text_type, - idx=self.db.FIELD_MAP['title'], mult=False), + idx=self.db.field_metadata['title']['rec_index'], mult=False), 'authors' : functools.partial(authors, - idx=self.db.FIELD_MAP['authors']), + idx=self.db.field_metadata['authors']['rec_index']), 'size' : functools.partial(size, - idx=self.db.FIELD_MAP['size']), + idx=self.db.field_metadata['size']['rec_index']), 'timestamp': functools.partial(datetime_type, - idx=self.db.FIELD_MAP['timestamp']), + idx=self.db.field_metadata['timestamp']['rec_index']), 'pubdate' : functools.partial(datetime_type, - idx=self.db.FIELD_MAP['pubdate']), + idx=self.db.field_metadata['pubdate']['rec_index']), 'rating' : functools.partial(rating_type, - idx=self.db.FIELD_MAP['rating']), + idx=self.db.field_metadata['rating']['rec_index']), 'publisher': functools.partial(text_type, - idx=self.db.FIELD_MAP['publisher'], mult=False), + idx=self.db.field_metadata['publisher']['rec_index'], mult=False), 'tags' : functools.partial(tags, - idx=self.db.FIELD_MAP['tags']), + idx=self.db.field_metadata['tags']['rec_index']), 'series' : functools.partial(series, - idx=self.db.FIELD_MAP['series'], - siix=self.db.FIELD_MAP['series_index']), + idx=self.db.field_metadata['series']['rec_index'], + siix=self.db.field_metadata['series_index']['rec_index']), 'ondevice' : functools.partial(text_type, - idx=self.db.FIELD_MAP['ondevice'], mult=False), + idx=self.db.field_metadata['ondevice']['rec_index'], mult=False), } self.dc_decorator = { 'ondevice':functools.partial(ondevice_decorator, - idx=self.db.FIELD_MAP['ondevice']), + idx=self.db.field_metadata['ondevice']['rec_index']), } # Add the custom columns to the data converters for col in self.custom_columns: - idx = self.db.FIELD_MAP[self.custom_columns[col]['num']] + idx = self.custom_columns[col]['rec_index'] datatype = self.custom_columns[col]['datatype'] if datatype in ('text', 'comments'): self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple']) @@ -632,8 +633,6 @@ class BooksModel(QAbstractTableModel): # {{{ return None if role == Qt.ToolTipRole: ht = self.column_map[section] - if self.is_custom_column(self.column_map[section]): - ht = self.db.field_metadata.custom_field_prefix + ht if ht == 'timestamp': # change help text because users know this field as 'date' ht = 'date' return QVariant(_('The lookup/search name is "{0}"').format(ht)) @@ -652,7 +651,7 @@ class BooksModel(QAbstractTableModel): # {{{ if colhead in self.editable_cols: flags |= Qt.ItemIsEditable elif self.is_custom_column(colhead): - if self.custom_columns[colhead]['editable']: + if self.custom_columns[colhead]['is_editable']: flags |= Qt.ItemIsEditable return flags @@ -679,7 +678,9 @@ class BooksModel(QAbstractTableModel): # {{{ if not val.isValid(): return False val = qt_to_dt(val, as_utc=False) - self.db.set_custom(self.db.id(row), val, label=colhead, num=None, append=False, notify=True) + self.db.set_custom(self.db.id(row), val, + label=self.db.field_metadata.key_to_label(colhead), + num=None, append=False, notify=True) return True def setData(self, index, value, role): diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 47529f223f..93891ee92b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -150,9 +150,8 @@ class ResultCache(SearchQueryParser): ''' Stores sorted and filtered metadata in memory. ''' - def __init__(self, FIELD_MAP, cc_label_map, field_metadata): + def __init__(self, FIELD_MAP, field_metadata): self.FIELD_MAP = FIELD_MAP - self.custom_column_label_map = cc_label_map self._map = self._map_filtered = self._data = [] self.first_sort = True self.search_restriction = '' @@ -598,9 +597,9 @@ class ResultCache(SearchQueryParser): elif field == 'title': field = 'sort' elif field == 'authors': field = 'author_sort' as_string = field not in ('size', 'rating', 'timestamp') - if field in self.custom_column_label_map: - as_string = self.custom_column_label_map[field]['datatype'] in ('comments', 'text') - field = self.custom_column_label_map[field]['num'] + if self.field_metadata[field]['is_custom']: + as_string = self.field_metadata[field]['datatype'] in ('comments', 'text') + field = self.field_metadata[field]['colnum'] if self.first_sort: subsort = True diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 7a63f37588..86b7d715ea 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -154,8 +154,9 @@ class CustomColumns(object): tn = 'custom_column_{0}'.format(v['num']) self.field_metadata.add_custom_field(label=v['label'], table=tn, column='value', datatype=v['datatype'], - is_multiple=is_m, colnum=v['num'], name=v['name'], - is_category=is_category) + colnum=v['num'], name=v['name'], display=v['display'], + is_multiple=is_m, is_category=is_category, + is_editable=v['editable']) def get_custom(self, idx, label=None, num=None, index_is_id=False): if label is not None: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 5d4c2a783e..5ba603cc52 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -232,8 +232,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() self.book_on_device_func = None - self.data = ResultCache(self.FIELD_MAP, self.custom_column_label_map, - self.field_metadata) + self.data = ResultCache(self.FIELD_MAP, self.field_metadata) self.search = self.data.search self.refresh = functools.partial(self.data.refresh, self) self.sort = self.data.sort diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 638b7c7dd0..6f1219ad03 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -282,7 +282,9 @@ class FieldMetadata(dict): self.custom_label_to_key_map = {} for k,v in self._field_metadata: self._tb_cats[k] = v - self._tb_cats[k]['label'] = k # saved some typing above... + self._tb_cats[k]['label'] = k + self._tb_cats[k]['display'] = {} + self._tb_cats[k]['is_editable'] = True self._add_search_terms_to_map(k, self._tb_cats[k]['search_terms']) self.custom_field_prefix = '#' self.get = self._tb_cats.get @@ -300,12 +302,12 @@ class FieldMetadata(dict): for key in self._tb_cats: yield key - def has_key(self, key): - return key in self._tb_cats - def __contains__(self, key): return self.has_key(key) + def has_key(self, key): + return key in self._tb_cats + def keys(self): return self._tb_cats.keys() @@ -339,8 +341,15 @@ class FieldMetadata(dict): def get_custom_fields(self): return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']] - def add_custom_field(self, label, table, column, datatype, colnum, - name, is_multiple, is_category): + def get_custom_field_metadata(self): + l = {} + for k in self._tb_cats: + if self._tb_cats[k]['is_custom']: + l[k] = self._tb_cats[k] + return l + + def add_custom_field(self, label, table, column, datatype, colnum, name, + display, is_editable, is_multiple, is_category): key = self.custom_field_prefix + label if key in self._tb_cats: raise ValueError('Duplicate custom field [%s]'%(label)) @@ -348,8 +357,9 @@ class FieldMetadata(dict): 'datatype':datatype, 'is_multiple':is_multiple, 'kind':'field', 'name':name, 'search_terms':[key], 'label':label, - 'colnum':colnum, 'is_custom':True, - 'is_category':is_category} + 'colnum':colnum, 'display':display, + 'is_custom':True, 'is_category':is_category, + 'is_editable': is_editable,} self._add_search_terms_to_map(key, [key]) self.custom_label_to_key_map[label] = key From f8a91050017c7f392263bf3751f85c1d48b7f050 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 11:15:18 -0600 Subject: [PATCH 06/18] Fix editing of custom is_multiple column when a value already exists --- src/calibre/library/custom_columns.py | 2 +- src/calibre/library/server/opds.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 535b8cfb72..90af46cc42 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -257,7 +257,7 @@ class CustomColumns(object): 'SELECT id FROM %s WHERE value=?'%table, (ex,), all=False) if ex != x: self.conn.execute( - 'UPDATE %s SET value=? WHERE id=?', (x, xid)) + 'UPDATE %s SET value=? WHERE id=?'%table, (x, xid)) else: xid = self.conn.execute( 'INSERT INTO %s(value) VALUES(?)'%table, (x,)).lastrowid diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index 86fae8bae8..dd1e2d0bb0 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -322,9 +322,17 @@ class OPDSServer(object): return self.get_opds_all_books(which, page_url, up_url, version=version, offset=offset) elif type_ == 'N': - return self.get_opds_navcatalog(which) + return self.get_opds_navcatalog(which, version=version, offset=offset) raise cherrypy.HTTPError(404, 'Not found') + def get_opds_navcatalog(self, which, version=0, offset=0): + categories = self.categories_cache( + self.get_opds_allowed_ids_for_version(version)) + if which not in categories: + raise cherrypy.HTTPError(404, 'Category %r not found'%which) + + + def opds(self, version=0): version = int(version) if version not in BASE_HREFS: From cb27515459d016a33498bb6b3393f2f24f314dfd Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 19:45:07 +0100 Subject: [PATCH 07/18] Add check that custom field begins with a letter --- src/calibre/gui2/dialogs/config/create_custom_column.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.py b/src/calibre/gui2/dialogs/config/create_custom_column.py index 693f079d12..5ff048ed0a 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.py +++ b/src/calibre/gui2/dialogs/config/create_custom_column.py @@ -92,8 +92,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): def accept(self): col = unicode(self.column_name_box.text()).lower() - if not col.isalnum(): - return self.simple_error('', _('The label must contain only letters and digits')) + if not col: + return self.simple_error('', _('No lookup name was provided')) + if not col.isalnum() or not col[0].isalpha(): + return self.simple_error('', _('The label must contain only letters and digits, and start with a letter')) col_heading = unicode(self.column_heading_box.text()) col_type = self.column_types[self.column_type_box.currentIndex()]['datatype'] if col_type == '*text': @@ -101,8 +103,6 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): is_multiple = True else: is_multiple = False - if not col: - return self.simple_error('', _('No lookup name was provided')) if not col_heading: return self.simple_error('', _('No column heading was provided')) bad_col = False From 04e8a48916603a513b3dcd7c8af0e0b81a380904 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 21:53:36 +0100 Subject: [PATCH 08/18] Changes to schema_upgrades.py and changing get_categories to not modify metadata. --- src/calibre/gui2/tag_view.py | 13 +++++++++++++ src/calibre/library/database2.py | 23 +++++++++++++---------- src/calibre/library/field_metadata.py | 4 ++++ src/calibre/library/schema_upgrades.py | 8 ++------ 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 8ecc26e30c..15270e14b1 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -15,6 +15,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \ from calibre.gui2 import config, NONE from calibre.utils.config import prefs from calibre.library.field_metadata import TagsIcons +from calibre.utils.search_query_parser import saved_searches class TagsView(QTreeView): # {{{ @@ -221,6 +222,18 @@ class TagsModel(QAbstractItemModel): # {{{ self.db = db self.search_restriction = '' self.ignore_next_search = 0 + + # Reconstruct the user categories, putting them into metadata + tb_cats = self.db.field_metadata + for k in tb_cats.keys(): + if tb_cats[k]['kind'] in ['user', 'search']: + del tb_cats[k] + for user_cat in sorted(prefs['user_categories'].keys()): + cat_name = user_cat+':' # add the ':' to avoid name collision + tb_cats.add_user_category(label=cat_name, name=user_cat) + if len(saved_searches.names()): + tb_cats.add_search_category(label='search', name=_('Searches')) + data = self.get_node_tree(config['sort_by_popularity']) self.root_item = TagTreeItem() for i, r in enumerate(self.row_map): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 5ba603cc52..ff8e352583 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -231,6 +231,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.executescript(script) self.conn.commit() + # Reconstruct the user categories, putting them into field_metadata + # Assumption is that someone else will fix them if they change. + tb_cats = self.field_metadata + for k in tb_cats.keys(): + if tb_cats[k]['kind'] in ['user', 'search']: + del tb_cats[k] + for user_cat in sorted(prefs['user_categories'].keys()): + cat_name = user_cat+':' # add the ':' to avoid name collision + tb_cats.add_user_category(label=cat_name, name=user_cat) + if len(saved_searches.names()): + tb_cats.add_search_category(label='search', name=_('Searches')) + self.book_on_device_func = None self.data = ResultCache(self.FIELD_MAP, self.field_metadata) self.search = self.data.search @@ -653,17 +665,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): raise TypeError('icon_map passed to get_categories must be of type TagIcons') tb_cats = self.field_metadata - - # remove all user categories from field_metadata. They can - # easily come and go. We will add all the existing ones in below. - for k in tb_cats.keys(): - if tb_cats[k]['kind'] in ['user', 'search']: - del tb_cats[k] - #### First, build the standard and custom-column categories #### for category in tb_cats.keys(): cat = tb_cats[category] - if not cat['is_category']: + if not cat['is_category'] or not cat['kind'] == 'field': continue tn = cat['table'] categories[category] = [] #reserve the position in the ordered list @@ -757,7 +762,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # else: do nothing, to not include nodes w zero counts if len(items): cat_name = user_cat+':' # add the ':' to avoid name collision - tb_cats.add_user_category(label=cat_name, name=user_cat) # Not a problem if we accumulate entries in the icon map if icon_map is not None: icon_map[cat_name] = icon_map[':user'] @@ -776,7 +780,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for srch in saved_searches.names(): items.append(Tag(srch, tooltip=saved_searches.lookup(srch), icon=icon)) if len(items): - tb_cats.add_search_category(label='search', name=_('Searches')) if icon_map is not None: icon_map['search'] = icon_map['search'] categories['search'] = items diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 5a175ab2aa..a373ce9bad 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -64,6 +64,7 @@ class FieldMetadata(dict): _field_metadata = [ ('authors', {'table':'authors', 'column':'name', + 'link_column':'author', 'datatype':'text', 'is_multiple':',', 'kind':'field', @@ -73,6 +74,7 @@ class FieldMetadata(dict): 'is_category':True}), ('series', {'table':'series', 'column':'name', + 'link_column':'series', 'datatype':'text', 'is_multiple':None, 'kind':'field', @@ -91,6 +93,7 @@ class FieldMetadata(dict): 'is_category':True}), ('publisher', {'table':'publishers', 'column':'name', + 'link_column':'publisher', 'datatype':'text', 'is_multiple':None, 'kind':'field', @@ -100,6 +103,7 @@ class FieldMetadata(dict): 'is_category':True}), ('rating', {'table':'ratings', 'column':'rating', + 'link_column':'rating', 'datatype':'rating', 'is_multiple':None, 'kind':'field', diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index 6222a8950f..070ad1f3a6 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -290,9 +290,5 @@ class SchemaUpgrade(object): self.conn.executescript(script) for field in self.field_metadata.itervalues(): - if field['is_category'] and not field['is_custom'] and \ - field['table'] != 'news' and field['table'] is not None: - cn = field['table'][:-1] - if cn == 'serie': - cn += 's' - create_tag_browser_view(field['table'], cn, field['column']) + if field['is_category'] and not field['is_custom'] and 'link_column' in field: + create_tag_browser_view(field['table'], field['link_column'], field['column']) From 6dc8abdc766f1c3adf5129f257b20a593bbd43bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 14:53:50 -0600 Subject: [PATCH 09/18] Sort tags case insensitively inthe Tag Editor --- src/calibre/gui2/dialogs/tag_editor.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_editor.py b/src/calibre/gui2/dialogs/tag_editor.py index 9959e07f51..34c61914fe 100644 --- a/src/calibre/gui2/dialogs/tag_editor.py +++ b/src/calibre/gui2/dialogs/tag_editor.py @@ -9,6 +9,9 @@ from calibre.constants import islinux class TagEditor(QDialog, Ui_TagEditor): + def tag_cmp(self, x, y): + return cmp(x.lower(), y.lower()) + def __init__(self, window, db, index=None): QDialog.__init__(self, window) Ui_TagEditor.__init__(self) @@ -22,7 +25,7 @@ class TagEditor(QDialog, Ui_TagEditor): tags = [] if tags: tags = [tag.strip() for tag in tags.split(',') if tag.strip()] - tags.sort() + tags.sort(cmp=self.tag_cmp) for tag in tags: self.applied_tags.addItem(tag) else: @@ -32,7 +35,7 @@ class TagEditor(QDialog, Ui_TagEditor): all_tags = [tag for tag in self.db.all_tags()] all_tags = list(set(all_tags)) - all_tags.sort() + all_tags.sort(cmp=self.tag_cmp) for tag in all_tags: if tag not in tags: self.available_tags.addItem(tag) @@ -79,7 +82,7 @@ class TagEditor(QDialog, Ui_TagEditor): self.tags.append(tag) self.available_tags.takeItem(self.available_tags.row(item)) - self.tags.sort() + self.tags.sort(cmp=self.tag_cmp) self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) @@ -93,12 +96,17 @@ class TagEditor(QDialog, Ui_TagEditor): self.tags.remove(tag) self.available_tags.addItem(tag) - self.tags.sort() + self.tags.sort(cmp=self.tag_cmp) self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) - self.available_tags.sortItems() + items = [unicode(self.available_tags.item(x).text()) for x in + range(self.available_tags.count())] + items.sort(cmp=self.tag_cmp) + self.available_tags.clear() + for item in items: + self.available_tags.addItem(item) def add_tag(self): tags = unicode(self.add_tag_input.text()).split(',') @@ -109,7 +117,7 @@ class TagEditor(QDialog, Ui_TagEditor): if tag not in self.tags: self.tags.append(tag) - self.tags.sort() + self.tags.sort(cmp=self.tag_cmp) self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) From 0697315e05bc57bf59933e91d8d5ef5dead471a7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 22:16:14 +0100 Subject: [PATCH 10/18] Put news back as a category --- src/calibre/library/database2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ff8e352583..c8b2646144 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -668,7 +668,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): #### First, build the standard and custom-column categories #### for category in tb_cats.keys(): cat = tb_cats[category] - if not cat['is_category'] or not cat['kind'] == 'field': + if not cat['is_category'] or cat['kind'] in ['user', 'search']: continue tn = cat['table'] categories[category] = [] #reserve the position in the ordered list From 2a988e89b6306357597bccfee654eb3cb4d893bd Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 22:21:42 +0100 Subject: [PATCH 11/18] Fix descriptive comments in field_metadata.py --- src/calibre/library/field_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 9bd91a78b1..9d5cd4edc1 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -43,7 +43,8 @@ class FieldMetadata(dict): is_category: is a tag browser category. If true, then: table: name of the db table used to construct item list - column: name of the column in the connection table to join on + column: name of the column in the normalized table to join on + link_column: name of the column in the connection table to join on If these are None, then the category constructor must know how to build the item list (e.g., formats). The order below is the order that the categories will From 50e035243e34ba89710c1aedbd86a62b537e09e2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 May 2010 22:57:10 +0100 Subject: [PATCH 12/18] Fix create_custom_column.py not to remove the '#' on the key used for the custcol dict --- src/calibre/gui2/dialogs/config/create_custom_column.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.py b/src/calibre/gui2/dialogs/config/create_custom_column.py index 5ff048ed0a..3d5cb8ba53 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.py +++ b/src/calibre/gui2/dialogs/config/create_custom_column.py @@ -131,9 +131,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): else: date_format = {'date_format': None} + key = self.parent.db.field_metadata.custom_field_prefix+col if not self.editing_col: self.parent.db.field_metadata - self.parent.custcols[self.parent.db.field_metadata.custom_field_prefix+col] = { + self.parent.custcols[key] = { 'label':col, 'name':col_heading, 'datatype':col_type, @@ -144,13 +145,13 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'is_multiple':is_multiple, } item = QListWidgetItem(col_heading, self.parent.columns) - item.setData(Qt.UserRole, QVariant(col)) + item.setData(Qt.UserRole, QVariant(key)) item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable) item.setCheckState(Qt.Checked) else: idx = self.parent.columns.currentRow() item = self.parent.columns.item(idx) - item.setData(Qt.UserRole, QVariant(col)) + item.setData(Qt.UserRole, QVariant(key)) item.setText(col_heading) self.parent.custcols[self.orig_column_name]['label'] = col self.parent.custcols[self.orig_column_name]['name'] = col_heading From e9bf92ed0ea71132df7ef65000b82c02069b57e8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 16:35:55 -0600 Subject: [PATCH 13/18] Use PNP drive number based sorting on windows when the device has identical main memory and card PROD ids --- resources/recipes/wired.recipe | 2 +- src/calibre/devices/__init__.py | 6 ++++-- src/calibre/devices/binatone/driver.py | 11 +---------- src/calibre/devices/eb600/driver.py | 9 --------- src/calibre/devices/edge/driver.py | 8 -------- src/calibre/devices/eslick/driver.py | 8 -------- src/calibre/devices/hanlin/driver.py | 26 -------------------------- src/calibre/devices/iriver/driver.py | 8 -------- src/calibre/devices/jetbook/driver.py | 8 -------- src/calibre/devices/nook/driver.py | 8 -------- src/calibre/devices/scanner.py | 19 +++++++++++++++++-- src/calibre/devices/teclast/driver.py | 11 ----------- src/calibre/devices/usbms/device.py | 25 +++++++++++++------------ src/calibre/ebooks/epub/output.py | 16 ++++++++++++++++ 14 files changed, 52 insertions(+), 113 deletions(-) diff --git a/resources/recipes/wired.recipe b/resources/recipes/wired.recipe index 33577447cc..d45d987f48 100644 --- a/resources/recipes/wired.recipe +++ b/resources/recipes/wired.recipe @@ -39,7 +39,7 @@ class Wired(BasicNewsRecipe): dict(name=['object','embed','iframe','link']) ,dict(name='div', attrs={'class':['podcast_storyboard','tweetmeme_button']}) ] - remove_attributes = ['height','width'] + remove_attributes = ['height','width'] def parse_index(self): diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index bcbd9b1640..1fe1ca7fcb 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint +import sys, time, pprint, operator from functools import partial from StringIO import StringIO @@ -54,7 +54,9 @@ def debug(ioreg_to_tmp=False, buf=None): if iswindows: drives = win_pnp_drives(debug=True) out('Drives detected:') - out(pprint.pformat(drives)) + for drive in sorted(drives.keys(), + key=operator.attrgetter('order')): + prints(u'\t(%d)'%drive.order, drive, '~', drives[drive]) ioreg = None if isosx: diff --git a/src/calibre/devices/binatone/driver.py b/src/calibre/devices/binatone/driver.py index aca08df27d..8b3c803901 100644 --- a/src/calibre/devices/binatone/driver.py +++ b/src/calibre/devices/binatone/driver.py @@ -35,19 +35,10 @@ class README(USBMS): SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives - def linux_swap_drives(self, drives): if len(drives) < 2: return drives drives = list(drives) t = drives[0] drives[0] = drives[1] drives[1] = t - return tuple(drives) \ No newline at end of file + return tuple(drives) diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index d0b6a8f345..307531c357 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -48,15 +48,6 @@ class EB600(USBMS): EBOOK_DIR_CARD_A = '' SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives - class COOL_ER(EB600): diff --git a/src/calibre/devices/edge/driver.py b/src/calibre/devices/edge/driver.py index 1c39531c37..d14763f313 100644 --- a/src/calibre/devices/edge/driver.py +++ b/src/calibre/devices/edge/driver.py @@ -36,12 +36,4 @@ class EDGE(USBMS): EBOOK_DIR_MAIN = 'download' SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives diff --git a/src/calibre/devices/eslick/driver.py b/src/calibre/devices/eslick/driver.py index 4854c5c730..a38f742ecf 100644 --- a/src/calibre/devices/eslick/driver.py +++ b/src/calibre/devices/eslick/driver.py @@ -36,12 +36,4 @@ class ESLICK(USBMS): SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives diff --git a/src/calibre/devices/hanlin/driver.py b/src/calibre/devices/hanlin/driver.py index dae1d39c05..49f9dfab57 100644 --- a/src/calibre/devices/hanlin/driver.py +++ b/src/calibre/devices/hanlin/driver.py @@ -39,23 +39,6 @@ class HANLINV3(USBMS): SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card > main: - drives['main'] = card - drives['carda'] = main - - if card and not main: - drives['main'] = card - drives['carda'] = None - - return drives - - def windows_open_callback(self, drives): - if 'main' not in drives and 'carda' in drives: - drives['main'] = drives.pop('carda') - return drives def osx_sort_names(self, names): main = names.get('main', None) @@ -129,13 +112,4 @@ class BOOX(HANLINV3): EBOOK_DIR_CARD_A = 'MyBooks' - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives - diff --git a/src/calibre/devices/iriver/driver.py b/src/calibre/devices/iriver/driver.py index ca570eed7a..3dd94802c4 100644 --- a/src/calibre/devices/iriver/driver.py +++ b/src/calibre/devices/iriver/driver.py @@ -36,12 +36,4 @@ class IRIVER_STORY(USBMS): SUPPORTS_SUB_DIRS = True - def windows_open_callback(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives diff --git a/src/calibre/devices/jetbook/driver.py b/src/calibre/devices/jetbook/driver.py index e4fd840dc0..671fea5d75 100644 --- a/src/calibre/devices/jetbook/driver.py +++ b/src/calibre/devices/jetbook/driver.py @@ -80,11 +80,3 @@ class JETBOOK(USBMS): return mi - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index 16bf9479d8..5793dc7187 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -77,14 +77,6 @@ class NOOK(USBMS): with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: coverfile.write(coverdata) - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives def sanitize_path_components(self, components): return [x.replace('#', '_') for x in components] diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index c47bdfb6fa..d8f9c90a17 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -5,7 +5,7 @@ Device scanner that fetches list of devices on system ina platform dependent manner. ''' -import sys, os +import sys, os, re from threading import RLock from calibre import iswindows, isosx, plugins, islinux @@ -23,6 +23,14 @@ elif isosx: except: raise RuntimeError('Failed to load the usbobserver plugin: %s'%plugins['usbobserver'][1]) +class Drive(str): + + def __new__(self, val, order=0): + typ = str.__new__(self, val) + typ.order = order + return typ + + class WinPNPScanner(object): def __init__(self): @@ -45,6 +53,13 @@ class WinPNPScanner(object): finally: win32api.SetErrorMode(oldError) + def drive_order(self, pnp_id): + order = 0 + match = re.search(r'REV_.*?&(\d+)', pnp_id) + if match is not None: + order = int(pnp_id) + return order + def __call__(self, debug=False): if self.scanner is None: return {} @@ -66,7 +81,7 @@ class WinPNPScanner(object): val = [x.upper() for x in val] val = [x for x in val if 'USBSTOR' in x] if val: - ans[key+':\\'] = val[-1] + ans[Drive(key+':\\', order=self.drive_order(val[-1]))] = val[-1] return ans win_pnp_drives = WinPNPScanner() diff --git a/src/calibre/devices/teclast/driver.py b/src/calibre/devices/teclast/driver.py index afc852816c..1701696fad 100644 --- a/src/calibre/devices/teclast/driver.py +++ b/src/calibre/devices/teclast/driver.py @@ -30,14 +30,6 @@ class TECLAST_K3(USBMS): EBOOK_DIR_CARD_A = '' SUPPORTS_SUB_DIRS = True - def windows_sort_drives(self, drives): - main = drives.get('main', None) - card = drives.get('carda', None) - if card and main and card < main: - drives['main'] = card - drives['carda'] = main - - return drives class NEWSMY(TECLAST_K3): name = 'Newsmy device interface' @@ -50,9 +42,6 @@ class NEWSMY(TECLAST_K3): WINDOWS_MAIN_MEM = 'NEWSMY' WINDOWS_CARD_A_MEM = 'USBDISK____SD' - def windows_sort_drives(self, drives): - return drives - class IPAPYRUS(TECLAST_K3): name = 'iPapyrus device interface' diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 897baf82ca..7b85eae93e 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -11,13 +11,7 @@ intended to be subclassed with the relevant parts implemented for a particular device. This class handles device detection. ''' -import os -import subprocess -import time -import re -import sys -import glob - +import os, subprocess, time, re, sys, glob, operator from itertools import repeat from calibre.devices.interface import DevicePlugin @@ -62,6 +56,8 @@ class Device(DeviceConfig, DevicePlugin): BCD = None VENDOR_NAME = None + + # These can be None, string, list of strings or compiled regex WINDOWS_MAIN_MEM = None WINDOWS_CARD_A_MEM = None WINDOWS_CARD_B_MEM = None @@ -245,21 +241,26 @@ class Device(DeviceConfig, DevicePlugin): drives.get('main', None) is None: drives['main'] = drives.pop('carda') - drives = self.windows_open_callback(drives) - if drives.get('main', None) is None: raise DeviceError( _('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__) + # Sort drives by their PNP drive numbers if the CARD and MAIN + # MEM strings are identical + if self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM, + self.WINDOWS_CARD_B_MEM) or \ + self.WINDOWS_CARD_A_MEM == self.WINDOWS_CARD_B_MEM: + letters = sorted(drives.values(), key=operator.itemgetter('order')) + drives = {} + for which, letter in zip(['main', 'carda', 'cardb'], letters): + drives[which] = letter + drives = self.windows_sort_drives(drives) self._main_prefix = drives.get('main') self._card_a_prefix = drives.get('carda', None) self._card_b_prefix = drives.get('cardb', None) - def windows_open_callback(self, drives): - return drives - @classmethod def run_ioreg(cls, raw=None): if raw is not None: diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/epub/output.py index 47d06c2255..180b0c1f23 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/epub/output.py @@ -121,11 +121,27 @@ class EPUBOutput(OutputFormatPlugin): if not pre.text and len(pre) == 0: pre.tag = 'div' + def upshift_markup(self): + 'Upgrade markup to comply with XHTML 1.1 where possible' + from calibre.ebooks.oeb.base import XPath + for x in self.oeb.spine: + root = x.data + body = XPath('//h:body')(root) + if body: + body = body[0] + + if not hasattr(body, 'xpath'): + continue + for u in XPath('//h:u')(root): + u.tag = 'span' + u.set('style', 'text-decoration:underline') + def convert(self, oeb, output_path, input_plugin, opts, log): self.log, self.opts, self.oeb = log, opts, oeb self.workaround_ade_quirks() self.workaround_webkit_quirks() + self.upshift_markup() from calibre.ebooks.oeb.transforms.rescale import RescaleImages RescaleImages()(oeb, opts) From 9d7907fc29ab9bdefd3ed6c5bd24623a212b3089 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 20:44:24 -0600 Subject: [PATCH 14/18] Fix #5621 (Tooltips and GTK theme) --- src/calibre/gui2/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 4c75758567..aa17979d7d 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -556,6 +556,7 @@ class Application(QApplication): border-radius: 10px; opacity: 200; background-color: #e1e1ff; + color: black; } ''') From 352ea8642809c9c3e70c8106fef3ae50c661a82c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 23:41:16 -0600 Subject: [PATCH 15/18] ... --- src/calibre/devices/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index d8f9c90a17..ceba5d37d0 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -57,7 +57,7 @@ class WinPNPScanner(object): order = 0 match = re.search(r'REV_.*?&(\d+)', pnp_id) if match is not None: - order = int(pnp_id) + order = int(match.group(1)) return order def __call__(self, debug=False): From 5ea351492a3e72def50fb84c5951513f3ad3792f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 May 2010 23:49:27 -0600 Subject: [PATCH 16/18] ... --- src/calibre/devices/usbms/device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 7b85eae93e..96a5fc0b43 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -251,7 +251,7 @@ class Device(DeviceConfig, DevicePlugin): if self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM, self.WINDOWS_CARD_B_MEM) or \ self.WINDOWS_CARD_A_MEM == self.WINDOWS_CARD_B_MEM: - letters = sorted(drives.values(), key=operator.itemgetter('order')) + letters = sorted(drives.values(), key=operator.attrgetter('order')) drives = {} for which, letter in zip(['main', 'carda', 'cardb'], letters): drives[which] = letter From 141003eb42e1f56d24e84ca477ae9a18a98c613b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 May 2010 00:43:07 -0600 Subject: [PATCH 17/18] EPUB Output: Use correct SVG code when not preserving aspect ratio for covers --- src/calibre/ebooks/oeb/transforms/cover.py | 37 +++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/cover.py b/src/calibre/ebooks/oeb/transforms/cover.py index 9aee46c591..bd11a92af8 100644 --- a/src/calibre/ebooks/oeb/transforms/cover.py +++ b/src/calibre/ebooks/oeb/transforms/cover.py @@ -5,10 +5,15 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import textwrap +import textwrap, cStringIO from urllib import unquote from lxml import etree +try: + from PIL import Image as PILImage + PILImage +except ImportError: + import Image as PILImage from calibre import __appname__, __version__, guess_type @@ -28,9 +33,9 @@ class CoverManager(object): - + @@ -93,7 +98,6 @@ class CoverManager(object): title = unicode(m.title[0]) authors = [unicode(x) for x in m.creator if x.role == 'aut'] - import cStringIO cover_file = cStringIO.StringIO() try: try: @@ -142,6 +146,18 @@ class CoverManager(object): self.log.exception('Failed to generate default cover') return None + def inspect_cover(self, href): + from calibre.ebooks.oeb.base import urlnormalize + for x in self.oeb.manifest: + if x.href == urlnormalize(href): + try: + raw = x.data + f = cStringIO.StringIO(raw) + im = PILImage.open(f) + return im.size + except: + self.log.exception('Failed to read image dimensions') + return None, None def insert_cover(self): from calibre.ebooks.oeb.base import urldefrag @@ -152,6 +168,19 @@ class CoverManager(object): href = g['cover'].href else: href = self.default_cover() + width, height = self.inspect_cover(href) + if width is None or height is None: + self.log.warning('Failed to read cover dimensions') + width, height = 600, 800 + if self.preserve_aspect_ratio: + width, height = 600, 800 + self.svg_template = self.svg_template.replace('__viewbox__', + '0 0 %d %d'%(width, height)) + self.svg_template = self.svg_template.replace('__width__', + str(width)) + self.svg_template = self.svg_template.replace('__height__', + str(height)) + if href is not None: templ = self.non_svg_template if self.no_svg_cover \ else self.svg_template From 59007222ab085fdf1a315e425d92c6b0cc8a9f9a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 28 May 2010 00:57:17 -0600 Subject: [PATCH 18/18] Improved recipes for Corriere Della Serra and Leggo --- .../recipes/corriere_della_sera_it.recipe | 55 +++++++++++++------ resources/recipes/leggo_it.recipe | 10 +++- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/resources/recipes/corriere_della_sera_it.recipe b/resources/recipes/corriere_della_sera_it.recipe index 15d0bac928..b3bcebf505 100644 --- a/resources/recipes/corriere_della_sera_it.recipe +++ b/resources/recipes/corriere_della_sera_it.recipe @@ -9,22 +9,25 @@ __description__ = 'Italian daily newspaper' ''' http://www.corriere.it/ ''' - +import time from calibre.web.feeds.news import BasicNewsRecipe class ilCorriere(BasicNewsRecipe): - __author__ = 'Lorenzo Vigentini, based on Darko Miletic' + __author__ = 'Lorenzo Vigentini, based on Darko Miletic, Gabriele Marini' description = 'Italian daily newspaper' - cover_url = 'http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520' - title = u'Il Corriere della sera ' +# cover_url = 'http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520 + + + title = u'Il Corriere della sera' publisher = 'RCS Digital' category = 'News, politics, culture, economy, general interest' + encoding = 'cp1252' language = 'it' timefmt = '[%a, %d %b, %Y]' - oldest_article = 1 + oldest_article = 10 max_articles_per_feed = 100 use_embedded_content = False recursion = 10 @@ -51,17 +54,35 @@ class ilCorriere(BasicNewsRecipe): remove_tags_after = dict(name='p', attrs={'class':'footnotes'}) + def get_cover_url(self): + cover = None + st = time.localtime() + year = str(st.tm_year) + month = "%.2d" % st.tm_mon + day = "%.2d" % st.tm_mday + #http://images.corriere.it/primapagina/storico/2010_05_17/images/prima_pagina_grande.png + cover='http://images.corriere.it/primapagina/storico/'+ year + '_' + month +'_' + day +'/images/prima_pagina_grande.png' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + self.log("\nCover unavailable") + cover ='http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520' + return cover + feeds = [ - (u'Ultimora' , u'http://www.corriere.it/rss/ultimora.xml' ), - (u'Editoriali', u'http://www.corriere.it/rss/editoriali.xml'), - (u'Cronache' , u'http://www.corriere.it/rss/cronache.xml' ), - (u'Politica' , u'http://www.corriere.it/rss/politica.xml' ), - (u'Esteri' , u'http://www.corriere.it/rss/esteri.xml' ), - (u'Economia' , u'http://www.corriere.it/rss/economia.xml' ), - (u'Cultura' , u'http://www.corriere.it/rss/cultura.xml' ), - (u'Scienze' , u'http://www.corriere.it/rss/scienze.xml' ), - (u'Salute' , u'http://www.corriere.it/rss/salute.xml' ), - (u'Spettacolo', u'http://www.corriere.it/rss/spettacoli.xml'), - (u'Cinema e TV', u'http://www.corriere.it/rss/cinema.xml' ), - (u'Sport' , u'http://www.corriere.it/rss/sport.xml' ) + (u'Ultimora' , u'http://www.corriere.it/rss/ultimora.xml' ), + (u'Editoriali' , u'http://www.corriere.it/rss/editoriali.xml'), + (u'Cronache' , u'http://www.corriere.it/rss/cronache.xml' ), + (u'Politica' , u'http://www.corriere.it/rss/politica.xml' ), + (u'Esteri' , u'http://www.corriere.it/rss/esteri.xml' ), + (u'Economia' , u'http://www.corriere.it/rss/economia.xml' ), + (u'Cultura' , u'http://www.corriere.it/rss/cultura.xml' ), + (u'Scienze' , u'http://www.corriere.it/rss/scienze.xml' ), + (u'Salute' , u'http://www.corriere.it/rss/salute.xml' ), + (u'Spettacolo' , u'http://www.corriere.it/rss/spettacoli.xml'), + (u'Cinema e TV', u'http://www.corriere.it/rss/cinema.xml' ), + (u'Sport' , u'http://www.corriere.it/rss/sport.xml' ), + (u'Roma' , u'http://www.corriere.it/rss/homepage_roma.xml'), + (u'Milano' , u'http://www.corriere.it/rss/homepage_milano.xml') ] diff --git a/resources/recipes/leggo_it.recipe b/resources/recipes/leggo_it.recipe index 5be21e7dfc..13b2ca9018 100644 --- a/resources/recipes/leggo_it.recipe +++ b/resources/recipes/leggo_it.recipe @@ -57,9 +57,13 @@ class LeggoIT(BasicNewsRecipe): try: br.open(cover) except: - self.log("\nCover unavailable") - cover = 'http://www.leggo.it/img/logo-leggo2.gif' - + cover='http://www.leggo.it/'+ year + month + day + '/jpeg/LEGGO_ROMA_3.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + self.log("\nCover unavailable") + cover = 'http://www.leggo.it/img/logo-leggo2.gif' return cover