From a4dbc37a90d6f50c6554efcafe27f0a9503d9443 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 14:18:20 +0530 Subject: [PATCH 01/24] Text field searching --- src/calibre/db/cache.py | 5 +- src/calibre/db/search.py | 106 ++++++++++++++++++++++++++++++-- src/calibre/db/tests/reading.py | 8 +++ 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 88a2196a61..e2ecb369ca 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -412,8 +412,9 @@ class Cache(object): return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) @read_api - def search(self, query, restriction): - return self._search_api(self, query, restriction) + def search(self, query, restriction, virtual_fields=None): + return self._search_api(self, query, restriction, + virtual_fields=virtual_fields) # }}} diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 334bc046d8..398b153cac 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -14,6 +14,7 @@ from datetime import timedelta from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now from calibre.utils.icu import primary_find +from calibre.utils.localization import lang_map, canonicalize_lang from calibre.utils.search_query_parser import SearchQueryParser, ParseException # TODO: Thread safety of saved searches @@ -392,7 +393,7 @@ class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations): + locations, virtual_fields): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -400,6 +401,9 @@ class Parser(SearchQueryParser): self.bool_search, self.keypair_search = bool_search, keypair_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) + self.virtual_fields = virtual_fields or {} + if 'marked' not in self.virtual_fields: + self.virtual_fields['marked'] = self super(Parser, self).__init__(locations, optimize=True) @property @@ -411,8 +415,15 @@ class Parser(SearchQueryParser): def field_iter(self, name, candidates): get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False) - return self.dbcache.fields[name].iter_searchable_values(get_metadata, - candidates) + try: + field = self.dbcache.fields[name] + except KeyError: + field = self.virtual_fields[name] + return field.iter_searchable_values(get_metadata, candidates) + + def iter_searchable_values(self, *args, **kwargs): + for x in []: + yield x, set() def get_matches(self, location, query, candidates=None, allow_recursion=True): @@ -480,6 +491,8 @@ class Parser(SearchQueryParser): pass return matches + upf = prefs['use_primary_find_in_search'] + if location in self.field_metadata: fm = self.field_metadata[location] dt = fm['datatype'] @@ -519,7 +532,6 @@ class Parser(SearchQueryParser): # is a special case within the case if fm.get('is_csp', False): field_iter = partial(self.field_iter, location, candidates) - upf = prefs['use_primary_find_in_search'] if location == 'identifiers' and original_location == 'isbn': return self.keypair_search('=isbn:'+query, field_iter, candidates, upf) @@ -529,6 +541,87 @@ class Parser(SearchQueryParser): if len(location) >= 2 and location.startswith('@'): return self.get_user_category_matches(location[1:], icu_lower(query), candidates) + # Everything else (and 'all' matches) + matchkind, query = _matchkind(query) + all_locs = set() + text_fields = set() + field_metadata = {} + + for x, fm in self.field_metadata.iteritems(): + if x.startswith('@'): continue + if fm['search_terms'] and x != 'series_sort': + all_locs.add(x) + field_metadata[x] = fm + if fm['datatype'] in { 'composite', 'text', 'comments', 'series', 'enumeration' }: + text_fields.add(x) + + locations = all_locs if location == 'all' else {location} + + current_candidates = set(candidates) + + try: + rating_query = int(float(query)) * 2 + except: + rating_query = None + + try: + int_query = int(float(query)) + except: + int_query = None + + try: + float_query = float(query) + except: + float_query = None + + for location in locations: + current_candidates -= matches + q = query + if location == 'languages': + q = canonicalize_lang(query) + if q is None: + lm = lang_map() + rm = {v.lower():k for k,v in lm.iteritems()} + q = rm.get(query, query) + + if matchkind == CONTAINS_MATCH and q in {'true', 'false'}: + found = set() + for val, book_ids in self.field_iter(location, current_candidates): + if val and (not hasattr(val, 'strip') or val.strip()): + found |= book_ids + matches |= (found if q == 'true' else (current_candidates-found)) + continue + + dt = field_metadata.get(location, {}).get('datatype', None) + if dt == 'rating': + if rating_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == rating_query: + matches |= book_ids + continue + + if dt == 'float': + if float_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == float_query: + matches |= book_ids + continue + + if dt == 'int': + if int_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == int_query: + matches |= book_ids + continue + + if location in text_fields: + for val, book_ids in self.field_iter(location, current_candidates): + if val is not None: + if isinstance(val, basestring): + val = (val,) + if _match(q, val, matchkind, use_primary_find_in_search=upf): + matches |= book_ids + return matches def get_user_category_matches(self, location, query, candidates): @@ -567,7 +660,7 @@ class Search(object): def change_locations(self, newlocs): self.all_search_locations = newlocs - def __call__(self, dbcache, query, search_restriction): + def __call__(self, dbcache, query, search_restriction, virtual_fields=None): ''' Return the set of ids of all records that match the specified query and restriction @@ -596,7 +689,8 @@ class Search(object): self.date_search, self.num_search, self.bool_search, self.keypair_search, prefs[ 'limit_search_columns' ], - prefs[ 'limit_search_columns_to' ], self.all_search_locations) + prefs[ 'limit_search_columns_to' ], self.all_search_locations, + virtual_fields) try: ret = sqp.parse(q) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 627a692860..8a0230704b 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -218,6 +218,14 @@ class ReadingTest(BaseTest): 'identifiers:t:n', 'identifiers:=test:=two', 'identifiers:x:y', 'identifiers:z', + # Text tests + 'title:="Title One"', 'title:~title', '#enum:=one', '#enum:tw', + '#enum:false', '#enum:true', 'series:one', 'tags:one', 'tags:true', + 'tags:false', '2', 'one', '20.02', '"publisher one"', + + # User categories + # '@Good Authors:One', + # TODO: Tests for searching the size column and # cover:true|false # TODO: Tests for user categories searching From acf4811fb67c420328cfeac4c35b907ab3963977 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 15:26:31 +0530 Subject: [PATCH 02/24] Searching done --- src/calibre/db/cache.py | 37 ++++++++++++++++++++++++++++++--- src/calibre/db/search.py | 2 +- src/calibre/db/tests/reading.py | 4 ++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e2ecb369ca..6803ab93ed 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, traceback from collections import defaultdict from functools import wraps, partial @@ -18,6 +18,7 @@ from calibre.db.tables import VirtualTable from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import now +from calibre.utils.icu import sort_key def api(f): f.is_cache_api = True @@ -67,6 +68,36 @@ class Cache(object): lock = self.read_lock if ira else self.write_lock setattr(self, name, wrap_simple(lock, func)) + self.initialize_dynamic() + + def initialize_dynamic(self): + # Reconstruct the user categories, putting them into field_metadata + # Assumption is that someone else will fix them if they change. + self.field_metadata.remove_dynamic_categories() + for user_cat in sorted(self.pref('user_categories', {}).iterkeys(), key=sort_key): + cat_name = '@' + user_cat # add the '@' to avoid name collision + self.field_metadata.add_user_category(label=cat_name, name=user_cat) + + # add grouped search term user categories + muc = frozenset(self.pref('grouped_search_make_user_categories', [])) + for cat in sorted(self.pref('grouped_search_terms', {}).iterkeys(), key=sort_key): + if cat in muc: + # There is a chance that these can be duplicates of an existing + # user category. Print the exception and continue. + try: + self.field_metadata.add_user_category(label=u'@' + cat, name=cat) + except: + traceback.print_exc() + + # TODO: Saved searches + # if len(saved_searches().names()): + # self.field_metadata.add_search_category(label='search', name=_('Searches')) + + self.field_metadata.add_grouped_search_terms( + self.pref('grouped_search_terms', {})) + + self._search_api.change_locations(self.field_metadata.get_search_terms()) + @property def field_metadata(self): return self.backend.field_metadata @@ -319,8 +350,8 @@ class Cache(object): return ans @read_api - def pref(self, name): - return self.backend.prefs[name] + def pref(self, name, default=None): + return self.backend.prefs.get(name, default) @api def get_metadata(self, book_id, diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 398b153cac..7eb747da67 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -650,7 +650,7 @@ class Parser(SearchQueryParser): class Search(object): - def __init__(self, all_search_locations): + def __init__(self, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 8a0230704b..b2b63f9e32 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -222,13 +222,13 @@ class ReadingTest(BaseTest): 'title:="Title One"', 'title:~title', '#enum:=one', '#enum:tw', '#enum:false', '#enum:true', 'series:one', 'tags:one', 'tags:true', 'tags:false', '2', 'one', '20.02', '"publisher one"', + '"my comments one"', # User categories - # '@Good Authors:One', + '@Good Authors:One', '@Good Series.good tags:two', # TODO: Tests for searching the size column and # cover:true|false - # TODO: Tests for user categories searching )} old = None From d52a659da28bfacd4cbf025856d26ced4fe8a3d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 21 Jan 2013 15:27:35 +0530 Subject: [PATCH 03/24] ... --- src/calibre/db/tests/reading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index b2b63f9e32..8183611f91 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -227,7 +227,7 @@ class ReadingTest(BaseTest): # User categories '@Good Authors:One', '@Good Series.good tags:two', - # TODO: Tests for searching the size column and + # TODO: Tests for searching the size and #formats columns and # cover:true|false )} old = None From e0d0eb1973a444e10fd318713ee71d7a4a336e97 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 09:45:10 +0530 Subject: [PATCH 04/24] Fix TSN and St. Louis Post DIspatch --- recipes/st_louis_post_dispatch.recipe | 12 ++++++---- recipes/tsn.recipe | 33 ++++++++------------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/recipes/st_louis_post_dispatch.recipe b/recipes/st_louis_post_dispatch.recipe index 3b7701cedc..6d22a327ab 100644 --- a/recipes/st_louis_post_dispatch.recipe +++ b/recipes/st_louis_post_dispatch.recipe @@ -7,12 +7,16 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): oldest_article = 1 max_articles_per_feed = 15 + use_embedded_content = False + + no_stylesheets = True + auto_cleanup = True masthead_url = 'http://farm5.static.flickr.com/4118/4929686950_0e22e2c88a.jpg' feeds = [ (u'News-Bill McClellan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fbill-mclellan&f=rss&t=article'), (u'News-Columns', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcolumns*&l=50&f=rss&t=article'), - (u'News-Crime & Courtshttp://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), + (u'News-Crime & Courts', 'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), (u'News-Deb Peterson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fdeb-peterson&f=rss&t=article'), (u'News-Education', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2feducation&f=rss&t=article'), (u'News-Government & Politics', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fgovt-and-politics&f=rss&t=article'), @@ -62,9 +66,9 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): (u'Entertainment-House-O-Fun', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fhouse-o-fun&l=100&f=rss&t=article'), (u'Entertainment-Kevin C. Johnson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fmusic%2Fkevin-johnson&l=100&f=rss&t=article') ] - remove_empty_feeds = True - remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] - keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] + #remove_empty_feeds = True + #remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] + #keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] extra_css = 'p {text-align: left;}' diff --git a/recipes/tsn.recipe b/recipes/tsn.recipe index e822ebc633..6c3dbe5159 100644 --- a/recipes/tsn.recipe +++ b/recipes/tsn.recipe @@ -7,28 +7,15 @@ class AdvancedUserRecipe1289990851(BasicNewsRecipe): language = 'en_CA' __author__ = 'Nexus' no_stylesheets = True + auto_cleanup = True + use_embedded_content = False INDEX = 'http://tsn.ca/nhl/story/?id=nhl' - keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), - dict(name='div', attrs={'id':['tsnStory']})] - remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), - dict(name='div', attrs={'class':'textSize'})] - - def parse_index(self): - feeds = [] - soup = self.index_to_soup(self.INDEX) - feed_parts = soup.findAll('div', attrs={'class': 'feature'}) - for feed_part in feed_parts: - articles = [] - if not feed_part.h2: - continue - feed_title = feed_part.h2.string - article_parts = feed_part.findAll('a') - for article_part in article_parts: - article_title = article_part.string - article_date = '' - article_url = 'http://tsn.ca/' + article_part['href'] - articles.append({'title': article_title, 'url': article_url, 'description':'', 'date':article_date}) - if articles: - feeds.append((feed_title, articles)) - return feeds + #keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), + #dict(name='div', attrs={'id':['tsnStory']})] + #remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), + #dict(name='div', attrs={'class':'textSize'})] + feeds = [ +('News', + 'http://www.tsn.ca/datafiles/rss/Stories.xml'), +] From a15d236830cb5fb9b83f1d66482d695c2c56e852 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 10:11:28 +0530 Subject: [PATCH 05/24] Fix #1102408 (Untranslated string "(# books)") --- src/calibre/library/database2.py | 2 +- src/calibre/translations/calibre.pot | 293 ++++++++++++++------------- 2 files changed, 157 insertions(+), 138 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ab85421697..ad15f1a022 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1220,7 +1220,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc.append(_('Card A')) if b is not None: loc.append(_('Card B')) - return ', '.join(loc) + ((' (%s books)'%count) if count > 1 else '') + return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') def set_book_on_device_func(self, func): self.book_on_device_func = func diff --git a/src/calibre/translations/calibre.pot b/src/calibre/translations/calibre.pot index d3f3538c27..9f2bb510fe 100644 --- a/src/calibre/translations/calibre.pot +++ b/src/calibre/translations/calibre.pot @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: calibre 0.9.15\n" -"POT-Creation-Date: 2013-01-18 09:12+IST\n" -"PO-Revision-Date: 2013-01-18 09:12+IST\n" +"POT-Creation-Date: 2013-01-22 10:10+IST\n" +"PO-Revision-Date: 2013-01-22 10:10+IST\n" "Last-Translator: Automatically generated\n" "Language-Team: LANGUAGE\n" "MIME-Version: 1.0\n" @@ -21,9 +21,9 @@ msgid "Does absolutely nothing" msgstr "" #: /home/kovid/work/calibre/src/calibre/customize/__init__.py:59 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:106 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:109 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:120 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:139 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:142 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:153 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:379 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:380 #: /home/kovid/work/calibre/src/calibre/devices/hanvon/driver.py:114 @@ -42,8 +42,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/prst1/driver.py:469 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:480 #: /home/kovid/work/calibre/src/calibre/ebooks/chm/metadata.py:57 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:109 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:112 +#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:183 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/comic_input.py:189 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:99 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:101 @@ -106,10 +105,10 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/ozon.py:130 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/worker.py:26 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/txt.py:18 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:27 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:95 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:154 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:193 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:28 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:98 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:156 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:195 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/mobi6.py:618 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/utils.py:316 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/writer2/indexer.py:463 @@ -155,11 +154,11 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/email.py:193 #: /home/kovid/work/calibre/src/calibre/gui2/email.py:208 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:439 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1103 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1319 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1322 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1325 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1413 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1104 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1320 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1323 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1326 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1414 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:250 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:261 @@ -884,7 +883,7 @@ msgstr "" msgid "Path to library too long. Must be less than %d characters." msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/cache.py:134 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:167 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:666 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:67 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:678 @@ -894,23 +893,88 @@ msgstr "" msgid "Yes" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:163 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:186 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1218 msgid "Main" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:165 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:188 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:77 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1220 msgid "Card A" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:167 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:190 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:79 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1222 msgid "Card B" msgstr "" +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:313 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:577 +msgid "checked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:311 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:575 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "yes" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:310 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:574 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "no" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:312 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:576 +msgid "unchecked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:110 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:313 +msgid "today" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:111 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:314 +msgid "yesterday" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:112 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:315 +msgid "thismonth" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:113 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:316 +msgid "daysago" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:314 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:578 +msgid "empty" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:315 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:579 +msgid "blank" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:324 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:591 +msgid "Invalid boolean query \"{0}\"" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/debug.py:70 #: /home/kovid/work/calibre/src/calibre/gui2/main.py:47 msgid "Cause a running calibre instance, if any, to be shutdown. Note that if there are running jobs, they will be silently aborted, so use with care." @@ -1123,8 +1187,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:268 #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:324 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:391 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1134 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1136 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1128 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1130 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:277 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:279 msgid "Transferring books to device..." @@ -1135,8 +1199,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:491 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:525 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:430 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1147 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1158 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1141 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1152 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:301 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:332 msgid "Adding books to device metadata listing..." @@ -1158,8 +1222,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:374 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:479 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:486 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1190 #: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1196 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1202 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:366 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:371 msgid "Removing books from device metadata listing..." @@ -1668,7 +1732,7 @@ msgid "Communicate with MTP devices" msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:167 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:950 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:952 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:95 msgid "Get device information..." msgstr "" @@ -1967,17 +2031,17 @@ msgstr "" msgid "Too many connection attempts from %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1312 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1306 #, python-format msgid "Invalid port in options: %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1320 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1314 #, python-format msgid "Failed to connect to port %d. Try a different value." msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1332 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1326 msgid "Failed to allocate a random port" msgstr "" @@ -3443,7 +3507,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/quickview.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/template_dialog.py:222 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:83 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:150 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:162 #: /home/kovid/work/calibre/src/calibre/gui2/store/search/models.py:39 @@ -3456,7 +3520,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:770 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:85 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 #: /home/kovid/work/calibre/src/calibre/gui2/store/stores/mobileread/models.py:23 msgid "Author(s)" msgstr "" @@ -3501,7 +3565,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:982 #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:1228 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:201 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:804 msgid "Tags" msgstr "" @@ -3742,7 +3806,7 @@ msgstr "" msgid "Downloads metadata and covers from OZON.ru" msgstr "" -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:58 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:61 msgid "Sample Book" msgstr "" @@ -3778,7 +3842,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1281 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/transforms/htmltoc.py:15 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:221 -#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:217 +#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:219 msgid "Table of Contents" msgstr "" @@ -3863,7 +3927,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:71 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:160 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:176 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:800 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 msgid "Rating" msgstr "" @@ -4985,8 +5049,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:101 #: /home/kovid/work/calibre/src/calibre/gui2/dnd.py:84 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:518 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:830 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 msgid "Download failed" msgstr "" @@ -5018,7 +5082,7 @@ msgid "Download complete" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:123 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:892 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:901 msgid "Download log" msgstr "" @@ -5249,7 +5313,7 @@ msgid "Click the show details button to see which ones." msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/show_book_details.py:16 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:807 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 msgid "Show book details" msgstr "" @@ -5799,7 +5863,7 @@ msgid "Click to open" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:180 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:856 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:858 msgid "Ids" msgstr "" @@ -5809,7 +5873,7 @@ msgid "Book %(sidx)s of %(series)s" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:233 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1113 msgid "Collections" msgstr "" @@ -8315,7 +8379,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/device_drivers/mtp_config.py:421 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:141 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:885 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:894 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/tweaks.py:344 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:227 msgid "Copy to clipboard" @@ -8815,7 +8879,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/delete_matching_from_device.py:77 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:87 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:35 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:76 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:365 @@ -8931,7 +8995,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/edit_authors_dialog.py:122 #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main.py:160 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:536 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main.py:729 msgid "No matches found" msgstr "" @@ -9110,8 +9174,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:196 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:251 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:950 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1059 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:959 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1074 #: /home/kovid/work/calibre/src/calibre/gui2/proceed.py:48 msgid "View log" msgstr "" @@ -11539,13 +11603,13 @@ msgid "Modified" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:819 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1455 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1456 #: /home/kovid/work/calibre/src/calibre/gui2/tag_browser/model.py:335 msgid "The lookup/search name is \"{0}\"" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:825 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1457 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1458 msgid "This book's UUID is \"{0}\"" msgstr "" @@ -11574,20 +11638,20 @@ msgstr "" msgid "Could not set data, click Show Details to see why." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1107 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 msgid "In Library" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:355 msgid "Size" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1437 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1438 msgid "Marked for deletion" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1440 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1441 msgid "Double click to edit me

" msgstr "" @@ -11690,7 +11754,7 @@ msgid "Previous Page" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main_ui.py:133 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:947 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:956 #: /home/kovid/work/calibre/src/calibre/gui2/store/web_store_dialog_ui.py:62 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:215 msgid "Back" @@ -12131,7 +12195,7 @@ msgid "Edit Metadata" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single.py:63 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:940 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:949 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:108 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:219 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:410 @@ -12288,62 +12352,62 @@ msgid "" "cover stage, and vice versa." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:292 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:301 msgid "See at" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:446 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:455 msgid "calibre is downloading metadata from: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:468 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:477 msgid "Please wait" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:500 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:509 msgid "Query: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:519 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 msgid "Failed to download metadata. Click Show Details to see details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:537 msgid "Failed to find any books that match your search. Try making the search less specific. For example, use only the author's last name and a single distinctive word from the title.

To see the full log, click Show Details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:636 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:645 msgid "Current cover" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:639 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:648 msgid "Searching..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:800 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:809 #, python-format msgid "Downloading covers for %s, please wait..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:831 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:840 msgid "Failed to download any covers, click \"Show details\" for details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:837 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:846 #, python-format msgid "Could not find any covers for %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:848 #, python-format msgid "Found %(num)d covers of %(title)s. Pick the one you like best." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:928 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:937 msgid "Downloading metadata..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1043 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1058 msgid "Downloading cover..." msgstr "" @@ -16693,56 +16757,6 @@ msgid "" "

Stanza should see your calibre collection automatically. If not, try adding the URL http://myhostname:8080 as a new catalog in the Stanza reader on your iPhone. Here myhostname should be the fully qualified hostname or the IP address of the computer calibre is running on." msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:617 -msgid "checked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:615 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "yes" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:614 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "no" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:616 -msgid "unchecked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:355 -msgid "today" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:356 -msgid "yesterday" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:357 -msgid "thismonth" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:358 -msgid "daysago" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:618 -msgid "empty" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:619 -msgid "blank" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:631 -msgid "Invalid boolean query \"{0}\"" -msgstr "" - #: /home/kovid/work/calibre/src/calibre/library/catalogs/bibtex.py:36 #, python-format msgid "" @@ -17766,6 +17780,11 @@ msgstr "" msgid "creating custom column " msgstr "" +#: /home/kovid/work/calibre/src/calibre/library/database2.py:1223 +#, python-format +msgid " (%s books)" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/library/database2.py:3698 #, python-format msgid "

Migrating old database to ebook library in %s

" @@ -17985,19 +18004,19 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:317 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:355 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:649 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:651 msgid "All books" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:318 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:354 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:648 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:650 #: /home/kovid/work/calibre/src/calibre/library/server/opds.py:584 msgid "Newest" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:65 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:518 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:520 msgid "Loading, please wait" msgstr "" @@ -18050,65 +18069,65 @@ msgstr "" msgid "Random book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:403 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:472 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:405 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:474 msgid "Browse books by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:408 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:410 msgid "Choose a category to browse by:" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:543 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:545 msgid "Browsing by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:544 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:546 msgid "Up" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:684 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:686 msgid "in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:687 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:689 msgid "Books in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:781 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:783 msgid "Other formats" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:788 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:790 #, python-format msgid "Read %(title)s in the %(fmt)s format" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:793 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:795 msgid "Get" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:806 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 msgid "Details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:810 msgid "Permalink" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:811 msgid "A permanent link to this book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:821 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:823 msgid "This book has been deleted" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:927 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 msgid "in search" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:931 msgid "Matching books" msgstr "" From c64783797e1d90f704e6f5ca1af2c8d3b64130b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:10:35 +0530 Subject: [PATCH 06/24] ... --- src/calibre/ebooks/oeb/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 50df05ed16..e6e499236d 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -98,6 +98,9 @@ _self_closing_pat = re.compile( def close_self_closing_tags(raw): return _self_closing_pat.sub(r'<\g\g>>', raw) +def uuid_id(): + return 'u'+unicode(uuid.uuid4()) + def iterlinks(root, find_links_in_css=True): ''' Iterate over all links in a OEB Document. @@ -1528,7 +1531,7 @@ class TOC(object): if parent is None: parent = etree.Element(NCX('navMap')) for node in self.nodes: - id = node.id or unicode(uuid.uuid4()) + id = node.id or uuid_id() po = node.play_order if po == 0: po = 1 @@ -1634,10 +1637,10 @@ class PageList(object): return self.pages.remove(page) def to_ncx(self, parent=None): - plist = element(parent, NCX('pageList'), id=str(uuid.uuid4())) + plist = element(parent, NCX('pageList'), id=uuid_id()) values = dict((t, count(1)) for t in ('front', 'normal', 'special')) for page in self.pages: - id = page.id or unicode(uuid.uuid4()) + id = page.id or uuid_id() type = page.type value = str(values[type].next()) attrib = {'id': id, 'value': value, 'type': type, 'playOrder': '0'} From b79fdfe65a07b81ee4f5652a736e3f09b74cee0a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:11:41 +0530 Subject: [PATCH 07/24] Conversion: Replace all non-ascii characters in CSS class anmes, as they cause problems with some broken EPUB renderers. Fixes #1102587 (ODT-EPUB conversion generates invalid CSS) --- src/calibre/ebooks/oeb/transforms/flatcss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 12fbd3b7f1..03410e1a65 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -18,7 +18,7 @@ from calibre import guess_type from calibre.ebooks.oeb.base import (XHTML, XHTML_NS, CSS_MIME, OEB_STYLES, namespace, barename, XPath) from calibre.ebooks.oeb.stylizer import Stylizer -from calibre.utils.filenames import ascii_filename +from calibre.utils.filenames import ascii_filename, ascii_text COLLAPSE = re.compile(r'[ \t\r\n\v]+') STRIPNUM = re.compile(r'[-0-9]+$') @@ -437,7 +437,7 @@ class CSSFlattener(object): items.sort() css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) classes = node.get('class', '').strip() or 'calibre' - klass = STRIPNUM.sub('', classes.split()[0].replace('_', '')) + klass = ascii_text(STRIPNUM.sub('', classes.split()[0].replace('_', ''))) if css in styles: match = styles[css] else: From d45534ccc8553389eb0a7d5a859a24d21e9b1cde Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:44:44 +0530 Subject: [PATCH 08/24] Conversion: Do not error out because of an error in user supplied search replace rules. See #1102647 --- src/calibre/ebooks/conversion/preprocess.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index 72032cb998..82cc4c0f4a 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -515,6 +515,7 @@ class HTMLPreProcessor(object): if not getattr(self.extra_opts, 'keep_ligatures', False): html = _ligpat.sub(lambda m:LIGATURES[m.group()], html) + user_sr_rules = {} # Function for processing search and replace def do_search_replace(search_pattern, replace_txt): try: @@ -522,6 +523,7 @@ class HTMLPreProcessor(object): if not replace_txt: replace_txt = '' rules.insert(0, (search_re, replace_txt)) + user_sr_rules[(search_re, replace_txt)] = search_pattern except Exception as e: self.log.error('Failed to parse %r regexp because %s' % (search, as_unicode(e))) @@ -587,7 +589,16 @@ class HTMLPreProcessor(object): #dump(html, 'pre-preprocess') for rule in rules + end_rules: - html = rule[0].sub(rule[1], html) + try: + html = rule[0].sub(rule[1], html) + except re.error as e: + if rule in user_sr_rules: + self.log.error( + 'User supplied search & replace rule: %s -> %s ' + 'failed with error: %s, ignoring.'%( + user_sr_rules[rule], rule[1], e)) + else: + raise if is_pdftohtml and length > -1: # Dehyphenate From c1729160a34acf453f9439683dcfdf587f0aad4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 11:52:23 +0530 Subject: [PATCH 09/24] PDF Output: Dont error out for open type fonts without OS/2 tables --- src/calibre/utils/fonts/sfnt/metrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/fonts/sfnt/metrics.py b/src/calibre/utils/fonts/sfnt/metrics.py index 4f86948ff2..4843893fc3 100644 --- a/src/calibre/utils/fonts/sfnt/metrics.py +++ b/src/calibre/utils/fonts/sfnt/metrics.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' from future_builtins import map from calibre.utils.fonts.utils import get_all_font_names +from calibre.utils.fonts.sfnt.container import UnsupportedFont class FontMetrics(object): @@ -31,7 +32,10 @@ class FontMetrics(object): self._advance_widths = hhea.advance_widths self.cmap = self.sfnt[b'cmap'] self.units_per_em = self.head.units_per_em - self.os2 = self.sfnt[b'OS/2'] + try: + self.os2 = self.sfnt[b'OS/2'] + except KeyError: + raise UnsupportedFont('This font has no OS/2 table') self.os2.read_data() self.post = self.sfnt[b'post'] self.post.read_data() From 4cf17349af12df50c3f83dd50b6daf8c2237b681 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 22:26:23 +0530 Subject: [PATCH 10/24] Fix #1102403 (Private bug) --- src/calibre/utils/fonts/sfnt/metrics.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/utils/fonts/sfnt/metrics.py b/src/calibre/utils/fonts/sfnt/metrics.py index 4843893fc3..6f8deff31b 100644 --- a/src/calibre/utils/fonts/sfnt/metrics.py +++ b/src/calibre/utils/fonts/sfnt/metrics.py @@ -20,6 +20,10 @@ class FontMetrics(object): ''' def __init__(self, sfnt): + for table in (b'head', b'hhea', b'hmtx', b'cmap', b'OS/2', b'post', + b'name'): + if table not in sfnt: + raise UnsupportedFont('This font has no %s table'%table) self.sfnt = sfnt self.head = self.sfnt[b'head'] @@ -32,10 +36,7 @@ class FontMetrics(object): self._advance_widths = hhea.advance_widths self.cmap = self.sfnt[b'cmap'] self.units_per_em = self.head.units_per_em - try: - self.os2 = self.sfnt[b'OS/2'] - except KeyError: - raise UnsupportedFont('This font has no OS/2 table') + self.os2 = self.sfnt[b'OS/2'] self.os2.read_data() self.post = self.sfnt[b'post'] self.post.read_data() From 993fdfa34e1b72af5671ea3fbe6ae5d741f024ac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Jan 2013 22:32:15 +0530 Subject: [PATCH 11/24] New download: Do not convert all downloaded images to JPG format. This fixes the problem of PNG images with transparent backgrounds being rendered with black backgrounds --- src/calibre/web/fetch/simple.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index bea45f1c8d..87f97a3395 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -10,8 +10,6 @@ UTF-8 encoding with any charset declarations removed. import sys, socket, os, urlparse, re, time, copy, urllib2, threading, traceback, imghdr from urllib import url2pathname, quote from httplib import responses -from PIL import Image -from cStringIO import StringIO from base64 import b64decode from calibre import browser, relpath, unicode_path @@ -21,6 +19,8 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.ebooks.chardet import xml_to_unicode from calibre.utils.config import OptionParser from calibre.utils.logging import Log +from calibre.utils.magick import Image +from calibre.utils.magick.draw import identify_data class FetchError(Exception): pass @@ -374,8 +374,8 @@ class RecursiveFetcher(object): fname = ascii_filename('img'+str(c)) if isinstance(fname, unicode): fname = fname.encode('ascii', 'replace') - imgpath = os.path.join(diskpath, fname+'.jpg') - if (imghdr.what(None, data) is None and b' Date: Wed, 23 Jan 2013 08:38:35 +0530 Subject: [PATCH 12/24] ... --- src/calibre/devices/prst1/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 7291d5dbcb..72533860d4 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -195,7 +195,7 @@ class PRST1(USBMS): for i, row in enumerate(cursor): try: comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000); - except (OSError, IOError): + except (OSError, IOError, TypeError): # In case the db has incorrect path info continue device_date = int(row[1]); From 3875cd176ca447d7f8d411836144f27eaa3e69ba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 12:34:10 +0530 Subject: [PATCH 13/24] News download: Add support for logging in to sites that require javascript for their logins. Fixes #1101809 (Private bug) --- recipes/barrons.recipe | 22 ++++++------ src/calibre/utils/browser.py | 4 +++ src/calibre/web/feeds/news.py | 53 +++++++++++++++++++++++++--- src/calibre/web/jsbrowser/browser.py | 10 ++++++ src/calibre/web/jsbrowser/test.py | 27 ++++++++++++++ 5 files changed, 99 insertions(+), 17 deletions(-) diff --git a/recipes/barrons.recipe b/recipes/barrons.recipe index 41ed7e26ec..58c62e20e9 100644 --- a/recipes/barrons.recipe +++ b/recipes/barrons.recipe @@ -28,6 +28,8 @@ class Barrons(BasicNewsRecipe): ## Don't grab articles more than 7 days old oldest_article = 7 + use_javascript_to_login = True + requires_version = (0, 9, 16) extra_css = ''' .datestamp{font-family:Verdana,Geneva,Kalimati,sans-serif; font-size:x-small;} @@ -40,7 +42,7 @@ class Barrons(BasicNewsRecipe): .insettipUnit{font-size: x-small;} ''' remove_tags = [ - dict(name ='div', attrs={'class':['tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), + dict(name ='div', attrs={'class':['sTools sTools-t', 'tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), dict(name = 'a', attrs ={'class':'insetClose'}) ] @@ -60,21 +62,17 @@ class Barrons(BasicNewsRecipe): ] ] - def get_browser(self): - br = BasicNewsRecipe.get_browser() - if self.username is not None and self.password is not None: - br.open('http://commerce.barrons.com/auth/login') - br.select_form(nr=0) - br['username'] = self.username - br['password'] = self.password - br.submit() - return br + def javascript_login(self, br, username, password): + br.visit('http://commerce.barrons.com/auth/login') + f = br.select_form(nr=0) + f['username'] = username + f['password'] = password + br.submit(timeout=120) ## Use the print version of a page when available. - def print_version(self, url): main, sep, rest = url.rpartition('?') - return main + '#printmode' + return main + '#text.print' def postprocess_html(self, soup, first): diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index de21158ed7..fc04044ad3 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -32,6 +32,10 @@ class Browser(B): B.set_cookiejar(self, *args, **kwargs) self._clone_actions['set_cookiejar'] = ('set_cookiejar', args, kwargs) + def copy_cookies_from_jsbrowser(self, jsbrowser): + for cookie in jsbrowser.cookies: + self.cookiejar.set_cookie(cookie) + @property def cookiejar(self): return self._clone_actions['set_cookiejar'][1][0] diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 14834ff88c..22901f3ccc 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -332,6 +332,12 @@ class BasicNewsRecipe(Recipe): #: ignore_duplicate_articles = {'title', 'url'} ignore_duplicate_articles = None + #: If you set this True, then calibre will use javascript to login to the + #: website. This is needed for some websites that require the use of + #: javascript to login. If you set this to True you must implement the + #: :meth:`javascript_login` method, to do the actual logging in. + use_javascript_to_login = False + # See the built-in profiles for examples of these settings. def short_title(self): @@ -404,8 +410,7 @@ class BasicNewsRecipe(Recipe): ''' return url - @classmethod - def get_browser(cls, *args, **kwargs): + def get_browser(self, *args, **kwargs): ''' Return a browser instance used to fetch documents from the web. By default it returns a `mechanize `_ @@ -427,9 +432,47 @@ class BasicNewsRecipe(Recipe): return br ''' - br = browser(*args, **kwargs) - br.addheaders += [('Accept', '*/*')] - return br + if self.use_javascript_to_login: + if getattr(self, 'browser', None) is not None: + return self.clone_browser(self.browser) + from calibre.web.jsbrowser.browser import Browser + br = Browser() + with br: + self.javascript_login(br, self.username, self.password) + kwargs['user_agent'] = br.user_agent + ans = browser(*args, **kwargs) + ans.copy_cookies_from_jsbrowser(br) + return ans + else: + br = browser(*args, **kwargs) + br.addheaders += [('Accept', '*/*')] + return br + + def javascript_login(self, browser, username, password): + ''' + This method is used to login to a website that uses javascript for its + login form. After the login is complete, the cookies returned from the + website are copied to a normal (non-javascript) browser and the + download proceeds using those cookies. + + An example implementation:: + + def javascript_login(self, browser, username, password): + browser.visit('http://some-page-that-has-a-login') + form = browser.select_form(nr=0) # Select the first form on the page + form['username'] = username + form['password'] = password + browser.submit(timeout=120) # Submit the form and wait at most two minutes for loading to complete + + Note that you can also select forms with CSS2 selectors, like this:: + + browser.select_form('form#login_form') + browser.select_from('form[name="someform"]') + + ''' + raise NotImplementedError('You must implement the javascript_login()' + ' method if you set use_javascript_to_login' + ' to True') def clone_browser(self, br): ''' diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py index dd87b000a7..d8f0e79bc4 100644 --- a/src/calibre/web/jsbrowser/browser.py +++ b/src/calibre/web/jsbrowser/browser.py @@ -303,6 +303,10 @@ class Browser(QObject, FormsMixin): self.nam = NetworkAccessManager(log, use_disk_cache=use_disk_cache, parent=self) self.page.setNetworkAccessManager(self.nam) + @property + def user_agent(self): + return self.page.user_agent + def _wait_for_load(self, timeout, url=None): loop = QEventLoop(self) start_time = time.time() @@ -422,3 +426,9 @@ class Browser(QObject, FormsMixin): pass self.nam = self.page = None + def __enter__(self): + pass + + def __exit__(self, *args): + self.close() + diff --git a/src/calibre/web/jsbrowser/test.py b/src/calibre/web/jsbrowser/test.py index 8527f3ec92..6f18d7b850 100644 --- a/src/calibre/web/jsbrowser/test.py +++ b/src/calibre/web/jsbrowser/test.py @@ -11,6 +11,7 @@ import unittest, pprint, threading, time import cherrypy +from calibre import browser from calibre.web.jsbrowser.browser import Browser from calibre.library.server.utils import (cookie_max_age_to_expires, cookie_time_fmt) @@ -105,6 +106,12 @@ class Server(object): import traceback traceback.print_exc() + @cherrypy.expose + def receive_cookies(self): + self.received_cookies = {n:(c.value, dict(c)) for n, c in + dict(cherrypy.request.cookie).iteritems()} + return pprint.pformat(self.received_cookies) + class Test(unittest.TestCase): @classmethod @@ -202,6 +209,26 @@ class Test(unittest.TestCase): if fexp: self.assertEqual(fexp, cexp) + def test_cookie_copy(self): + 'Test copying of cookies from jsbrowser to mechanize' + self.assertEqual(self.browser.visit('http://127.0.0.1:%d/cookies'%self.port), + True) + sent_cookies = self.server.sent_cookies.copy() + self.browser.visit('http://127.0.0.1:%d/receive_cookies'%self.port) + orig_rc = self.server.received_cookies.copy() + br = browser(user_agent=self.browser.user_agent) + br.copy_cookies_from_jsbrowser(self.browser) + br.open('http://127.0.0.1:%d/receive_cookies'%self.port) + for name, vals in sent_cookies.iteritems(): + val = vals[0] + try: + rval = self.server.received_cookies[name][0] + except: + self.fail('The cookie: %s was not received by the server') + self.assertEqual(val, rval, + 'The received value for the cookie: %s, %s != %s'%( + name, rval, val)) + self.assertEqual(orig_rc, self.server.received_cookies) def tests(): return unittest.TestLoader().loadTestsFromTestCase(Test) From 9b6613c192e200e906bc510101d8a01bb69a20d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 12:37:45 +0530 Subject: [PATCH 14/24] ... --- src/calibre/ebooks/metadata/sources/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 46c6f7a313..e00c2e78d3 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -200,7 +200,7 @@ class Source(Plugin): #: during the identify phase touched_fields = frozenset() - #: Set this to True if your plugin return HTML formatted comments + #: Set this to True if your plugin returns HTML formatted comments has_html_comments = False #: Setting this to True means that the browser object will add From 0ce501fa0d059e6daaf97dab324183f6fc9abce8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 13:39:42 +0530 Subject: [PATCH 15/24] Store caches outside the config directory for non-portable calibre installs --- src/calibre/constants.py | 36 +++++++++++++++++++ src/calibre/devices/apple/driver.py | 6 ++-- .../library/catalogs/epub_mobi_builder.py | 5 ++- src/calibre/web/jsbrowser/browser.py | 7 ++-- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 40d39b0ad4..dff477bbc4 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -79,6 +79,42 @@ def debug(): global DEBUG DEBUG = True +_cache_dir = None + +def _get_cache_dir(): + confcache = os.path.join(config_dir, u'caches') + if isportable: + return confcache + if os.environ.has_key('CALIBRE_CACHE_DIRECTORY'): + return os.path.abspath(os.environ['CALIBRE_CACHE_DIRECTORY']) + + if iswindows: + w = plugins['winutil'][0] + candidate = os.path.join(w.special_folder_path(w.CSIDL_LOCAL_APPDATA), u'%s-cache'%__appname__) + elif isosx: + candidate = os.path.join(os.path.expanduser(u'~/Library/Caches'), __appname__) + else: + candidate = os.environ.get('XDG_CACHE_HOME', u'~/.cache') + candidate = os.path.join(os.path.expanduser(candidate), + __appname__) + if isinstance(candidate, bytes): + try: + candidate = candidate.decode(filesystem_encoding) + except ValueError: + candidate = confcache + if not os.path.exists(candidate): + try: + os.makedirs(candidate) + except: + candidate = confcache + return candidate + +def cache_dir(): + global _cache_dir + if _cache_dir is None: + _cache_dir = _get_cache_dir() + return _cache_dir + # plugins {{{ class Plugins(collections.Mapping): diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index eacb143790..95dedf546a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time -from calibre.constants import __appname__, __version__, DEBUG +from calibre.constants import __appname__, __version__, DEBUG, cache_dir from calibre import fit_image, confirm_config_name, strftime as _strftime from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback @@ -289,9 +289,7 @@ class ITUNES(DriverBase): # Properties cached_books = {} - cache_dir = os.path.join(config_dir, 'caches', 'itunes') calibre_library_path = prefs['library_path'] - archive_path = os.path.join(cache_dir, "thumbs.zip") description_prefix = "added by calibre" ejected = False iTunes = None @@ -884,6 +882,8 @@ class ITUNES(DriverBase): logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)]) logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)]) + self.cache_dir = os.path.join(cache_dir(), 'itunes') + self.archive_path = os.path.join(self.cache_dir, "thumbs.zip") # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 8f27db61be..9f946e2ee0 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -9,7 +9,7 @@ from xml.sax.saxutils import escape from calibre import (prepare_string_for_xml, strftime, force_unicode, isbytestring) -from calibre.constants import isosx +from calibre.constants import isosx, cache_dir from calibre.customize.conversion import DummyReporter from calibre.customize.ui import output_profiles from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString @@ -18,7 +18,6 @@ from calibre.ebooks.metadata import author_to_author_sort from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \ InvalidGenresSourceFieldException from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.config import config_dir from calibre.utils.date import format_date, is_date_undefined, now as nowf from calibre.utils.filenames import ascii_text, shorten_components_to from calibre.utils.icu import capitalize, collation_order, sort_key @@ -109,7 +108,7 @@ class CatalogBuilder(object): self.plugin = plugin self.reporter = report_progress self.stylesheet = stylesheet - self.cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.cache_dir = os.path.join(cache_dir(), 'catalog') self.catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') self.content_dir = os.path.join(self.catalog_path, "content") self.excluded_tags = self.get_excluded_tags() diff --git a/src/calibre/web/jsbrowser/browser.py b/src/calibre/web/jsbrowser/browser.py index d8f0e79bc4..c22d912128 100644 --- a/src/calibre/web/jsbrowser/browser.py +++ b/src/calibre/web/jsbrowser/browser.py @@ -16,7 +16,7 @@ from PyQt4.Qt import (QObject, QNetworkAccessManager, QNetworkDiskCache, from PyQt4.QtWebKit import QWebPage, QWebSettings, QWebView, QWebElement from calibre import USER_AGENT, prints, get_proxies, get_proxy_info -from calibre.constants import ispy3, config_dir +from calibre.constants import ispy3, cache_dir from calibre.utils.logging import ThreadSafeLog from calibre.gui2 import must_use_qt from calibre.web.jsbrowser.forms import FormsMixin @@ -44,7 +44,7 @@ class WebPage(QWebPage): # {{{ settings = self.settings() if enable_developer_tools: settings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True) - QWebSettings.enablePersistentStorage(os.path.join(config_dir, 'caches', + QWebSettings.enablePersistentStorage(os.path.join(cache_dir(), 'webkit-persistence')) QWebSettings.setMaximumPagesInCache(0) @@ -135,8 +135,7 @@ class NetworkAccessManager(QNetworkAccessManager): # {{{ self.log = log if use_disk_cache: self.cache = QNetworkDiskCache(self) - self.cache.setCacheDirectory(os.path.join(config_dir, 'caches', - 'jsbrowser')) + self.cache.setCacheDirectory(os.path.join(cache_dir(), 'jsbrowser')) self.setCache(self.cache) self.sslErrors.connect(self.on_ssl_errors) self.pf = ProxyFactory(log) From d774d54c2f880d9a14dd75e328265972b1040d4f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 14:01:35 +0530 Subject: [PATCH 16/24] Move the mobileread get books cache into the cache directory --- .../store/stores/mobileread/mobileread_plugin.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py index 9e41aa45a1..942f345820 100644 --- a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py +++ b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py @@ -6,10 +6,12 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import os from threading import Lock from PyQt4.Qt import (QUrl, QCoreApplication) +from calibre.constants import cache_dir from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig @@ -19,12 +21,20 @@ from calibre.gui2.store.stores.mobileread.models import SearchFilter from calibre.gui2.store.stores.mobileread.cache_progress_dialog import CacheProgressDialog from calibre.gui2.store.stores.mobileread.cache_update_thread import CacheUpdateThread from calibre.gui2.store.stores.mobileread.store_dialog import MobileReadStoreDialog +from calibre.utils.config import JSONConfig + +class Cache(JSONConfig): + + def __init__(self): + JSONConfig.__init__(self, 'mobileread_store') + self.file_path = os.path.join(cache_dir(), 'mobileread_get_books.json') class MobileReadStore(BasicStoreConfig, StorePlugin): def __init__(self, *args, **kwargs): StorePlugin.__init__(self, *args, **kwargs) self.lock = Lock() + self.cache = Cache() def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' @@ -61,7 +71,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): suppress_progress=False): if self.lock.acquire(False): try: - update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout) + update_thread = CacheUpdateThread(self.cache, self.seralize_books, timeout) if not suppress_progress: progress = CacheProgressDialog(parent) progress.set_message(_('Updating MobileRead book cache...')) @@ -85,7 +95,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): self.lock.release() def get_book_list(self): - return self.deseralize_books(self.config.get('book_list', [])) + return self.deseralize_books(self.cache.get('book_list', [])) def seralize_books(self, books): sbooks = [] From 7e2ad6914db15eb7ffd829019be66e2a43ef4035 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 14:08:01 +0530 Subject: [PATCH 17/24] Delay load MR cache --- .../stores/mobileread/mobileread_plugin.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py index 942f345820..bf1b2013dd 100644 --- a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py +++ b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py @@ -21,20 +21,22 @@ from calibre.gui2.store.stores.mobileread.models import SearchFilter from calibre.gui2.store.stores.mobileread.cache_progress_dialog import CacheProgressDialog from calibre.gui2.store.stores.mobileread.cache_update_thread import CacheUpdateThread from calibre.gui2.store.stores.mobileread.store_dialog import MobileReadStoreDialog -from calibre.utils.config import JSONConfig - -class Cache(JSONConfig): - - def __init__(self): - JSONConfig.__init__(self, 'mobileread_store') - self.file_path = os.path.join(cache_dir(), 'mobileread_get_books.json') class MobileReadStore(BasicStoreConfig, StorePlugin): def __init__(self, *args, **kwargs): StorePlugin.__init__(self, *args, **kwargs) self.lock = Lock() - self.cache = Cache() + + @property + def cache(self): + if not hasattr(self, '_mr_cache'): + from calibre.utils.config import JSONConfig + self._mr_cache = JSONConfig('mobileread_get_books') + self._mr_cache.file_path = os.path.join(cache_dir(), + 'mobileread_get_books.json') + self._mr_cache.refresh() + return self._mr_cache def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' From 276939c39b4105c98a3ae488d48605d0dd26d3b7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 23 Jan 2013 22:12:50 +0530 Subject: [PATCH 18/24] Fix #1103504 (Crashed by column color tab) --- src/calibre/gui2/preferences/coloring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 4b867f347d..72fe4fb12b 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -413,6 +413,7 @@ class RulesModel(QAbstractListModel): # {{{ rules = list(prefs['column_color_rules']) self.rules = [] for col, template in rules: + if col not in self.fm: continue try: rule = rule_from_template(self.fm, template) except: From 50082b9779932ac9189fc257a18265d90568bb49 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 09:20:04 +0530 Subject: [PATCH 19/24] Driver for LG E400. Fixes #1103741 (Device not recognized) --- src/calibre/devices/android/driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index acee4938f5..d709ae91a9 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -141,7 +141,7 @@ class ANDROID(USBMS): # LG 0x1004 : { - 0x61c5 : [0x100, 0x226, 0x227, 0x9999], + 0x61c5 : [0x100, 0x226, 0x227, 0x229, 0x9999], 0x61cc : [0x226, 0x227, 0x9999, 0x100], 0x61ce : [0x226, 0x227, 0x9999, 0x100], 0x618e : [0x226, 0x227, 0x9999, 0x100], @@ -235,7 +235,7 @@ class ANDROID(USBMS): 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', - 'ICS'] + 'ICS', 'E400'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -246,7 +246,7 @@ class ANDROID(USBMS): 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', - 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F'] + 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD'] OSX_MAIN_MEM = 'Android Device Main Memory' From dd001ca8c1a8ac68dcf2002311c7a39a1fba9f90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 09:27:03 +0530 Subject: [PATCH 20/24] ... --- src/calibre/devices/apple/driver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 95dedf546a..f941ccd659 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -307,6 +307,14 @@ class ITUNES(DriverBase): update_msg = None update_needed = False + @property + def cache_dir(self): + return os.path.join(cache_dir(), 'itunes') + + @property + def archive_path(self): + return os.path.join(self.cache_dir, "thumbs.zip") + # Public methods def add_books_to_metadata(self, locations, metadata, booklists): ''' @@ -882,8 +890,6 @@ class ITUNES(DriverBase): logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)]) logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)]) - self.cache_dir = os.path.join(cache_dir(), 'itunes') - self.archive_path = os.path.join(self.cache_dir, "thumbs.zip") # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: From 68f58f2c30b31accecd135391c2e2ff0183f83ce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 14:41:03 +0530 Subject: [PATCH 21/24] Start work on get_categories() for the new db backend --- src/calibre/db/cache.py | 4 +- src/calibre/db/categories.py | 105 +++++++++++++++++++++++++++++++ src/calibre/db/fields.py | 88 ++++++++++++++++++++++---- src/calibre/db/tables.py | 28 +++------ src/calibre/library/database2.py | 36 +---------- 5 files changed, 191 insertions(+), 70 deletions(-) create mode 100644 src/calibre/db/categories.py diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 6803ab93ed..fec8d4920c 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -293,13 +293,13 @@ class Cache(object): Return all the books associated with the item identified by ``item_id``, where the item belongs to the field ``name``. - Returned value is a tuple of book ids, or the empty tuple if the item + Returned value is a set of book ids, or the empty set if the item or the field does not exist. ''' try: return self.fields[name].books_for(item_id) except (KeyError, IndexError): - return () + return set() @read_api def all_book_ids(self, type=frozenset): diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py new file mode 100644 index 0000000000..5e8e5ae687 --- /dev/null +++ b/src/calibre/db/categories.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from functools import partial + +from calibre.library.field_metadata import TagsIcons +from calibre.utils.config_base import tweaks + +CATEGORY_SORTS = { 'name', 'popularity', 'rating' } + +class Tag(object): + + def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, + tooltip=None, icon=None, category=None, id_set=None, + is_editable=True, is_searchable=True, use_sort_as_name=False): + self.name = self.original_name = name + self.id = id + self.count = count + self.state = state + self.is_hierarchical = '' + self.is_editable = is_editable + self.is_searchable = is_searchable + self.id_set = id_set if id_set is not None else set([]) + self.avg_rating = avg/2.0 if avg is not None else 0 + self.sort = sort + self.use_sort_as_name = use_sort_as_name + if self.avg_rating > 0: + if tooltip: + tooltip = tooltip + ': ' + tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict( + tt=tooltip, rating=self.avg_rating) + self.tooltip = tooltip + self.icon = icon + self.category = category + + def __unicode__(self): + return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state, + self.category, self.tooltip) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __repr__(self): + return str(self) + +def find_categories(field_metadata): + for category, cat in field_metadata.iteritems(): + if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and + category not in {'news', 'formats'} and not cat.get('is_csp', False)): + yield (category, cat['is_multiple'].get('cache_to_list', None), False) + elif (cat['datatype'] == 'composite' and + cat['display'].get('make_category', False)): + yield (category, cat['is_multiple'].get('cache_to_list', None), True) + +def create_tag_class(category, fm, icon_map): + cat = fm[category] + icon = None + tooltip = '(' + category + ')' + label = fm.key_to_label(category) + if icon_map: + if not fm.is_custom_field(category): + if category in icon_map: + icon = icon_map[label] + else: + icon = icon_map['custom:'] + icon_map[category] = icon + is_editable = category not in { 'news', 'rating', 'languages' } + + if (tweaks['categories_use_field_for_author_name'] == 'author_sort' and + (category == 'authors' or + (cat['display'].get('is_names', False) and + cat['is_custom'] and cat['is_multiple'] and + cat['datatype'] == 'text'))): + use_sort_as_name = True + else: + use_sort_as_name = False + + return partial(Tag, use_sort_as_name=use_sort_as_name, icon=icon, + tooltip=tooltip, is_editable=is_editable, + category=category) + + +def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): + if icon_map is not None and type(icon_map) != TagsIcons: + raise TypeError('icon_map passed to get_categories must be of type TagIcons') + if sort not in CATEGORY_SORTS: + raise ValueError('sort ' + sort + ' not a valid value') + + fm = dbcache.field_metadata + book_rating_map = dbcache.fields['rating'].book_value_map + + categories = {} + book_ids = frozenset(book_ids) + for category, is_multiple, is_composite in find_categories(fm): + tag_class = create_tag_class(category, fm, icon_map) + categories[category] = dbcache.fields[category].get_categories( + tag_class, book_rating_map, sort, dbcache.fields['language'], book_ids) + + diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e0950fff3b..f6baec4bdc 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -9,7 +9,8 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from threading import Lock -from collections import defaultdict +from collections import defaultdict, Counter +from operator import attrgetter from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY from calibre.ebooks.metadata import title_sort @@ -24,22 +25,28 @@ class Field(object): def __init__(self, name, table): self.name, self.table = name, table - self.has_text_data = self.metadata['datatype'] in ('text', 'comments', - 'series', 'enumeration') - self.table_type = self.table.table_type dt = self.metadata['datatype'] + self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'} + self.table_type = self.table.table_type self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x) self._default_sort_key = '' - if self.metadata['datatype'] in ('int', 'float', 'rating'): + if dt in { 'int', 'float', 'rating' }: self._default_sort_key = 0 - elif self.metadata['datatype'] == 'bool': + elif dt == 'bool': self._default_sort_key = None - elif self.metadata['datatype'] == 'datetime': + elif dt == 'datetime': self._default_sort_key = UNDEFINED_DATE if self.name == 'languages': self._sort_key = lambda x:sort_key(calibre_langcode_to_name(x)) self.is_multiple = (bool(self.metadata['is_multiple']) or self.name == 'formats') + self.category_formatter = type(u'') + self.category_sort_reverse = False + if dt == 'rating': + self.category_formatter = lambda x:'\u2605'*int(x/2) + self.category_sort_reverse = True + elif name == 'languages': + self.category_formatter = calibre_langcode_to_name @property def metadata(self): @@ -63,7 +70,7 @@ class Field(object): def books_for(self, item_id): ''' Return the ids of all books associated with the item identified by - item_id as a tuple. An empty tuple is returned if no books are found. + item_id as a set. An empty set is returned if no books are found. ''' raise NotImplementedError() @@ -94,6 +101,34 @@ class Field(object): ''' raise NotImplementedError() + def get_categories(self, tag_class, book_rating_map, sort, lang_field, book_ids=None): + ans = [] + if not self.is_many: + return ans + + special_sort = hasattr(self, 'category_sort_value') + for item_id, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for + book_id in item_book_ids) if r > 0) + avg = sum(ratings)/len(ratings) + name = self.category_formatter(self.table.id_map[item_id]) + sval = (self.category_sort_value(item_id, item_book_ids, lang_field) + if special_sort else name) + c = tag_class(name, id=item_id, sort=sval, avg=avg, + id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + if sort == 'popularity': + key=attrgetter('count') + elif sort == 'rating': + key=attrgetter('avg_rating') + else: + key=lambda x:sort_key(x.sort or x.name) + ans.sort(key=key, reverse=self.category_sort_reverse) + return ans + class OneToOneField(Field): def for_book(self, book_id, default_value=None): @@ -103,7 +138,7 @@ class OneToOneField(Field): return (book_id,) def books_for(self, item_id): - return (item_id,) + return {item_id} def __iter__(self): return self.table.book_col_map.iterkeys() @@ -223,7 +258,7 @@ class ManyToOneField(Field): return (id_,) def books_for(self, item_id): - return self.table.col_book_map.get(item_id, ()) + return self.table.col_book_map.get(item_id, set()) def __iter__(self): return self.table.id_map.iterkeys() @@ -238,11 +273,17 @@ class ManyToOneField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.col_book_map + empty = set() for item_id, val in self.table.id_map.iteritems(): - book_ids = set(cbm.get(item_id, ())).intersection(candidates) + book_ids = cbm.get(item_id, empty).intersection(candidates) if book_ids: yield val, book_ids + @property + def book_value_map(self): + return {book_id:self.table.id_map[item_id] for book_id, item_id in + self.book_col_map.iteritems()} + class ManyToManyField(Field): is_many = True @@ -263,7 +304,7 @@ class ManyToManyField(Field): return self.table.book_col_map.get(book_id, ()) def books_for(self, item_id): - return self.table.col_book_map.get(item_id, ()) + return self.table.col_book_map.get(item_id, set()) def __iter__(self): return self.table.id_map.iterkeys() @@ -282,8 +323,9 @@ class ManyToManyField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.col_book_map + empty = set() for item_id, val in self.table.id_map.iteritems(): - book_ids = set(cbm.get(item_id, ())).intersection(candidates) + book_ids = cbm.get(item_id, empty).intersection(candidates) if book_ids: yield val, book_ids @@ -327,6 +369,9 @@ class AuthorsField(ManyToManyField): 'link' : self.table.alink_map[author_id], } + def category_sort_value(self, item_id, book_ids, language_field): + return self.table.asort_map[item_id] + class FormatsField(ManyToManyField): def for_book(self, book_id, default_value=None): @@ -361,6 +406,23 @@ class SeriesField(ManyToOneField): return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id in all_book_ids} + def category_sort_value(self, item_id, book_ids, language_field): + lang = None + tss = tweaks['title_series_sorting'] + if tss != 'strictly_alphabetic': + lang_map = language_field.book_col_map + c = Counter() + + for book_id in book_ids: + l = lang_map.get(book_id, None) + if l: + c[l[0]] += 1 + + if c: + lang = c.most_common(1)[0][0] + val = self.table.id_map[item_id] + return title_sort(val, order=tss, lang=lang) + def create_field(name, table): cls = { ONE_ONE : OneToOneField, diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 234a7fe4a8..4a8b4492fd 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -132,13 +132,10 @@ class ManyToOneTable(Table): 'SELECT book, {0} FROM {1}'.format( self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) self.book_col_map[row[0]] = row[1] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - class ManyToManyTable(ManyToOneTable): ''' @@ -154,15 +151,12 @@ class ManyToManyTable(ManyToOneTable): for row in db.conn.execute( self.selectq.format(self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(row[1]) - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(self.book_col_map[key]) @@ -191,8 +185,8 @@ class FormatsTable(ManyToManyTable): if row[1] is not None: fmt = row[1].upper() if fmt not in self.col_book_map: - self.col_book_map[fmt] = [] - self.col_book_map[fmt].append(row[0]) + self.col_book_map[fmt] = set() + self.col_book_map[fmt].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(fmt) @@ -200,9 +194,6 @@ class FormatsTable(ManyToManyTable): self.fname_map[row[0]] = {} self.fname_map[row[0]][fmt] = row[2] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) @@ -215,15 +206,12 @@ class IdentifiersTable(ManyToManyTable): for row in db.conn.execute('SELECT book, type, val FROM identifiers'): if row[1] is not None and row[2] is not None: if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = {} self.book_col_map[row[0]][row[1]] = row[2] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - class LanguagesTable(ManyToManyTable): def read_id_maps(self, db): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ad15f1a022..5149b7a0de 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -44,47 +44,13 @@ from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions from calibre.db.errors import NoSuchFormat from calibre.db.lazy import FormatMetadata, FormatsList +from calibre.db.categories import Tag from calibre.utils.localization import (canonicalize_lang, calibre_langcode_to_name) copyfile = os.link if hasattr(os, 'link') else shutil.copyfile SPOOL_SIZE = 30*1024*1024 -class Tag(object): - - def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, - tooltip=None, icon=None, category=None, id_set=None, - is_editable = True, is_searchable=True, use_sort_as_name=False): - self.name = self.original_name = name - self.id = id - self.count = count - self.state = state - self.is_hierarchical = '' - self.is_editable = is_editable - self.is_searchable = is_searchable - self.id_set = id_set if id_set is not None else set([]) - self.avg_rating = avg/2.0 if avg is not None else 0 - self.sort = sort - self.use_sort_as_name = use_sort_as_name - if self.avg_rating > 0: - if tooltip: - tooltip = tooltip + ': ' - tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict( - tt=tooltip, rating=self.avg_rating) - self.tooltip = tooltip - self.icon = icon - self.category = category - - def __unicode__(self): - return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state, - self.category, self.tooltip) - - def __str__(self): - return unicode(self).encode('utf-8') - - def __repr__(self): - return str(self) - class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' An ebook metadata database that stores references to ebook files on disk. From 6a9fb786d938ef0c4c5193ef064834d97673ed8e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 14:53:05 +0530 Subject: [PATCH 22/24] Avoid per book function call when sorting based on book language --- src/calibre/db/cache.py | 8 +++----- src/calibre/db/categories.py | 3 ++- src/calibre/db/fields.py | 40 +++++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index fec8d4920c..65e2e614eb 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -415,9 +415,7 @@ class Cache(object): all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None else ids_to_sort) get_metadata = partial(self._get_metadata, get_user_categories=False) - def get_lang(book_id): - ans = self._field_for('languages', book_id) - return ans[0] if ans else None + lang_map = self.fields['languages'].book_value_map fm = {'title':'sort', 'authors':'author_sort'} @@ -426,10 +424,10 @@ class Cache(object): idx = field + '_index' is_series = idx in self.fields ans = self.fields[fm.get(field, field)].sort_keys_for_books( - get_metadata, get_lang, all_book_ids,) + get_metadata, lang_map, all_book_ids,) if is_series: idx_ans = self.fields[idx].sort_keys_for_books( - get_metadata, get_lang, all_book_ids) + get_metadata, lang_map, all_book_ids) ans = {k:(v, idx_ans[k]) for k, v in ans.iteritems()} return ans diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index 5e8e5ae687..0def3b144e 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -94,12 +94,13 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): fm = dbcache.field_metadata book_rating_map = dbcache.fields['rating'].book_value_map + lang_map = dbcache.fileds['languages'].book_value_map categories = {} book_ids = frozenset(book_ids) for category, is_multiple, is_composite in find_categories(fm): tag_class = create_tag_class(category, fm, icon_map) categories[category] = dbcache.fields[category].get_categories( - tag_class, book_rating_map, sort, dbcache.fields['language'], book_ids) + tag_class, book_rating_map, sort, lang_map, book_ids) diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index f6baec4bdc..6a6deacf10 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -84,7 +84,7 @@ class Field(object): ''' return iter(()) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ''' Return a mapping of book_id -> sort_key. The sort key is suitable for use in sorting the list of all books by this field, via the python cmp @@ -101,7 +101,7 @@ class Field(object): ''' raise NotImplementedError() - def get_categories(self, tag_class, book_rating_map, sort, lang_field, book_ids=None): + def get_categories(self, tag_class, book_rating_map, sort, lang_map, book_ids=None): ans = [] if not self.is_many: return ans @@ -115,7 +115,7 @@ class Field(object): book_id in item_book_ids) if r > 0) avg = sum(ratings)/len(ratings) name = self.category_formatter(self.table.id_map[item_id]) - sval = (self.category_sort_value(item_id, item_book_ids, lang_field) + sval = (self.category_sort_value(item_id, item_book_ids, lang_map) if special_sort else name) c = tag_class(name, id=item_id, sort=sval, avg=avg, id_set=item_book_ids, count=len(item_book_ids)) @@ -143,7 +143,7 @@ class OneToOneField(Field): def __iter__(self): return self.table.book_col_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : self._sort_key(self.table.book_col_map.get(id_, self._default_sort_key)) for id_ in all_book_ids} @@ -185,7 +185,7 @@ class CompositeField(OneToOneField): ans = mi.get('#'+self.metadata['label']) return ans - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in all_book_ids} @@ -228,7 +228,7 @@ class OnDeviceField(OneToOneField): def __iter__(self): return iter(()) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : self.for_book(id_) for id_ in all_book_ids} @@ -263,7 +263,7 @@ class ManyToOneField(Field): def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, None) for id_ in all_book_ids} sk_map = {cid : (self._default_sort_key if cid is None else @@ -309,7 +309,7 @@ class ManyToManyField(Field): def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} all_cids = set() @@ -337,6 +337,11 @@ class ManyToManyField(Field): for count, book_ids in val_map.iteritems(): yield count, book_ids + @property + def book_value_map(self): + return {book_id:tuple(self.table.id_map[item_id] for item_id in item_ids) + for book_id, item_ids in self.table.book_col_map.iteritems()} + class IdentifiersField(ManyToManyField): def for_book(self, book_id, default_value=None): @@ -345,7 +350,7 @@ class IdentifiersField(ManyToManyField): ids = default_value return ids - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): 'Sort by identifier keys' ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} @@ -369,7 +374,7 @@ class AuthorsField(ManyToManyField): 'link' : self.table.alink_map[author_id], } - def category_sort_value(self, item_id, book_ids, language_field): + def category_sort_value(self, item_id, book_ids, lang_map): return self.table.asort_map[item_id] class FormatsField(ManyToManyField): @@ -393,24 +398,25 @@ class FormatsField(ManyToManyField): class SeriesField(ManyToOneField): - def sort_key_for_series(self, book_id, get_lang, series_sort_order): + def sort_key_for_series(self, book_id, lang_map, series_sort_order): sid = self.table.book_col_map.get(book_id, None) if sid is None: return self._default_sort_key + lang = lang_map.get(book_id, None) or None + if lang: + lang = lang[0] return self._sort_key(title_sort(self.table.id_map[sid], - order=series_sort_order, - lang=get_lang(book_id))) + order=series_sort_order, lang=lang)) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): sso = tweaks['title_series_sorting'] - return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id + return {book_id:self.sort_key_for_series(book_id, lang_map, sso) for book_id in all_book_ids} - def category_sort_value(self, item_id, book_ids, language_field): + def category_sort_value(self, item_id, book_ids, lang_map): lang = None tss = tweaks['title_series_sorting'] if tss != 'strictly_alphabetic': - lang_map = language_field.book_col_map c = Counter() for book_id in book_ids: From c7e8509e10b9a6d4ded0322b12ec3f903b136d14 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 17:28:14 +0530 Subject: [PATCH 23/24] Categories for identifiers --- src/calibre/db/cache.py | 6 ++++++ src/calibre/db/categories.py | 24 ++++++++++++++++++------ src/calibre/db/fields.py | 28 +++++++++++++++------------- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 65e2e614eb..47197aff3a 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -11,6 +11,7 @@ import os, traceback from collections import defaultdict from functools import wraps, partial +from calibre.db.categories import get_categories from calibre.db.locking import create_locks, RecordLock from calibre.db.fields import create_field from calibre.db.search import Search @@ -445,6 +446,11 @@ class Cache(object): return self._search_api(self, query, restriction, virtual_fields=virtual_fields) + @read_api + def get_categories(self, sort='name', book_ids=None, icon_map=None): + return get_categories(self, sort=sort, book_ids=book_ids, + icon_map=icon_map) + # }}} class SortKey(object): diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index 0def3b144e..f439cb5543 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -8,9 +8,11 @@ __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' from functools import partial +from operator import attrgetter from calibre.library.field_metadata import TagsIcons from calibre.utils.config_base import tweaks +from calibre.utils.icu import sort_key CATEGORY_SORTS = { 'name', 'popularity', 'rating' } @@ -52,7 +54,7 @@ class Tag(object): def find_categories(field_metadata): for category, cat in field_metadata.iteritems(): if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and - category not in {'news', 'formats'} and not cat.get('is_csp', False)): + category not in {'news', 'formats'}): yield (category, cat['is_multiple'].get('cache_to_list', None), False) elif (cat['datatype'] == 'composite' and cat['display'].get('make_category', False)): @@ -61,7 +63,7 @@ def find_categories(field_metadata): def create_tag_class(category, fm, icon_map): cat = fm[category] icon = None - tooltip = '(' + category + ')' + tooltip = None if category == 'identifiers' else ('(' + category + ')') label = fm.key_to_label(category) if icon_map: if not fm.is_custom_field(category): @@ -94,13 +96,23 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): fm = dbcache.field_metadata book_rating_map = dbcache.fields['rating'].book_value_map - lang_map = dbcache.fileds['languages'].book_value_map + lang_map = dbcache.fields['languages'].book_value_map categories = {} - book_ids = frozenset(book_ids) + book_ids = frozenset(book_ids) if book_ids else book_ids for category, is_multiple, is_composite in find_categories(fm): tag_class = create_tag_class(category, fm, icon_map) - categories[category] = dbcache.fields[category].get_categories( - tag_class, book_rating_map, sort, lang_map, book_ids) + cats = dbcache.fields[category].get_categories( + tag_class, book_rating_map, lang_map, book_ids) + if sort == 'popularity': + key=attrgetter('count') + elif sort == 'rating': + key=attrgetter('avg_rating') + else: + key=lambda x:sort_key(x.sort or x.name) + cats.sort(key=key) + categories[category] = cats + + return categories diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 6a6deacf10..13983ea5d5 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en' from threading import Lock from collections import defaultdict, Counter -from operator import attrgetter from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY from calibre.ebooks.metadata import title_sort @@ -41,10 +40,8 @@ class Field(object): self.is_multiple = (bool(self.metadata['is_multiple']) or self.name == 'formats') self.category_formatter = type(u'') - self.category_sort_reverse = False if dt == 'rating': self.category_formatter = lambda x:'\u2605'*int(x/2) - self.category_sort_reverse = True elif name == 'languages': self.category_formatter = calibre_langcode_to_name @@ -101,7 +98,7 @@ class Field(object): ''' raise NotImplementedError() - def get_categories(self, tag_class, book_rating_map, sort, lang_map, book_ids=None): + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): ans = [] if not self.is_many: return ans @@ -113,20 +110,13 @@ class Field(object): if item_book_ids: ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for book_id in item_book_ids) if r > 0) - avg = sum(ratings)/len(ratings) + avg = sum(ratings)/len(ratings) if ratings else 0 name = self.category_formatter(self.table.id_map[item_id]) sval = (self.category_sort_value(item_id, item_book_ids, lang_map) if special_sort else name) c = tag_class(name, id=item_id, sort=sval, avg=avg, id_set=item_book_ids, count=len(item_book_ids)) ans.append(c) - if sort == 'popularity': - key=attrgetter('count') - elif sort == 'rating': - key=attrgetter('avg_rating') - else: - key=lambda x:sort_key(x.sort or x.name) - ans.sort(key=key, reverse=self.category_sort_reverse) return ans class OneToOneField(Field): @@ -282,7 +272,7 @@ class ManyToOneField(Field): @property def book_value_map(self): return {book_id:self.table.id_map[item_id] for book_id, item_id in - self.book_col_map.iteritems()} + self.table.book_col_map.iteritems()} class ManyToManyField(Field): @@ -365,6 +355,18 @@ class IdentifiersField(ManyToManyField): if val: yield val, {book_id} + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): + ans = [] + + for id_key, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + name = id_key + c = tag_class(name, id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + return ans + class AuthorsField(ManyToManyField): def author_data(self, author_id): From dee27d1a844d043d3bb9fdb2810c6fc382235b09 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jan 2013 17:39:49 +0530 Subject: [PATCH 24/24] Implement formats category --- src/calibre/db/categories.py | 10 +++++----- src/calibre/db/fields.py | 14 ++++++++++++-- src/calibre/db/tests/base.py | 1 + src/calibre/db/tests/reading.py | 12 ++++++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py index f439cb5543..092b6da877 100644 --- a/src/calibre/db/categories.py +++ b/src/calibre/db/categories.py @@ -53,8 +53,8 @@ class Tag(object): def find_categories(field_metadata): for category, cat in field_metadata.iteritems(): - if (cat['is_category'] and cat['kind'] not in { 'user', 'search' } and - category not in {'news', 'formats'}): + if (cat['is_category'] and cat['kind'] not in {'user', 'search'} and + category != 'news'): yield (category, cat['is_multiple'].get('cache_to_list', None), False) elif (cat['datatype'] == 'composite' and cat['display'].get('make_category', False)): @@ -63,7 +63,7 @@ def find_categories(field_metadata): def create_tag_class(category, fm, icon_map): cat = fm[category] icon = None - tooltip = None if category == 'identifiers' else ('(' + category + ')') + tooltip = None if category in {'formats', 'identifiers'} else ('(' + category + ')') label = fm.key_to_label(category) if icon_map: if not fm.is_custom_field(category): @@ -72,7 +72,8 @@ def create_tag_class(category, fm, icon_map): else: icon = icon_map['custom:'] icon_map[category] = icon - is_editable = category not in { 'news', 'rating', 'languages' } + is_editable = category not in {'news', 'rating', 'languages', 'formats', + 'identifiers'} if (tweaks['categories_use_field_for_author_name'] == 'author_sort' and (category == 'authors' or @@ -87,7 +88,6 @@ def create_tag_class(category, fm, icon_map): tooltip=tooltip, is_editable=is_editable, category=category) - def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 13983ea5d5..4e73b5badf 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -362,8 +362,7 @@ class IdentifiersField(ManyToManyField): if book_ids is not None: item_book_ids = item_book_ids.intersection(book_ids) if item_book_ids: - name = id_key - c = tag_class(name, id_set=item_book_ids, count=len(item_book_ids)) + c = tag_class(id_key, id_set=item_book_ids, count=len(item_book_ids)) ans.append(c) return ans @@ -398,6 +397,17 @@ class FormatsField(ManyToManyField): for val, book_ids in val_map.iteritems(): yield val, book_ids + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): + ans = [] + + for fmt, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + c = tag_class(fmt, id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + return ans + class SeriesField(ManyToOneField): def sort_key_for_series(self, book_id, lang_map, series_sort_order): diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index 8e72721c4e..b626551576 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -42,6 +42,7 @@ class BaseTest(unittest.TestCase): if attr == 'format_metadata': continue # TODO: Not implemented yet attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) if attr == 'formats': + continue # TODO: Not implemented yet attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2)) self.assertEqual(attr1, attr2, '%s not the same: %r != %r'%(attr, attr1, attr2)) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 8183611f91..b1d4bd3142 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -241,6 +241,18 @@ class ReadingTest(BaseTest): # }}} + def test_get_categories(self): # {{{ + 'Check that get_categories() returns the same data for both backends' + from calibre.library.database2 import LibraryDatabase2 + old = LibraryDatabase2(self.library_path) + old_categories = old.get_categories() + cache = self.init_cache(self.library_path) + import pprint + pprint.pprint(old_categories) + pprint.pprint(cache.get_categories()) + + # }}} + def tests(): return unittest.TestLoader().loadTestsFromTestCase(ReadingTest)