diff --git a/recipes/barrons.recipe b/recipes/barrons.recipe index 41ed7e26ec..58c62e20e9 100644 --- a/recipes/barrons.recipe +++ b/recipes/barrons.recipe @@ -28,6 +28,8 @@ class Barrons(BasicNewsRecipe): ## Don't grab articles more than 7 days old oldest_article = 7 + use_javascript_to_login = True + requires_version = (0, 9, 16) extra_css = ''' .datestamp{font-family:Verdana,Geneva,Kalimati,sans-serif; font-size:x-small;} @@ -40,7 +42,7 @@ class Barrons(BasicNewsRecipe): .insettipUnit{font-size: x-small;} ''' remove_tags = [ - dict(name ='div', attrs={'class':['tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), + dict(name ='div', attrs={'class':['sTools sTools-t', 'tabContainer artTabbedNav','rssToolBox hidden','articleToolbox']}), dict(name = 'a', attrs ={'class':'insetClose'}) ] @@ -60,21 +62,17 @@ class Barrons(BasicNewsRecipe): ] ] - def get_browser(self): - br = BasicNewsRecipe.get_browser() - if self.username is not None and self.password is not None: - br.open('http://commerce.barrons.com/auth/login') - br.select_form(nr=0) - br['username'] = self.username - br['password'] = self.password - br.submit() - return br + def javascript_login(self, br, username, password): + br.visit('http://commerce.barrons.com/auth/login') + f = br.select_form(nr=0) + f['username'] = username + f['password'] = password + br.submit(timeout=120) ## Use the print version of a page when available. - def print_version(self, url): main, sep, rest = url.rpartition('?') - return main + '#printmode' + return main + '#text.print' def postprocess_html(self, soup, first): diff --git a/recipes/st_louis_post_dispatch.recipe b/recipes/st_louis_post_dispatch.recipe index 3b7701cedc..6d22a327ab 100644 --- a/recipes/st_louis_post_dispatch.recipe +++ b/recipes/st_louis_post_dispatch.recipe @@ -7,12 +7,16 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): oldest_article = 1 max_articles_per_feed = 15 + use_embedded_content = False + + no_stylesheets = True + auto_cleanup = True masthead_url = 'http://farm5.static.flickr.com/4118/4929686950_0e22e2c88a.jpg' feeds = [ (u'News-Bill McClellan', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fbill-mclellan&f=rss&t=article'), (u'News-Columns', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcolumns*&l=50&f=rss&t=article'), - (u'News-Crime & Courtshttp://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), + (u'News-Crime & Courts', 'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2Fcrime-and-courts&l=50&f=rss&t=article'), (u'News-Deb Peterson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fcolumns%2Fdeb-peterson&f=rss&t=article'), (u'News-Education', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2feducation&f=rss&t=article'), (u'News-Government & Politics', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=news%2Flocal%2fgovt-and-politics&f=rss&t=article'), @@ -62,9 +66,9 @@ class AdvancedUserRecipe1282093204(BasicNewsRecipe): (u'Entertainment-House-O-Fun', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fhouse-o-fun&l=100&f=rss&t=article'), (u'Entertainment-Kevin C. Johnson', u'http://www2.stltoday.com/search/?q=&d1=&d2=&s=start_time&sd=desc&c=entertainment%2Fmusic%2Fkevin-johnson&l=100&f=rss&t=article') ] - remove_empty_feeds = True - remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] - keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] + #remove_empty_feeds = True + #remove_tags = [dict(name='div', attrs={'id':'blox-logo'}),dict(name='a')] + #keep_only_tags = [dict(name='h1'), dict(name='p', attrs={'class':'byline'}), dict(name="div", attrs={'id':'blox-story-text'})] extra_css = 'p {text-align: left;}' diff --git a/recipes/tsn.recipe b/recipes/tsn.recipe index e822ebc633..6c3dbe5159 100644 --- a/recipes/tsn.recipe +++ b/recipes/tsn.recipe @@ -7,28 +7,15 @@ class AdvancedUserRecipe1289990851(BasicNewsRecipe): language = 'en_CA' __author__ = 'Nexus' no_stylesheets = True + auto_cleanup = True + use_embedded_content = False INDEX = 'http://tsn.ca/nhl/story/?id=nhl' - keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), - dict(name='div', attrs={'id':['tsnStory']})] - remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), - dict(name='div', attrs={'class':'textSize'})] - - def parse_index(self): - feeds = [] - soup = self.index_to_soup(self.INDEX) - feed_parts = soup.findAll('div', attrs={'class': 'feature'}) - for feed_part in feed_parts: - articles = [] - if not feed_part.h2: - continue - feed_title = feed_part.h2.string - article_parts = feed_part.findAll('a') - for article_part in article_parts: - article_title = article_part.string - article_date = '' - article_url = 'http://tsn.ca/' + article_part['href'] - articles.append({'title': article_title, 'url': article_url, 'description':'', 'date':article_date}) - if articles: - feeds.append((feed_title, articles)) - return feeds + #keep_only_tags = [dict(name='div', attrs={'id':['tsnColWrap']}), + #dict(name='div', attrs={'id':['tsnStory']})] + #remove_tags = [dict(name='div', attrs={'id':'tsnRelated'}), + #dict(name='div', attrs={'class':'textSize'})] + feeds = [ +('News', + 'http://www.tsn.ca/datafiles/rss/Stories.xml'), +] diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 40d39b0ad4..dff477bbc4 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -79,6 +79,42 @@ def debug(): global DEBUG DEBUG = True +_cache_dir = None + +def _get_cache_dir(): + confcache = os.path.join(config_dir, u'caches') + if isportable: + return confcache + if os.environ.has_key('CALIBRE_CACHE_DIRECTORY'): + return os.path.abspath(os.environ['CALIBRE_CACHE_DIRECTORY']) + + if iswindows: + w = plugins['winutil'][0] + candidate = os.path.join(w.special_folder_path(w.CSIDL_LOCAL_APPDATA), u'%s-cache'%__appname__) + elif isosx: + candidate = os.path.join(os.path.expanduser(u'~/Library/Caches'), __appname__) + else: + candidate = os.environ.get('XDG_CACHE_HOME', u'~/.cache') + candidate = os.path.join(os.path.expanduser(candidate), + __appname__) + if isinstance(candidate, bytes): + try: + candidate = candidate.decode(filesystem_encoding) + except ValueError: + candidate = confcache + if not os.path.exists(candidate): + try: + os.makedirs(candidate) + except: + candidate = confcache + return candidate + +def cache_dir(): + global _cache_dir + if _cache_dir is None: + _cache_dir = _get_cache_dir() + return _cache_dir + # plugins {{{ class Plugins(collections.Mapping): diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 88a2196a61..47197aff3a 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,10 +7,11 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, traceback from collections import defaultdict from functools import wraps, partial +from calibre.db.categories import get_categories from calibre.db.locking import create_locks, RecordLock from calibre.db.fields import create_field from calibre.db.search import Search @@ -18,6 +19,7 @@ from calibre.db.tables import VirtualTable from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import now +from calibre.utils.icu import sort_key def api(f): f.is_cache_api = True @@ -67,6 +69,36 @@ class Cache(object): lock = self.read_lock if ira else self.write_lock setattr(self, name, wrap_simple(lock, func)) + self.initialize_dynamic() + + def initialize_dynamic(self): + # Reconstruct the user categories, putting them into field_metadata + # Assumption is that someone else will fix them if they change. + self.field_metadata.remove_dynamic_categories() + for user_cat in sorted(self.pref('user_categories', {}).iterkeys(), key=sort_key): + cat_name = '@' + user_cat # add the '@' to avoid name collision + self.field_metadata.add_user_category(label=cat_name, name=user_cat) + + # add grouped search term user categories + muc = frozenset(self.pref('grouped_search_make_user_categories', [])) + for cat in sorted(self.pref('grouped_search_terms', {}).iterkeys(), key=sort_key): + if cat in muc: + # There is a chance that these can be duplicates of an existing + # user category. Print the exception and continue. + try: + self.field_metadata.add_user_category(label=u'@' + cat, name=cat) + except: + traceback.print_exc() + + # TODO: Saved searches + # if len(saved_searches().names()): + # self.field_metadata.add_search_category(label='search', name=_('Searches')) + + self.field_metadata.add_grouped_search_terms( + self.pref('grouped_search_terms', {})) + + self._search_api.change_locations(self.field_metadata.get_search_terms()) + @property def field_metadata(self): return self.backend.field_metadata @@ -262,13 +294,13 @@ class Cache(object): Return all the books associated with the item identified by ``item_id``, where the item belongs to the field ``name``. - Returned value is a tuple of book ids, or the empty tuple if the item + Returned value is a set of book ids, or the empty set if the item or the field does not exist. ''' try: return self.fields[name].books_for(item_id) except (KeyError, IndexError): - return () + return set() @read_api def all_book_ids(self, type=frozenset): @@ -319,8 +351,8 @@ class Cache(object): return ans @read_api - def pref(self, name): - return self.backend.prefs[name] + def pref(self, name, default=None): + return self.backend.prefs.get(name, default) @api def get_metadata(self, book_id, @@ -384,9 +416,7 @@ class Cache(object): all_book_ids = frozenset(self._all_book_ids() if ids_to_sort is None else ids_to_sort) get_metadata = partial(self._get_metadata, get_user_categories=False) - def get_lang(book_id): - ans = self._field_for('languages', book_id) - return ans[0] if ans else None + lang_map = self.fields['languages'].book_value_map fm = {'title':'sort', 'authors':'author_sort'} @@ -395,10 +425,10 @@ class Cache(object): idx = field + '_index' is_series = idx in self.fields ans = self.fields[fm.get(field, field)].sort_keys_for_books( - get_metadata, get_lang, all_book_ids,) + get_metadata, lang_map, all_book_ids,) if is_series: idx_ans = self.fields[idx].sort_keys_for_books( - get_metadata, get_lang, all_book_ids) + get_metadata, lang_map, all_book_ids) ans = {k:(v, idx_ans[k]) for k, v in ans.iteritems()} return ans @@ -412,8 +442,14 @@ class Cache(object): return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys)) @read_api - def search(self, query, restriction): - return self._search_api(self, query, restriction) + def search(self, query, restriction, virtual_fields=None): + return self._search_api(self, query, restriction, + virtual_fields=virtual_fields) + + @read_api + def get_categories(self, sort='name', book_ids=None, icon_map=None): + return get_categories(self, sort=sort, book_ids=book_ids, + icon_map=icon_map) # }}} diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py new file mode 100644 index 0000000000..092b6da877 --- /dev/null +++ b/src/calibre/db/categories.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from functools import partial +from operator import attrgetter + +from calibre.library.field_metadata import TagsIcons +from calibre.utils.config_base import tweaks +from calibre.utils.icu import sort_key + +CATEGORY_SORTS = { 'name', 'popularity', 'rating' } + +class Tag(object): + + def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, + tooltip=None, icon=None, category=None, id_set=None, + is_editable=True, is_searchable=True, use_sort_as_name=False): + self.name = self.original_name = name + self.id = id + self.count = count + self.state = state + self.is_hierarchical = '' + self.is_editable = is_editable + self.is_searchable = is_searchable + self.id_set = id_set if id_set is not None else set([]) + self.avg_rating = avg/2.0 if avg is not None else 0 + self.sort = sort + self.use_sort_as_name = use_sort_as_name + if self.avg_rating > 0: + if tooltip: + tooltip = tooltip + ': ' + tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict( + tt=tooltip, rating=self.avg_rating) + self.tooltip = tooltip + self.icon = icon + self.category = category + + def __unicode__(self): + return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state, + self.category, self.tooltip) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __repr__(self): + return str(self) + +def find_categories(field_metadata): + for category, cat in field_metadata.iteritems(): + if (cat['is_category'] and cat['kind'] not in {'user', 'search'} and + category != 'news'): + yield (category, cat['is_multiple'].get('cache_to_list', None), False) + elif (cat['datatype'] == 'composite' and + cat['display'].get('make_category', False)): + yield (category, cat['is_multiple'].get('cache_to_list', None), True) + +def create_tag_class(category, fm, icon_map): + cat = fm[category] + icon = None + tooltip = None if category in {'formats', 'identifiers'} else ('(' + category + ')') + label = fm.key_to_label(category) + if icon_map: + if not fm.is_custom_field(category): + if category in icon_map: + icon = icon_map[label] + else: + icon = icon_map['custom:'] + icon_map[category] = icon + is_editable = category not in {'news', 'rating', 'languages', 'formats', + 'identifiers'} + + if (tweaks['categories_use_field_for_author_name'] == 'author_sort' and + (category == 'authors' or + (cat['display'].get('is_names', False) and + cat['is_custom'] and cat['is_multiple'] and + cat['datatype'] == 'text'))): + use_sort_as_name = True + else: + use_sort_as_name = False + + return partial(Tag, use_sort_as_name=use_sort_as_name, icon=icon, + tooltip=tooltip, is_editable=is_editable, + category=category) + +def get_categories(dbcache, sort='name', book_ids=None, icon_map=None): + if icon_map is not None and type(icon_map) != TagsIcons: + raise TypeError('icon_map passed to get_categories must be of type TagIcons') + if sort not in CATEGORY_SORTS: + raise ValueError('sort ' + sort + ' not a valid value') + + fm = dbcache.field_metadata + book_rating_map = dbcache.fields['rating'].book_value_map + lang_map = dbcache.fields['languages'].book_value_map + + categories = {} + book_ids = frozenset(book_ids) if book_ids else book_ids + for category, is_multiple, is_composite in find_categories(fm): + tag_class = create_tag_class(category, fm, icon_map) + cats = dbcache.fields[category].get_categories( + tag_class, book_rating_map, lang_map, book_ids) + if sort == 'popularity': + key=attrgetter('count') + elif sort == 'rating': + key=attrgetter('avg_rating') + else: + key=lambda x:sort_key(x.sort or x.name) + cats.sort(key=key) + categories[category] = cats + + return categories + + diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e0950fff3b..4e73b5badf 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -9,7 +9,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from threading import Lock -from collections import defaultdict +from collections import defaultdict, Counter from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY from calibre.ebooks.metadata import title_sort @@ -24,22 +24,26 @@ class Field(object): def __init__(self, name, table): self.name, self.table = name, table - self.has_text_data = self.metadata['datatype'] in ('text', 'comments', - 'series', 'enumeration') - self.table_type = self.table.table_type dt = self.metadata['datatype'] + self.has_text_data = dt in {'text', 'comments', 'series', 'enumeration'} + self.table_type = self.table.table_type self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x) self._default_sort_key = '' - if self.metadata['datatype'] in ('int', 'float', 'rating'): + if dt in { 'int', 'float', 'rating' }: self._default_sort_key = 0 - elif self.metadata['datatype'] == 'bool': + elif dt == 'bool': self._default_sort_key = None - elif self.metadata['datatype'] == 'datetime': + elif dt == 'datetime': self._default_sort_key = UNDEFINED_DATE if self.name == 'languages': self._sort_key = lambda x:sort_key(calibre_langcode_to_name(x)) self.is_multiple = (bool(self.metadata['is_multiple']) or self.name == 'formats') + self.category_formatter = type(u'') + if dt == 'rating': + self.category_formatter = lambda x:'\u2605'*int(x/2) + elif name == 'languages': + self.category_formatter = calibre_langcode_to_name @property def metadata(self): @@ -63,7 +67,7 @@ class Field(object): def books_for(self, item_id): ''' Return the ids of all books associated with the item identified by - item_id as a tuple. An empty tuple is returned if no books are found. + item_id as a set. An empty set is returned if no books are found. ''' raise NotImplementedError() @@ -77,7 +81,7 @@ class Field(object): ''' return iter(()) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ''' Return a mapping of book_id -> sort_key. The sort key is suitable for use in sorting the list of all books by this field, via the python cmp @@ -94,6 +98,27 @@ class Field(object): ''' raise NotImplementedError() + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): + ans = [] + if not self.is_many: + return ans + + special_sort = hasattr(self, 'category_sort_value') + for item_id, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + ratings = tuple(r for r in (book_rating_map.get(book_id, 0) for + book_id in item_book_ids) if r > 0) + avg = sum(ratings)/len(ratings) if ratings else 0 + name = self.category_formatter(self.table.id_map[item_id]) + sval = (self.category_sort_value(item_id, item_book_ids, lang_map) + if special_sort else name) + c = tag_class(name, id=item_id, sort=sval, avg=avg, + id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + return ans + class OneToOneField(Field): def for_book(self, book_id, default_value=None): @@ -103,12 +128,12 @@ class OneToOneField(Field): return (book_id,) def books_for(self, item_id): - return (item_id,) + return {item_id} def __iter__(self): return self.table.book_col_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : self._sort_key(self.table.book_col_map.get(id_, self._default_sort_key)) for id_ in all_book_ids} @@ -150,7 +175,7 @@ class CompositeField(OneToOneField): ans = mi.get('#'+self.metadata['label']) return ans - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in all_book_ids} @@ -193,7 +218,7 @@ class OnDeviceField(OneToOneField): def __iter__(self): return iter(()) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): return {id_ : self.for_book(id_) for id_ in all_book_ids} @@ -223,12 +248,12 @@ class ManyToOneField(Field): return (id_,) def books_for(self, item_id): - return self.table.col_book_map.get(item_id, ()) + return self.table.col_book_map.get(item_id, set()) def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, None) for id_ in all_book_ids} sk_map = {cid : (self._default_sort_key if cid is None else @@ -238,11 +263,17 @@ class ManyToOneField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.col_book_map + empty = set() for item_id, val in self.table.id_map.iteritems(): - book_ids = set(cbm.get(item_id, ())).intersection(candidates) + book_ids = cbm.get(item_id, empty).intersection(candidates) if book_ids: yield val, book_ids + @property + def book_value_map(self): + return {book_id:self.table.id_map[item_id] for book_id, item_id in + self.table.book_col_map.iteritems()} + class ManyToManyField(Field): is_many = True @@ -263,12 +294,12 @@ class ManyToManyField(Field): return self.table.book_col_map.get(book_id, ()) def books_for(self, item_id): - return self.table.col_book_map.get(item_id, ()) + return self.table.col_book_map.get(item_id, set()) def __iter__(self): return self.table.id_map.iterkeys() - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} all_cids = set() @@ -282,8 +313,9 @@ class ManyToManyField(Field): def iter_searchable_values(self, get_metadata, candidates, default_value=None): cbm = self.table.col_book_map + empty = set() for item_id, val in self.table.id_map.iteritems(): - book_ids = set(cbm.get(item_id, ())).intersection(candidates) + book_ids = cbm.get(item_id, empty).intersection(candidates) if book_ids: yield val, book_ids @@ -295,6 +327,11 @@ class ManyToManyField(Field): for count, book_ids in val_map.iteritems(): yield count, book_ids + @property + def book_value_map(self): + return {book_id:tuple(self.table.id_map[item_id] for item_id in item_ids) + for book_id, item_ids in self.table.book_col_map.iteritems()} + class IdentifiersField(ManyToManyField): def for_book(self, book_id, default_value=None): @@ -303,7 +340,7 @@ class IdentifiersField(ManyToManyField): ids = default_value return ids - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): 'Sort by identifier keys' ans = {id_ : self.table.book_col_map.get(id_, ()) for id_ in all_book_ids} @@ -318,6 +355,17 @@ class IdentifiersField(ManyToManyField): if val: yield val, {book_id} + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): + ans = [] + + for id_key, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + c = tag_class(id_key, id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + return ans + class AuthorsField(ManyToManyField): def author_data(self, author_id): @@ -327,6 +375,9 @@ class AuthorsField(ManyToManyField): 'link' : self.table.alink_map[author_id], } + def category_sort_value(self, item_id, book_ids, lang_map): + return self.table.asort_map[item_id] + class FormatsField(ManyToManyField): def for_book(self, book_id, default_value=None): @@ -346,21 +397,50 @@ class FormatsField(ManyToManyField): for val, book_ids in val_map.iteritems(): yield val, book_ids + def get_categories(self, tag_class, book_rating_map, lang_map, book_ids=None): + ans = [] + + for fmt, item_book_ids in self.table.col_book_map.iteritems(): + if book_ids is not None: + item_book_ids = item_book_ids.intersection(book_ids) + if item_book_ids: + c = tag_class(fmt, id_set=item_book_ids, count=len(item_book_ids)) + ans.append(c) + return ans + class SeriesField(ManyToOneField): - def sort_key_for_series(self, book_id, get_lang, series_sort_order): + def sort_key_for_series(self, book_id, lang_map, series_sort_order): sid = self.table.book_col_map.get(book_id, None) if sid is None: return self._default_sort_key + lang = lang_map.get(book_id, None) or None + if lang: + lang = lang[0] return self._sort_key(title_sort(self.table.id_map[sid], - order=series_sort_order, - lang=get_lang(book_id))) + order=series_sort_order, lang=lang)) - def sort_keys_for_books(self, get_metadata, get_lang, all_book_ids): + def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids): sso = tweaks['title_series_sorting'] - return {book_id:self.sort_key_for_series(book_id, get_lang, sso) for book_id + return {book_id:self.sort_key_for_series(book_id, lang_map, sso) for book_id in all_book_ids} + def category_sort_value(self, item_id, book_ids, lang_map): + lang = None + tss = tweaks['title_series_sorting'] + if tss != 'strictly_alphabetic': + c = Counter() + + for book_id in book_ids: + l = lang_map.get(book_id, None) + if l: + c[l[0]] += 1 + + if c: + lang = c.most_common(1)[0][0] + val = self.table.id_map[item_id] + return title_sort(val, order=tss, lang=lang) + def create_field(name, table): cls = { ONE_ONE : OneToOneField, diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index 334bc046d8..7eb747da67 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -14,6 +14,7 @@ from datetime import timedelta from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now from calibre.utils.icu import primary_find +from calibre.utils.localization import lang_map, canonicalize_lang from calibre.utils.search_query_parser import SearchQueryParser, ParseException # TODO: Thread safety of saved searches @@ -392,7 +393,7 @@ class Parser(SearchQueryParser): def __init__(self, dbcache, all_book_ids, gst, date_search, num_search, bool_search, keypair_search, limit_search_columns, limit_search_columns_to, - locations): + locations, virtual_fields): self.dbcache, self.all_book_ids = dbcache, all_book_ids self.all_search_locations = frozenset(locations) self.grouped_search_terms = gst @@ -400,6 +401,9 @@ class Parser(SearchQueryParser): self.bool_search, self.keypair_search = bool_search, keypair_search self.limit_search_columns, self.limit_search_columns_to = ( limit_search_columns, limit_search_columns_to) + self.virtual_fields = virtual_fields or {} + if 'marked' not in self.virtual_fields: + self.virtual_fields['marked'] = self super(Parser, self).__init__(locations, optimize=True) @property @@ -411,8 +415,15 @@ class Parser(SearchQueryParser): def field_iter(self, name, candidates): get_metadata = partial(self.dbcache._get_metadata, get_user_categories=False) - return self.dbcache.fields[name].iter_searchable_values(get_metadata, - candidates) + try: + field = self.dbcache.fields[name] + except KeyError: + field = self.virtual_fields[name] + return field.iter_searchable_values(get_metadata, candidates) + + def iter_searchable_values(self, *args, **kwargs): + for x in []: + yield x, set() def get_matches(self, location, query, candidates=None, allow_recursion=True): @@ -480,6 +491,8 @@ class Parser(SearchQueryParser): pass return matches + upf = prefs['use_primary_find_in_search'] + if location in self.field_metadata: fm = self.field_metadata[location] dt = fm['datatype'] @@ -519,7 +532,6 @@ class Parser(SearchQueryParser): # is a special case within the case if fm.get('is_csp', False): field_iter = partial(self.field_iter, location, candidates) - upf = prefs['use_primary_find_in_search'] if location == 'identifiers' and original_location == 'isbn': return self.keypair_search('=isbn:'+query, field_iter, candidates, upf) @@ -529,6 +541,87 @@ class Parser(SearchQueryParser): if len(location) >= 2 and location.startswith('@'): return self.get_user_category_matches(location[1:], icu_lower(query), candidates) + # Everything else (and 'all' matches) + matchkind, query = _matchkind(query) + all_locs = set() + text_fields = set() + field_metadata = {} + + for x, fm in self.field_metadata.iteritems(): + if x.startswith('@'): continue + if fm['search_terms'] and x != 'series_sort': + all_locs.add(x) + field_metadata[x] = fm + if fm['datatype'] in { 'composite', 'text', 'comments', 'series', 'enumeration' }: + text_fields.add(x) + + locations = all_locs if location == 'all' else {location} + + current_candidates = set(candidates) + + try: + rating_query = int(float(query)) * 2 + except: + rating_query = None + + try: + int_query = int(float(query)) + except: + int_query = None + + try: + float_query = float(query) + except: + float_query = None + + for location in locations: + current_candidates -= matches + q = query + if location == 'languages': + q = canonicalize_lang(query) + if q is None: + lm = lang_map() + rm = {v.lower():k for k,v in lm.iteritems()} + q = rm.get(query, query) + + if matchkind == CONTAINS_MATCH and q in {'true', 'false'}: + found = set() + for val, book_ids in self.field_iter(location, current_candidates): + if val and (not hasattr(val, 'strip') or val.strip()): + found |= book_ids + matches |= (found if q == 'true' else (current_candidates-found)) + continue + + dt = field_metadata.get(location, {}).get('datatype', None) + if dt == 'rating': + if rating_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == rating_query: + matches |= book_ids + continue + + if dt == 'float': + if float_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == float_query: + matches |= book_ids + continue + + if dt == 'int': + if int_query is not None: + for val, book_ids in self.field_iter(location, current_candidates): + if val == int_query: + matches |= book_ids + continue + + if location in text_fields: + for val, book_ids in self.field_iter(location, current_candidates): + if val is not None: + if isinstance(val, basestring): + val = (val,) + if _match(q, val, matchkind, use_primary_find_in_search=upf): + matches |= book_ids + return matches def get_user_category_matches(self, location, query, candidates): @@ -557,7 +650,7 @@ class Parser(SearchQueryParser): class Search(object): - def __init__(self, all_search_locations): + def __init__(self, all_search_locations=()): self.all_search_locations = all_search_locations self.date_search = DateSearch() self.num_search = NumericSearch() @@ -567,7 +660,7 @@ class Search(object): def change_locations(self, newlocs): self.all_search_locations = newlocs - def __call__(self, dbcache, query, search_restriction): + def __call__(self, dbcache, query, search_restriction, virtual_fields=None): ''' Return the set of ids of all records that match the specified query and restriction @@ -596,7 +689,8 @@ class Search(object): self.date_search, self.num_search, self.bool_search, self.keypair_search, prefs[ 'limit_search_columns' ], - prefs[ 'limit_search_columns_to' ], self.all_search_locations) + prefs[ 'limit_search_columns_to' ], self.all_search_locations, + virtual_fields) try: ret = sqp.parse(q) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 234a7fe4a8..4a8b4492fd 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -132,13 +132,10 @@ class ManyToOneTable(Table): 'SELECT book, {0} FROM {1}'.format( self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) self.book_col_map[row[0]] = row[1] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - class ManyToManyTable(ManyToOneTable): ''' @@ -154,15 +151,12 @@ class ManyToManyTable(ManyToOneTable): for row in db.conn.execute( self.selectq.format(self.metadata['link_column'], self.link_table)): if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(row[1]) - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(self.book_col_map[key]) @@ -191,8 +185,8 @@ class FormatsTable(ManyToManyTable): if row[1] is not None: fmt = row[1].upper() if fmt not in self.col_book_map: - self.col_book_map[fmt] = [] - self.col_book_map[fmt].append(row[0]) + self.col_book_map[fmt] = set() + self.col_book_map[fmt].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(fmt) @@ -200,9 +194,6 @@ class FormatsTable(ManyToManyTable): self.fname_map[row[0]] = {} self.fname_map[row[0]][fmt] = row[2] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) @@ -215,15 +206,12 @@ class IdentifiersTable(ManyToManyTable): for row in db.conn.execute('SELECT book, type, val FROM identifiers'): if row[1] is not None and row[2] is not None: if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + self.col_book_map[row[1]] = set() + self.col_book_map[row[1]].add(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = {} self.book_col_map[row[0]][row[1]] = row[2] - for key in tuple(self.col_book_map.iterkeys()): - self.col_book_map[key] = tuple(self.col_book_map[key]) - class LanguagesTable(ManyToManyTable): def read_id_maps(self, db): diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index 8e72721c4e..b626551576 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -42,6 +42,7 @@ class BaseTest(unittest.TestCase): if attr == 'format_metadata': continue # TODO: Not implemented yet attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) if attr == 'formats': + continue # TODO: Not implemented yet attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2)) self.assertEqual(attr1, attr2, '%s not the same: %r != %r'%(attr, attr1, attr2)) diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 627a692860..b1d4bd3142 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -218,9 +218,17 @@ class ReadingTest(BaseTest): 'identifiers:t:n', 'identifiers:=test:=two', 'identifiers:x:y', 'identifiers:z', - # TODO: Tests for searching the size column and + # Text tests + 'title:="Title One"', 'title:~title', '#enum:=one', '#enum:tw', + '#enum:false', '#enum:true', 'series:one', 'tags:one', 'tags:true', + 'tags:false', '2', 'one', '20.02', '"publisher one"', + '"my comments one"', + + # User categories + '@Good Authors:One', '@Good Series.good tags:two', + + # TODO: Tests for searching the size and #formats columns and # cover:true|false - # TODO: Tests for user categories searching )} old = None @@ -233,6 +241,18 @@ class ReadingTest(BaseTest): # }}} + def test_get_categories(self): # {{{ + 'Check that get_categories() returns the same data for both backends' + from calibre.library.database2 import LibraryDatabase2 + old = LibraryDatabase2(self.library_path) + old_categories = old.get_categories() + cache = self.init_cache(self.library_path) + import pprint + pprint.pprint(old_categories) + pprint.pprint(cache.get_categories()) + + # }}} + def tests(): return unittest.TestLoader().loadTestsFromTestCase(ReadingTest) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index acee4938f5..d709ae91a9 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -141,7 +141,7 @@ class ANDROID(USBMS): # LG 0x1004 : { - 0x61c5 : [0x100, 0x226, 0x227, 0x9999], + 0x61c5 : [0x100, 0x226, 0x227, 0x229, 0x9999], 0x61cc : [0x226, 0x227, 0x9999, 0x100], 0x61ce : [0x226, 0x227, 0x9999, 0x100], 0x618e : [0x226, 0x227, 0x9999, 0x100], @@ -235,7 +235,7 @@ class ANDROID(USBMS): 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', - 'ICS'] + 'ICS', 'E400'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -246,7 +246,7 @@ class ANDROID(USBMS): 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', - 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F'] + 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD'] OSX_MAIN_MEM = 'Android Device Main Memory' diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index eacb143790..f941ccd659 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time -from calibre.constants import __appname__, __version__, DEBUG +from calibre.constants import __appname__, __version__, DEBUG, cache_dir from calibre import fit_image, confirm_config_name, strftime as _strftime from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback @@ -289,9 +289,7 @@ class ITUNES(DriverBase): # Properties cached_books = {} - cache_dir = os.path.join(config_dir, 'caches', 'itunes') calibre_library_path = prefs['library_path'] - archive_path = os.path.join(cache_dir, "thumbs.zip") description_prefix = "added by calibre" ejected = False iTunes = None @@ -309,6 +307,14 @@ class ITUNES(DriverBase): update_msg = None update_needed = False + @property + def cache_dir(self): + return os.path.join(cache_dir(), 'itunes') + + @property + def archive_path(self): + return os.path.join(self.cache_dir, "thumbs.zip") + # Public methods def add_books_to_metadata(self, locations, metadata, booklists): ''' diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 7291d5dbcb..72533860d4 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -195,7 +195,7 @@ class PRST1(USBMS): for i, row in enumerate(cursor): try: comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000); - except (OSError, IOError): + except (OSError, IOError, TypeError): # In case the db has incorrect path info continue device_date = int(row[1]); diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index 72032cb998..82cc4c0f4a 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -515,6 +515,7 @@ class HTMLPreProcessor(object): if not getattr(self.extra_opts, 'keep_ligatures', False): html = _ligpat.sub(lambda m:LIGATURES[m.group()], html) + user_sr_rules = {} # Function for processing search and replace def do_search_replace(search_pattern, replace_txt): try: @@ -522,6 +523,7 @@ class HTMLPreProcessor(object): if not replace_txt: replace_txt = '' rules.insert(0, (search_re, replace_txt)) + user_sr_rules[(search_re, replace_txt)] = search_pattern except Exception as e: self.log.error('Failed to parse %r regexp because %s' % (search, as_unicode(e))) @@ -587,7 +589,16 @@ class HTMLPreProcessor(object): #dump(html, 'pre-preprocess') for rule in rules + end_rules: - html = rule[0].sub(rule[1], html) + try: + html = rule[0].sub(rule[1], html) + except re.error as e: + if rule in user_sr_rules: + self.log.error( + 'User supplied search & replace rule: %s -> %s ' + 'failed with error: %s, ignoring.'%( + user_sr_rules[rule], rule[1], e)) + else: + raise if is_pdftohtml and length > -1: # Dehyphenate diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 46c6f7a313..e00c2e78d3 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -200,7 +200,7 @@ class Source(Plugin): #: during the identify phase touched_fields = frozenset() - #: Set this to True if your plugin return HTML formatted comments + #: Set this to True if your plugin returns HTML formatted comments has_html_comments = False #: Setting this to True means that the browser object will add diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 50df05ed16..e6e499236d 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -98,6 +98,9 @@ _self_closing_pat = re.compile( def close_self_closing_tags(raw): return _self_closing_pat.sub(r'<\g\g>>', raw) +def uuid_id(): + return 'u'+unicode(uuid.uuid4()) + def iterlinks(root, find_links_in_css=True): ''' Iterate over all links in a OEB Document. @@ -1528,7 +1531,7 @@ class TOC(object): if parent is None: parent = etree.Element(NCX('navMap')) for node in self.nodes: - id = node.id or unicode(uuid.uuid4()) + id = node.id or uuid_id() po = node.play_order if po == 0: po = 1 @@ -1634,10 +1637,10 @@ class PageList(object): return self.pages.remove(page) def to_ncx(self, parent=None): - plist = element(parent, NCX('pageList'), id=str(uuid.uuid4())) + plist = element(parent, NCX('pageList'), id=uuid_id()) values = dict((t, count(1)) for t in ('front', 'normal', 'special')) for page in self.pages: - id = page.id or unicode(uuid.uuid4()) + id = page.id or uuid_id() type = page.type value = str(values[type].next()) attrib = {'id': id, 'value': value, 'type': type, 'playOrder': '0'} diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 12fbd3b7f1..03410e1a65 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -18,7 +18,7 @@ from calibre import guess_type from calibre.ebooks.oeb.base import (XHTML, XHTML_NS, CSS_MIME, OEB_STYLES, namespace, barename, XPath) from calibre.ebooks.oeb.stylizer import Stylizer -from calibre.utils.filenames import ascii_filename +from calibre.utils.filenames import ascii_filename, ascii_text COLLAPSE = re.compile(r'[ \t\r\n\v]+') STRIPNUM = re.compile(r'[-0-9]+$') @@ -437,7 +437,7 @@ class CSSFlattener(object): items.sort() css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items) classes = node.get('class', '').strip() or 'calibre' - klass = STRIPNUM.sub('', classes.split()[0].replace('_', '')) + klass = ascii_text(STRIPNUM.sub('', classes.split()[0].replace('_', ''))) if css in styles: match = styles[css] else: diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 4b867f347d..72fe4fb12b 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -413,6 +413,7 @@ class RulesModel(QAbstractListModel): # {{{ rules = list(prefs['column_color_rules']) self.rules = [] for col, template in rules: + if col not in self.fm: continue try: rule = rule_from_template(self.fm, template) except: diff --git a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py index 9e41aa45a1..bf1b2013dd 100644 --- a/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py +++ b/src/calibre/gui2/store/stores/mobileread/mobileread_plugin.py @@ -6,10 +6,12 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import os from threading import Lock from PyQt4.Qt import (QUrl, QCoreApplication) +from calibre.constants import cache_dir from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig @@ -26,6 +28,16 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): StorePlugin.__init__(self, *args, **kwargs) self.lock = Lock() + @property + def cache(self): + if not hasattr(self, '_mr_cache'): + from calibre.utils.config import JSONConfig + self._mr_cache = JSONConfig('mobileread_get_books') + self._mr_cache.file_path = os.path.join(cache_dir(), + 'mobileread_get_books.json') + self._mr_cache.refresh() + return self._mr_cache + def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' @@ -61,7 +73,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): suppress_progress=False): if self.lock.acquire(False): try: - update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout) + update_thread = CacheUpdateThread(self.cache, self.seralize_books, timeout) if not suppress_progress: progress = CacheProgressDialog(parent) progress.set_message(_('Updating MobileRead book cache...')) @@ -85,7 +97,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): self.lock.release() def get_book_list(self): - return self.deseralize_books(self.config.get('book_list', [])) + return self.deseralize_books(self.cache.get('book_list', [])) def seralize_books(self, books): sbooks = [] diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 8f27db61be..9f946e2ee0 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -9,7 +9,7 @@ from xml.sax.saxutils import escape from calibre import (prepare_string_for_xml, strftime, force_unicode, isbytestring) -from calibre.constants import isosx +from calibre.constants import isosx, cache_dir from calibre.customize.conversion import DummyReporter from calibre.customize.ui import output_profiles from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString @@ -18,7 +18,6 @@ from calibre.ebooks.metadata import author_to_author_sort from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \ InvalidGenresSourceFieldException from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.config import config_dir from calibre.utils.date import format_date, is_date_undefined, now as nowf from calibre.utils.filenames import ascii_text, shorten_components_to from calibre.utils.icu import capitalize, collation_order, sort_key @@ -109,7 +108,7 @@ class CatalogBuilder(object): self.plugin = plugin self.reporter = report_progress self.stylesheet = stylesheet - self.cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.cache_dir = os.path.join(cache_dir(), 'catalog') self.catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') self.content_dir = os.path.join(self.catalog_path, "content") self.excluded_tags = self.get_excluded_tags() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ab85421697..5149b7a0de 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -44,47 +44,13 @@ from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions from calibre.db.errors import NoSuchFormat from calibre.db.lazy import FormatMetadata, FormatsList +from calibre.db.categories import Tag from calibre.utils.localization import (canonicalize_lang, calibre_langcode_to_name) copyfile = os.link if hasattr(os, 'link') else shutil.copyfile SPOOL_SIZE = 30*1024*1024 -class Tag(object): - - def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, - tooltip=None, icon=None, category=None, id_set=None, - is_editable = True, is_searchable=True, use_sort_as_name=False): - self.name = self.original_name = name - self.id = id - self.count = count - self.state = state - self.is_hierarchical = '' - self.is_editable = is_editable - self.is_searchable = is_searchable - self.id_set = id_set if id_set is not None else set([]) - self.avg_rating = avg/2.0 if avg is not None else 0 - self.sort = sort - self.use_sort_as_name = use_sort_as_name - if self.avg_rating > 0: - if tooltip: - tooltip = tooltip + ': ' - tooltip = _('%(tt)sAverage rating is %(rating)3.1f')%dict( - tt=tooltip, rating=self.avg_rating) - self.tooltip = tooltip - self.icon = icon - self.category = category - - def __unicode__(self): - return u'%s:%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state, - self.category, self.tooltip) - - def __str__(self): - return unicode(self).encode('utf-8') - - def __repr__(self): - return str(self) - class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' An ebook metadata database that stores references to ebook files on disk. @@ -1220,7 +1186,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc.append(_('Card A')) if b is not None: loc.append(_('Card B')) - return ', '.join(loc) + ((' (%s books)'%count) if count > 1 else '') + return ', '.join(loc) + ((_(' (%s books)')%count) if count > 1 else '') def set_book_on_device_func(self, func): self.book_on_device_func = func diff --git a/src/calibre/translations/calibre.pot b/src/calibre/translations/calibre.pot index d3f3538c27..9f2bb510fe 100644 --- a/src/calibre/translations/calibre.pot +++ b/src/calibre/translations/calibre.pot @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: calibre 0.9.15\n" -"POT-Creation-Date: 2013-01-18 09:12+IST\n" -"PO-Revision-Date: 2013-01-18 09:12+IST\n" +"POT-Creation-Date: 2013-01-22 10:10+IST\n" +"PO-Revision-Date: 2013-01-22 10:10+IST\n" "Last-Translator: Automatically generated\n" "Language-Team: LANGUAGE\n" "MIME-Version: 1.0\n" @@ -21,9 +21,9 @@ msgid "Does absolutely nothing" msgstr "" #: /home/kovid/work/calibre/src/calibre/customize/__init__.py:59 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:106 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:109 -#: /home/kovid/work/calibre/src/calibre/db/cache.py:120 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:139 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:142 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:153 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:379 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:380 #: /home/kovid/work/calibre/src/calibre/devices/hanvon/driver.py:114 @@ -42,8 +42,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/prst1/driver.py:469 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:480 #: /home/kovid/work/calibre/src/calibre/ebooks/chm/metadata.py:57 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:109 -#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:112 +#: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:183 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/comic_input.py:189 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:99 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/fb2_input.py:101 @@ -106,10 +105,10 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/ozon.py:130 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/sources/worker.py:26 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/txt.py:18 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:27 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:95 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:154 -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:193 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:28 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:98 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:156 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:195 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/mobi6.py:618 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/utils.py:316 #: /home/kovid/work/calibre/src/calibre/ebooks/mobi/writer2/indexer.py:463 @@ -155,11 +154,11 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/email.py:193 #: /home/kovid/work/calibre/src/calibre/gui2/email.py:208 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:439 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1103 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1319 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1322 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1325 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1413 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1104 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1320 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1323 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1326 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1414 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:250 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:261 @@ -884,7 +883,7 @@ msgstr "" msgid "Path to library too long. Must be less than %d characters." msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/cache.py:134 +#: /home/kovid/work/calibre/src/calibre/db/cache.py:167 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:666 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:67 #: /home/kovid/work/calibre/src/calibre/gui2/custom_column_widgets.py:678 @@ -894,23 +893,88 @@ msgstr "" msgid "Yes" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:163 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:186 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1218 msgid "Main" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:165 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:188 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:77 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1220 msgid "Card A" msgstr "" -#: /home/kovid/work/calibre/src/calibre/db/fields.py:167 +#: /home/kovid/work/calibre/src/calibre/db/fields.py:190 #: /home/kovid/work/calibre/src/calibre/gui2/layout.py:79 #: /home/kovid/work/calibre/src/calibre/library/database2.py:1222 msgid "Card B" msgstr "" +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:313 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:577 +msgid "checked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:33 +#: /home/kovid/work/calibre/src/calibre/db/search.py:311 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:135 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:575 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "yes" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:310 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:574 +#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 +msgid "no" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:35 +#: /home/kovid/work/calibre/src/calibre/db/search.py:312 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:137 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:576 +msgid "unchecked" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:110 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:313 +msgid "today" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:111 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:314 +msgid "yesterday" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:112 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:315 +msgid "thismonth" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:113 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:316 +msgid "daysago" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:314 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:578 +msgid "empty" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:315 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:579 +msgid "blank" +msgstr "" + +#: /home/kovid/work/calibre/src/calibre/db/search.py:324 +#: /home/kovid/work/calibre/src/calibre/library/caches.py:591 +msgid "Invalid boolean query \"{0}\"" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/debug.py:70 #: /home/kovid/work/calibre/src/calibre/gui2/main.py:47 msgid "Cause a running calibre instance, if any, to be shutdown. Note that if there are running jobs, they will be silently aborted, so use with care." @@ -1123,8 +1187,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:268 #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:324 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:391 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1134 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1136 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1128 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1130 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:277 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:279 msgid "Transferring books to device..." @@ -1135,8 +1199,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:491 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:525 #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:430 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1147 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1158 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1141 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1152 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:301 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:332 msgid "Adding books to device metadata listing..." @@ -1158,8 +1222,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/bambook/driver.py:374 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:479 #: /home/kovid/work/calibre/src/calibre/devices/kobo/driver.py:486 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1190 #: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1196 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1202 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:366 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:371 msgid "Removing books from device metadata listing..." @@ -1668,7 +1732,7 @@ msgid "Communicate with MTP devices" msgstr "" #: /home/kovid/work/calibre/src/calibre/devices/mtp/driver.py:167 -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:950 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:952 #: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:95 msgid "Get device information..." msgstr "" @@ -1967,17 +2031,17 @@ msgstr "" msgid "Too many connection attempts from %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1312 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1306 #, python-format msgid "Invalid port in options: %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1320 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1314 #, python-format msgid "Failed to connect to port %d. Try a different value." msgstr "" -#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1332 +#: /home/kovid/work/calibre/src/calibre/devices/smart_device_app/driver.py:1326 msgid "Failed to allocate a random port" msgstr "" @@ -3443,7 +3507,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/quickview.py:85 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/template_dialog.py:222 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:83 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:150 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:162 #: /home/kovid/work/calibre/src/calibre/gui2/store/search/models.py:39 @@ -3456,7 +3520,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/book/base.py:770 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:85 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1109 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 #: /home/kovid/work/calibre/src/calibre/gui2/store/stores/mobileread/models.py:23 msgid "Author(s)" msgstr "" @@ -3501,7 +3565,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:982 #: /home/kovid/work/calibre/src/calibre/library/catalogs/epub_mobi_builder.py:1228 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:201 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:804 msgid "Tags" msgstr "" @@ -3742,7 +3806,7 @@ msgstr "" msgid "Downloads metadata and covers from OZON.ru" msgstr "" -#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:58 +#: /home/kovid/work/calibre/src/calibre/ebooks/mobi/reader/headers.py:61 msgid "Sample Book" msgstr "" @@ -3778,7 +3842,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/base.py:1281 #: /home/kovid/work/calibre/src/calibre/ebooks/oeb/transforms/htmltoc.py:15 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:221 -#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:217 +#: /home/kovid/work/calibre/src/calibre/gui2/viewer/toc.py:219 msgid "Table of Contents" msgstr "" @@ -3863,7 +3927,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:71 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/metadata_sources.py:160 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:176 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:800 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:802 msgid "Rating" msgstr "" @@ -4985,8 +5049,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:101 #: /home/kovid/work/calibre/src/calibre/gui2/dnd.py:84 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:518 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:830 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 msgid "Download failed" msgstr "" @@ -5018,7 +5082,7 @@ msgid "Download complete" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:123 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:892 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:901 msgid "Download log" msgstr "" @@ -5249,7 +5313,7 @@ msgid "Click the show details button to see which ones." msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/actions/show_book_details.py:16 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:807 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 msgid "Show book details" msgstr "" @@ -5799,7 +5863,7 @@ msgid "Click to open" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:180 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:856 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:858 msgid "Ids" msgstr "" @@ -5809,7 +5873,7 @@ msgid "Book %(sidx)s of %(series)s" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/book_details.py:233 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1113 msgid "Collections" msgstr "" @@ -8315,7 +8379,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/device_drivers/mtp_config.py:421 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:141 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:885 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:894 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/tweaks.py:344 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:227 msgid "Copy to clipboard" @@ -8815,7 +8879,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/delete_matching_from_device.py:77 #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:87 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1110 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:35 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/create_custom_column.py:76 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:365 @@ -8931,7 +8995,7 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/edit_authors_dialog.py:122 #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main.py:160 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:527 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:536 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main.py:729 msgid "No matches found" msgstr "" @@ -9110,8 +9174,8 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:196 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:251 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:950 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1059 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:959 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1074 #: /home/kovid/work/calibre/src/calibre/gui2/proceed.py:48 msgid "View log" msgstr "" @@ -11539,13 +11603,13 @@ msgid "Modified" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:819 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1455 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1456 #: /home/kovid/work/calibre/src/calibre/gui2/tag_browser/model.py:335 msgid "The lookup/search name is \"{0}\"" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:825 -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1457 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1458 msgid "This book's UUID is \"{0}\"" msgstr "" @@ -11574,20 +11638,20 @@ msgstr "" msgid "Could not set data, click Show Details to see why." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1107 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1108 msgid "In Library" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1111 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1112 #: /home/kovid/work/calibre/src/calibre/library/field_metadata.py:355 msgid "Size" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1437 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1438 msgid "Marked for deletion" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1440 +#: /home/kovid/work/calibre/src/calibre/gui2/library/models.py:1441 msgid "Double click to edit me

" msgstr "" @@ -11690,7 +11754,7 @@ msgid "Previous Page" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main_ui.py:133 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:947 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:956 #: /home/kovid/work/calibre/src/calibre/gui2/store/web_store_dialog_ui.py:62 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:215 msgid "Back" @@ -12131,7 +12195,7 @@ msgid "Edit Metadata" msgstr "" #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single.py:63 -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:940 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:949 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:108 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:219 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:410 @@ -12288,62 +12352,62 @@ msgid "" "cover stage, and vice versa." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:292 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:301 msgid "See at" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:446 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:455 msgid "calibre is downloading metadata from: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:468 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:477 msgid "Please wait" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:500 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:509 msgid "Query: " msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:519 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 msgid "Failed to download metadata. Click Show Details to see details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:528 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:537 msgid "Failed to find any books that match your search. Try making the search less specific. For example, use only the author's last name and a single distinctive word from the title.

To see the full log, click Show Details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:636 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:645 msgid "Current cover" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:639 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:648 msgid "Searching..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:800 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:809 #, python-format msgid "Downloading covers for %s, please wait..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:831 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:840 msgid "Failed to download any covers, click \"Show details\" for details." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:837 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:846 #, python-format msgid "Could not find any covers for %s" msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:839 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:848 #, python-format msgid "Found %(num)d covers of %(title)s. Pick the one you like best." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:928 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:937 msgid "Downloading metadata..." msgstr "" -#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1043 +#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1058 msgid "Downloading cover..." msgstr "" @@ -16693,56 +16757,6 @@ msgid "" "

Stanza should see your calibre collection automatically. If not, try adding the URL http://myhostname:8080 as a new catalog in the Stanza reader on your iPhone. Here myhostname should be the fully qualified hostname or the IP address of the computer calibre is running on." msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:617 -msgid "checked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:177 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:615 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "yes" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:614 -#: /home/kovid/work/calibre/src/calibre/library/save_to_disk.py:229 -msgid "no" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:179 -#: /home/kovid/work/calibre/src/calibre/library/caches.py:616 -msgid "unchecked" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:355 -msgid "today" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:356 -msgid "yesterday" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:357 -msgid "thismonth" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:358 -msgid "daysago" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:618 -msgid "empty" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:619 -msgid "blank" -msgstr "" - -#: /home/kovid/work/calibre/src/calibre/library/caches.py:631 -msgid "Invalid boolean query \"{0}\"" -msgstr "" - #: /home/kovid/work/calibre/src/calibre/library/catalogs/bibtex.py:36 #, python-format msgid "" @@ -17766,6 +17780,11 @@ msgstr "" msgid "creating custom column " msgstr "" +#: /home/kovid/work/calibre/src/calibre/library/database2.py:1223 +#, python-format +msgid " (%s books)" +msgstr "" + #: /home/kovid/work/calibre/src/calibre/library/database2.py:3698 #, python-format msgid "

Migrating old database to ebook library in %s

" @@ -17985,19 +18004,19 @@ msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:317 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:355 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:649 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:651 msgid "All books" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/ajax.py:318 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:354 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:648 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:650 #: /home/kovid/work/calibre/src/calibre/library/server/opds.py:584 msgid "Newest" msgstr "" #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:65 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:518 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:520 msgid "Loading, please wait" msgstr "" @@ -18050,65 +18069,65 @@ msgstr "" msgid "Random book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:403 -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:472 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:405 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:474 msgid "Browse books by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:408 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:410 msgid "Choose a category to browse by:" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:543 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:545 msgid "Browsing by" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:544 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:546 msgid "Up" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:684 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:686 msgid "in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:687 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:689 msgid "Books in" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:781 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:783 msgid "Other formats" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:788 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:790 #, python-format msgid "Read %(title)s in the %(fmt)s format" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:793 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:795 msgid "Get" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:806 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 msgid "Details" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:808 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:810 msgid "Permalink" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:809 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:811 msgid "A permanent link to this book" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:821 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:823 msgid "This book has been deleted" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:927 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 msgid "in search" msgstr "" -#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:929 +#: /home/kovid/work/calibre/src/calibre/library/server/browse.py:931 msgid "Matching books" msgstr "" diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index de21158ed7..fc04044ad3 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -32,6 +32,10 @@ class Browser(B): B.set_cookiejar(self, *args, **kwargs) self._clone_actions['set_cookiejar'] = ('set_cookiejar', args, kwargs) + def copy_cookies_from_jsbrowser(self, jsbrowser): + for cookie in jsbrowser.cookies: + self.cookiejar.set_cookie(cookie) + @property def cookiejar(self): return self._clone_actions['set_cookiejar'][1][0] diff --git a/src/calibre/utils/fonts/sfnt/metrics.py b/src/calibre/utils/fonts/sfnt/metrics.py index 4f86948ff2..6f8deff31b 100644 --- a/src/calibre/utils/fonts/sfnt/metrics.py +++ b/src/calibre/utils/fonts/sfnt/metrics.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' from future_builtins import map from calibre.utils.fonts.utils import get_all_font_names +from calibre.utils.fonts.sfnt.container import UnsupportedFont class FontMetrics(object): @@ -19,6 +20,10 @@ class FontMetrics(object): ''' def __init__(self, sfnt): + for table in (b'head', b'hhea', b'hmtx', b'cmap', b'OS/2', b'post', + b'name'): + if table not in sfnt: + raise UnsupportedFont('This font has no %s table'%table) self.sfnt = sfnt self.head = self.sfnt[b'head'] diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 14834ff88c..22901f3ccc 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -332,6 +332,12 @@ class BasicNewsRecipe(Recipe): #: ignore_duplicate_articles = {'title', 'url'} ignore_duplicate_articles = None + #: If you set this True, then calibre will use javascript to login to the + #: website. This is needed for some websites that require the use of + #: javascript to login. If you set this to True you must implement the + #: :meth:`javascript_login` method, to do the actual logging in. + use_javascript_to_login = False + # See the built-in profiles for examples of these settings. def short_title(self): @@ -404,8 +410,7 @@ class BasicNewsRecipe(Recipe): ''' return url - @classmethod - def get_browser(cls, *args, **kwargs): + def get_browser(self, *args, **kwargs): ''' Return a browser instance used to fetch documents from the web. By default it returns a `mechanize `_ @@ -427,9 +432,47 @@ class BasicNewsRecipe(Recipe): return br ''' - br = browser(*args, **kwargs) - br.addheaders += [('Accept', '*/*')] - return br + if self.use_javascript_to_login: + if getattr(self, 'browser', None) is not None: + return self.clone_browser(self.browser) + from calibre.web.jsbrowser.browser import Browser + br = Browser() + with br: + self.javascript_login(br, self.username, self.password) + kwargs['user_agent'] = br.user_agent + ans = browser(*args, **kwargs) + ans.copy_cookies_from_jsbrowser(br) + return ans + else: + br = browser(*args, **kwargs) + br.addheaders += [('Accept', '*/*')] + return br + + def javascript_login(self, browser, username, password): + ''' + This method is used to login to a website that uses javascript for its + login form. After the login is complete, the cookies returned from the + website are copied to a normal (non-javascript) browser and the + download proceeds using those cookies. + + An example implementation:: + + def javascript_login(self, browser, username, password): + browser.visit('http://some-page-that-has-a-login') + form = browser.select_form(nr=0) # Select the first form on the page + form['username'] = username + form['password'] = password + browser.submit(timeout=120) # Submit the form and wait at most two minutes for loading to complete + + Note that you can also select forms with CSS2 selectors, like this:: + + browser.select_form('form#login_form') + browser.select_from('form[name="someform"]') + + ''' + raise NotImplementedError('You must implement the javascript_login()' + ' method if you set use_javascript_to_login' + ' to True') def clone_browser(self, br): ''' diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index bea45f1c8d..87f97a3395 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -10,8 +10,6 @@ UTF-8 encoding with any charset declarations removed. import sys, socket, os, urlparse, re, time, copy, urllib2, threading, traceback, imghdr from urllib import url2pathname, quote from httplib import responses -from PIL import Image -from cStringIO import StringIO from base64 import b64decode from calibre import browser, relpath, unicode_path @@ -21,6 +19,8 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.ebooks.chardet import xml_to_unicode from calibre.utils.config import OptionParser from calibre.utils.logging import Log +from calibre.utils.magick import Image +from calibre.utils.magick.draw import identify_data class FetchError(Exception): pass @@ -374,8 +374,8 @@ class RecursiveFetcher(object): fname = ascii_filename('img'+str(c)) if isinstance(fname, unicode): fname = fname.encode('ascii', 'replace') - imgpath = os.path.join(diskpath, fname+'.jpg') - if (imghdr.what(None, data) is None and b'