From c0803510940f762fd13418f4a0ee2a902401966e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 12 Feb 2011 08:18:23 +0000 Subject: [PATCH 01/21] Fix deleting tags in tags_list_editor --- src/calibre/gui2/dialogs/tag_list_editor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py index 9694a9a459..6c3ebb22d5 100644 --- a/src/calibre/gui2/dialogs/tag_list_editor.py +++ b/src/calibre/gui2/dialogs/tag_list_editor.py @@ -99,8 +99,8 @@ class TagListEditor(QDialog, Ui_TagListEditor): return self.available_tags.editItem(item) - def delete_tags(self, item=None): - deletes = self.available_tags.selectedItems() if item is None else [item] + def delete_tags(self): + deletes = self.available_tags.selectedItems() if not deletes: error_dialog(self, _('No items selected'), _('You must select at least one items from the list.')).exec_() From 2ad44078eaaa581f47e6dbdde6ef98d217ac9305 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 12 Feb 2011 13:02:44 +0000 Subject: [PATCH 02/21] Fix #8925: Allow different libraries top display different items in the Tag Browser --- src/calibre/gui2/__init__.py | 8 ++++++-- src/calibre/gui2/tag_view.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index b33166dd33..efe09e8866 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -137,14 +137,18 @@ def _config(): help=_('Automatically download the cover, if available')) c.add_opt('enforce_cpu_limit', default=True, help=_('Limit max simultaneous jobs to number of CPUs')) - c.add_opt('tag_browser_hidden_categories', default=set(), - help=_('tag browser categories not to display')) c.add_opt('gui_layout', choices=['wide', 'narrow'], help=_('The layout of the user interface'), default='wide') c.add_opt('show_avg_rating', default=True, help=_('Show the average rating per item indication in the tag browser')) c.add_opt('disable_animations', default=False, help=_('Disable UI animations')) + + # This option is no longer used. It remains for compatibility with upgrades + # so the value can be migrated + c.add_opt('tag_browser_hidden_categories', default=set(), + help=_('tag browser categories not to display')) + c.add_opt return ConfigProxy(c) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 79199c6881..3bc5d724ba 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -116,7 +116,14 @@ class TagsView(QTreeView): # {{{ self.set_new_model(self._model.get_filter_categories_by()) def set_database(self, db, tag_match, sort_by): - self.hidden_categories = config['tag_browser_hidden_categories'] + self.hidden_categories = db.prefs.get('tag_browser_hidden_categories', None) + # migrate from config to db prefs + if self.hidden_categories is None: + self.hidden_categories = config['tag_browser_hidden_categories'] + db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories)) + else: + self.hidden_categories = set(self.hidden_categories) + old = getattr(self, '_model', None) if old is not None: old.break_cycles() @@ -234,7 +241,7 @@ class TagsView(QTreeView): # {{{ gprefs['tags_browser_partition_method'] = category elif action == 'defaults': self.hidden_categories.clear() - config.set('tag_browser_hidden_categories', self.hidden_categories) + self.db.prefs.set('tag_browser_hidden_categories', list(self.hidden_categories)) self.set_new_model() except: return From 5ca7a4a8845a69e67456df86c16921c1bd089453 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 12 Feb 2011 11:14:45 -0700 Subject: [PATCH 03/21] Fix #8916 (Cybook Orizon connection) --- src/calibre/devices/usbms/device.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index a31897c8e5..b0857de909 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -232,16 +232,37 @@ class Device(DeviceConfig, DevicePlugin): time.sleep(5) drives = {} + seen = set() + prod_pat = re.compile(r'PROD_(.+?)&') + dup_prod_id = False + + def check_for_dups(pnp_id): + try: + match = prod_pat.search(pnp_id) + if match is not None: + prodid = match.group(1) + if prodid in seen: + return True + else: + seen.add(prodid) + except: + pass + return False + + for drive, pnp_id in win_pnp_drives().items(): if self.windows_match_device(pnp_id, 'WINDOWS_CARD_A_MEM') and \ not drives.get('carda', False): drives['carda'] = drive + dup_prod_id |= check_for_dups(pnp_id) elif self.windows_match_device(pnp_id, 'WINDOWS_CARD_B_MEM') and \ not drives.get('cardb', False): drives['cardb'] = drive + dup_prod_id |= check_for_dups(pnp_id) elif self.windows_match_device(pnp_id, 'WINDOWS_MAIN_MEM') and \ not drives.get('main', False): drives['main'] = drive + dup_prod_id |= check_for_dups(pnp_id) if 'main' in drives.keys() and 'carda' in drives.keys() and \ 'cardb' in drives.keys(): @@ -263,7 +284,8 @@ class Device(DeviceConfig, DevicePlugin): # Sort drives by their PNP drive numbers if the CARD and MAIN # MEM strings are identical - if self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM, + if dup_prod_id or \ + self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM, self.WINDOWS_CARD_B_MEM) or \ self.WINDOWS_CARD_A_MEM == self.WINDOWS_CARD_B_MEM: letters = sorted(drives.values(), cmp=drivecmp) From cc2f6b8d5eec722999059715a99b5a060c9958b3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 12 Feb 2011 22:08:38 -0700 Subject: [PATCH 04/21] Fix #405 (New news feed). Apple daily by MrLai --- resources/recipes/apple_daily.recipe | 161 +++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 resources/recipes/apple_daily.recipe diff --git a/resources/recipes/apple_daily.recipe b/resources/recipes/apple_daily.recipe new file mode 100644 index 0000000000..1e9953af43 --- /dev/null +++ b/resources/recipes/apple_daily.recipe @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +import re +from calibre.web.feeds.recipes import BasicNewsRecipe + +class AppleDaily(BasicNewsRecipe): + + title = u'蘋果日報' + __author__ = u'蘋果日報' + __publisher__ = u'蘋果日報' + description = u'蘋果日報' + masthead_url = 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif' + language = 'zh_TW' + encoding = 'UTF-8' + timefmt = ' [%a, %d %b, %Y]' + needs_subscription = False + remove_javascript = True + remove_tags_before = dict(name=['ul', 'h1']) + remove_tags_after = dict(name='form') + remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix']}), + dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']), + dict(name=['script', 'noscript', 'style', 'form'])] + no_stylesheets = True + extra_css = ''' + @font-face {font-family: "uming", serif, sans-serif; src: url(res:///usr/share/fonts/truetype/arphic/uming.ttc); }\n + body {margin-right: 8pt; font-family: 'uming', serif;} + h1 {font-family: 'uming', serif, sans-serif} + ''' + #extra_css = 'h1 {font: sans-serif large;}\n.byline {font:monospace;}' + + preprocess_regexps = [ + (re.compile(r'img.php?server=(?P[^&]+)&path=(?P[^&]+).*', re.DOTALL|re.IGNORECASE), + lambda match: 'http://' + match.group('server') + '/' + match.group('path')), + ] + + def get_cover_url(self): + return 'http://hk.apple.nextmedia.com/template/common/header/2009/images/atnextheader_logo_appledaily.gif' + + + #def get_browser(self): + #br = BasicNewsRecipe.get_browser() + #if self.username is not None and self.password is not None: + # br.open('http://www.nytimes.com/auth/login') + # br.select_form(name='login') + # br['USERID'] = self.username + # br['PASSWORD'] = self.password + # br.submit() + #return br + + def preprocess_html(self, soup): + #process all the images + for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')): + iurl = tag['src'] + #print 'checking image: ' + iurl + + #img\.php?server\=(?P[^&]+)&path=(?P[^&]+) + p = re.compile(r'img\.php\?server=(?P[^&]+)&path=(?P[^&]+)', re.DOTALL|re.IGNORECASE) + + m = p.search(iurl) + + if m is not None: + iurl = 'http://' + m.group('server') + '/' + m.group('path') + #print 'working! new url: ' + iurl + tag['src'] = iurl + #else: + #print 'not good' + + for tag in soup.findAll(lambda tag: tag.name.lower()=='a' and tag.has_key('href')): + iurl = tag['href'] + #print 'checking image: ' + iurl + + #img\.php?server\=(?P[^&]+)&path=(?P[^&]+) + p = re.compile(r'img\.php\?server=(?P[^&]+)&path=(?P[^&]+)', re.DOTALL|re.IGNORECASE) + + m = p.search(iurl) + + if m is not None: + iurl = 'http://' + m.group('server') + '/' + m.group('path') + #print 'working! new url: ' + iurl + tag['href'] = iurl + #else: + #print 'not good' + + return soup + + + def parse_index(self): + base = 'http://news.hotpot.hk/fruit' + soup = self.index_to_soup('http://news.hotpot.hk/fruit/index.php') + + #def feed_title(div): + # return ''.join(div.findAll(text=True, recursive=False)).strip() + + articles = {} + key = None + ans = [] + for div in soup.findAll('li'): + key = div.find(text=True, recursive=True); + #if key == u'豪情': + # continue; + + print 'section=' + key + + articles[key] = [] + + ans.append(key) + + a = div.find('a', href=True) + + if not a: + continue + + url = base + '/' + a['href'] + print 'url=' + url + + if not articles.has_key(key): + articles[key] = [] + else: + # sub page + subSoup = self.index_to_soup(url) + + for subDiv in subSoup.findAll('li'): + subA = subDiv.find('a', href=True) + subTitle = subDiv.find(text=True, recursive=True) + subUrl = base + '/' + subA['href'] + + print 'subUrl' + subUrl + + articles[key].append( + dict(title=subTitle, + url=subUrl, + date='', + description='', + content='')) + + +# elif div['class'] in ['story', 'story headline']: +# a = div.find('a', href=True) +# if not a: +# continue +# url = re.sub(r'\?.*', '', a['href']) +# url += '?pagewanted=all' +# title = self.tag_to_string(a, use_alt=True).strip() +# description = '' +# pubdate = strftime('%a, %d %b') +# summary = div.find(True, attrs={'class':'summary'}) +# if summary: +# description = self.tag_to_string(summary, use_alt=False) +# +# feed = key if key is not None else 'Uncategorized' +# if not articles.has_key(feed): +# articles[feed] = [] +# if not 'podcasts' in url: +# articles[feed].append( +# dict(title=title, url=url, date=pubdate, +# description=description, +# content='')) +# ans = self.sort_index_by(ans, {'The Front Page':-1, 'Dining In, Dining Out':1, 'Obituaries':2}) + ans = [(unicode(key), articles[key]) for key in ans if articles.has_key(key)] + return ans + + From b2669d5ba43892896837f386bdc871c699bd3bde Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 13 Feb 2011 10:06:24 +0000 Subject: [PATCH 05/21] 1) #8945: Bulk edit dialog still offers tristate choice for custom Yes/No column 2) change sort to not distinguish between None and False when bools are not tristate. 3) make the check of the tristate tweak always check for 'no' instead of not 'yes'. Intent is to make the system more stable if the user enters garbage into the tweak. --- src/calibre/gui2/custom_column_widgets.py | 19 +++++++++++++++++-- src/calibre/gui2/library/models.py | 2 +- src/calibre/library/caches.py | 7 +++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index eae6dc79c3..fa7ba3c56d 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -551,7 +551,11 @@ class BulkBool(BulkBase, Bool): def setup_ui(self, parent): self.make_widgets(parent, QComboBox) - items = [_('Yes'), _('No'), _('Undefined')] + items = [_('Yes'), _('No')] + if tweaks['bool_custom_columns_are_tristate'] == 'no': + items.append('') + else: + items.append(_('Undefined')) icons = [I('ok.png'), I('list_remove.png'), I('blank.png')] self.main_widget.blockSignals(True) for icon, text in zip(icons, items): @@ -560,7 +564,10 @@ class BulkBool(BulkBase, Bool): def getter(self): val = self.main_widget.currentIndex() - return {2: None, 1: False, 0: True}[val] + if tweaks['bool_custom_columns_are_tristate'] == 'no': + return {2: False, 1: False, 0: True}[val] + else: + return {2: None, 1: False, 0: True}[val] def setter(self, val): val = {None: 2, False: 1, True: 0}[val] @@ -576,6 +583,14 @@ class BulkBool(BulkBase, Bool): val = False self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) + def a_c_checkbox_changed(self): + if not self.ignore_change_signals: + if tweaks['bool_custom_columns_are_tristate'] == 'no' and \ + self.main_widget.currentIndex() == 2: + self.a_c_checkbox.setChecked(False) + else: + self.a_c_checkbox.setChecked(True) + class BulkInt(BulkBase): def setup_ui(self, parent): diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 48668d3376..88008e7ec4 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -685,7 +685,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.dc[col] = functools.partial(bool_type, idx=idx) self.dc_decorator[col] = functools.partial( bool_type_decorator, idx=idx, - bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes') + bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] != 'no') elif datatype == 'rating': self.dc[col] = functools.partial(rating_type, idx=idx) elif datatype == 'series': diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 1330d10e59..70e1fec131 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -528,7 +528,7 @@ class ResultCache(SearchQueryParser): # {{{ location[i] = db_col[loc] # get the tweak here so that the string lookup and compare aren't in the loop - bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes' + bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] != 'no' for loc in location: # location is now an array of field indices if loc == db_col['authors']: @@ -812,7 +812,10 @@ class SortKeyGenerator(object): val = self.string_sort_key(val) elif dt == 'bool': - val = {True: 1, False: 2, None: 3}.get(val, 3) + if tweaks['bool_custom_columns_are_tristate'] == 'no': + val = {True: 1, False: 2, None: 2}.get(val, 2) + else: + val = {True: 1, False: 2, None: 3}.get(val, 3) yield val From ed363bd3392987f318fa9e6cd5cf916b47e3e287 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 13 Feb 2011 12:14:52 +0000 Subject: [PATCH 06/21] Improve performance of device connection with huge libraries --- src/calibre/gui2/device.py | 90 ++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 3540575f81..e4096f5761 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1292,6 +1292,16 @@ class DeviceMixin(object): # {{{ to both speed up matching and to count matches. ''' + if not self.device_manager.is_device_connected: + return False + + # It might be possible to get here without having initialized the + # library view. In this case, simply give up + try: + db = self.library_view.model().db + except: + return False + string_pat = re.compile('(?u)\W|[_]') def clean_string(x): x = x.lower() if x else '' @@ -1299,26 +1309,19 @@ class DeviceMixin(object): # {{{ update_metadata = prefs['manage_device_metadata'] == 'on_connect' + get_covers = False + if update_metadata and self.device_manager.is_device_connected: + if self.device_manager.device.WANTS_UPDATED_THUMBNAILS: + get_covers = True + # Force a reset if the caches are not initialized if reset or not hasattr(self, 'db_book_title_cache'): # Build a cache (map) of the library, so the search isn't On**2 db_book_title_cache = {} db_book_uuid_cache = {} - # It might be possible to get here without having initialized the - # library view. In this case, simply give up - try: - db = self.library_view.model().db - except: - return False - get_covers = False - if update_metadata and self.device_manager.is_device_connected: - if self.device_manager.device.WANTS_UPDATED_THUMBNAILS: - get_covers = True - - for id in db.data.iterallids(): - mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers) - title = clean_string(mi.title) + for id_ in db.data.iterallids(): + title = clean_string(db.title(id_, index_is_id=True)) if title not in db_book_title_cache: db_book_title_cache[title] = \ {'authors':{}, 'author_sort':{}, 'db_ids':{}} @@ -1326,14 +1329,14 @@ class DeviceMixin(object): # {{{ # and author, then remember the last one. That is OK, because as # we can't tell the difference between the books, one is as good # as another. - if mi.authors: - authors = clean_string(authors_to_string(mi.authors)) - db_book_title_cache[title]['authors'][authors] = mi - if mi.author_sort: - aus = clean_string(mi.author_sort) - db_book_title_cache[title]['author_sort'][aus] = mi - db_book_title_cache[title]['db_ids'][mi.application_id] = mi - db_book_uuid_cache[mi.uuid] = mi + authors = clean_string(db.authors(id_, index_is_id=True)) + if authors: + db_book_title_cache[title]['authors'][authors] = id_ + if db.author_sort(id_, index_is_id=True): + aus = clean_string(db.author_sort(id_, index_is_id=True)) + db_book_title_cache[title]['author_sort'][aus] = id_ + db_book_title_cache[title]['db_ids'][id_] = id_ + db_book_uuid_cache[db.uuid(id_, index_is_id=True)] = id_ self.db_book_title_cache = db_book_title_cache self.db_book_uuid_cache = db_book_uuid_cache @@ -1341,19 +1344,22 @@ class DeviceMixin(object): # {{{ # in_library field. If the UUID matches a book in the library, then # do not consider that book for other matching. In all cases set # the application_id to the db_id of the matching book. This value - # will be used by books_on_device to indicate matches. + # will be used by books_on_device to indicate matches. While we are + # going by, update the metadata for a book if automatic management is on for booklist in booklists: for book in booklist: book.in_library = None if getattr(book, 'uuid', None) in self.db_book_uuid_cache: + id_ = db_book_uuid_cache[book.uuid] if update_metadata: - book.smart_update(self.db_book_uuid_cache[book.uuid], + book.smart_update(db.get_metadata(id_, + index_is_id=True, + get_cover=get_covers), replace_metadata=True) book.in_library = 'UUID' # ensure that the correct application_id is set - book.application_id = \ - self.db_book_uuid_cache[book.uuid].application_id + book.application_id = id_ continue # No UUID exact match. Try metadata matching. book_title = clean_string(book.title) @@ -1363,21 +1369,25 @@ class DeviceMixin(object): # {{{ # will match if any of the db_id, author, or author_sort # also match. if getattr(book, 'application_id', None) in d['db_ids']: - # app_id already matches a db_id. No need to set it. if update_metadata: - book.smart_update(d['db_ids'][book.application_id], + id_ = getattr(book, 'application_id', None) + book.smart_update(db.get_metadata(id_, + index_is_id=True, + get_cover=get_covers), replace_metadata=True) book.in_library = 'APP_ID' + # app_id already matches a db_id. No need to set it. continue # Sonys know their db_id independent of the application_id # in the metadata cache. Check that as well. if getattr(book, 'db_id', None) in d['db_ids']: if update_metadata: - book.smart_update(d['db_ids'][book.db_id], + book.smart_update(db.get_metadata(book.db_id, + index_is_id=True, + get_cover=get_covers), replace_metadata=True) book.in_library = 'DB_ID' - book.application_id = \ - d['db_ids'][book.db_id].application_id + book.application_id = book.db_id continue # We now know that the application_id is not right. Set it # to None to prevent book_on_device from accidentally @@ -1389,19 +1399,23 @@ class DeviceMixin(object): # {{{ # either can appear as the author book_authors = clean_string(authors_to_string(book.authors)) if book_authors in d['authors']: + id_ = d['authors'][book_authors] if update_metadata: - book.smart_update(d['authors'][book_authors], - replace_metadata=True) + book.smart_update(db.get_metadata(id_, + index_is_id=True, + get_cover=get_covers), + replace_metadata=True) book.in_library = 'AUTHOR' - book.application_id = \ - d['authors'][book_authors].application_id + book.application_id = id_ elif book_authors in d['author_sort']: + id_ = d['author_sort'][book_authors] if update_metadata: - book.smart_update(d['author_sort'][book_authors], + book.smart_update(db.get_metadata(id_, + index_is_id=True, + get_cover=get_covers), replace_metadata=True) book.in_library = 'AUTH_SORT' - book.application_id = \ - d['author_sort'][book_authors].application_id + book.application_id = id_ else: # Book definitely not matched. Clear its application ID book.application_id = None From 6f99e31f8150f706fcf38684a8db4fbffa8987c9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 07:43:16 -0700 Subject: [PATCH 07/21] Plugin config: Allow config widget to indicate that configuration cannot be carried out --- src/calibre/customize/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 13e1f20a2d..1f44eb4ae2 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -90,6 +90,11 @@ class Plugin(object): # {{{ an optional method validate() that takes no arguments and is called immediately after the user clicks OK. Changes are applied if and only if the method returns True. + + If for some reason you cannot perform the configuration at this time, + return a tuple of two strings (message, details), these will be + displayed as a warning dialog to the user and the process will be + aborted. ''' raise NotImplementedError() @@ -133,6 +138,12 @@ class Plugin(object): # {{{ except NotImplementedError: config_widget = None + if isinstance(config_widget, tuple): + from calibre.gui2 import warning_dialog + warning_dialog(parent, _('Cannot configure'), config_widget[0], + det_msg=config_widget[1], show=True) + return False + if config_widget is not None: v.addWidget(config_widget) v.addWidget(button_box) From f1348401c34e988eb874e91b61a32191a6eaca75 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 08:01:42 -0700 Subject: [PATCH 08/21] Fix #8924 (my wolder mibuklife is not detected by calibre) --- src/calibre/devices/jetbook/driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/jetbook/driver.py b/src/calibre/devices/jetbook/driver.py index f108de3347..0d328ba637 100644 --- a/src/calibre/devices/jetbook/driver.py +++ b/src/calibre/devices/jetbook/driver.py @@ -93,11 +93,11 @@ class MIBUK(USBMS): VENDOR_ID = [0x0525] PRODUCT_ID = [0xa4a5] - BCD = [0x314] + BCD = [0x314, 0x319] SUPPORTS_SUB_DIRS = True - VENDOR_NAME = 'LINUX' - WINDOWS_MAIN_MEM = 'WOLDERMIBUK' + VENDOR_NAME = ['LINUX', 'FILE_BAC'] + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['WOLDERMIBUK', 'KED_STORAGE_GADG'] class JETBOOK_MINI(USBMS): From 8c5dbd764178242fcdc58afd5c6bb5c7c3e05d30 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 09:55:28 -0700 Subject: [PATCH 09/21] Device driver customization: Allow user to tell calibre to send any ebook format to the device rather than just the list of formats that calibre thinks the device will support --- src/calibre/devices/apple/driver.py | 1 + src/calibre/devices/bambook/driver.py | 3 ++- src/calibre/devices/kobo/driver.py | 5 ++-- src/calibre/devices/usbms/deviceconfig.py | 6 ++++- src/calibre/devices/usbms/driver.py | 4 ++- src/calibre/gui2/actions/add.py | 3 ++- .../gui2/device_drivers/configwidget.py | 25 ++++++++++++++++--- 7 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 369c470e2b..cc4d39d3c5 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -39,6 +39,7 @@ if iswindows: class DriverBase(DeviceConfig, DevicePlugin): # Needed for config_widget to work FORMATS = ['epub', 'pdf'] + USER_CAN_ADD_NEW_FORMATS = False SUPPORTS_SUB_DIRS = True # To enable second checkbox in customize widget @classmethod diff --git a/src/calibre/devices/bambook/driver.py b/src/calibre/devices/bambook/driver.py index e7fa66c939..3cc0245cf7 100644 --- a/src/calibre/devices/bambook/driver.py +++ b/src/calibre/devices/bambook/driver.py @@ -32,6 +32,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin): ip = None FORMATS = [ "snb" ] + USER_CAN_ADD_NEW_FORMATS = False VENDOR_ID = 0x230b PRODUCT_ID = 0x0001 BCD = None @@ -421,7 +422,7 @@ class BAMBOOK(DeviceConfig, DevicePlugin): from calibre.gui2.device_drivers.configwidget import ConfigWidget cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS, cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT, - cls.EXTRA_CUSTOMIZATION_MESSAGE) + cls.EXTRA_CUSTOMIZATION_MESSAGE, cls) # Turn off the Save template cw.opt_save_template.setVisible(False) cw.label.setVisible(False) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index c5e8f5ca0f..ac5f9d4cce 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -98,7 +98,6 @@ class KOBO(USBMS): def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType): changed = False - # if path_to_ext(path) in self.FORMATS: try: lpath = path.partition(self.normalize_path(prefix))[2] if lpath.startswith(os.sep): @@ -220,7 +219,7 @@ class KOBO(USBMS): # 2) volume_shorcover # 2) content - debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) + debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType) connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite')) cursor = connection.cursor() t = (ContentID,) @@ -532,7 +531,7 @@ class KOBO(USBMS): if result is None: datelastread = '1970-01-01T00:00:00' else: - datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' + datelastread = result[0] if result[0] is not None else '1970-01-01T00:00:00' t = (datelastread,ContentID,) diff --git a/src/calibre/devices/usbms/deviceconfig.py b/src/calibre/devices/usbms/deviceconfig.py index 940ea96f38..3c79652463 100644 --- a/src/calibre/devices/usbms/deviceconfig.py +++ b/src/calibre/devices/usbms/deviceconfig.py @@ -34,6 +34,10 @@ class DeviceConfig(object): #: If None the default is used SAVE_TEMPLATE = None + #: If True the user can add new formats to the driver + USER_CAN_ADD_NEW_FORMATS = True + + @classmethod def _default_save_template(cls): from calibre.library.save_to_disk import config @@ -73,7 +77,7 @@ class DeviceConfig(object): from calibre.gui2.device_drivers.configwidget import ConfigWidget cw = ConfigWidget(cls.settings(), cls.FORMATS, cls.SUPPORTS_SUB_DIRS, cls.MUST_READ_METADATA, cls.SUPPORTS_USE_AUTHOR_SORT, - cls.EXTRA_CUSTOMIZATION_MESSAGE) + cls.EXTRA_CUSTOMIZATION_MESSAGE, cls) return cw @classmethod diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 2f26c4a353..6f8f17f5c9 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -93,9 +93,11 @@ class USBMS(CLI, Device): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx + all_formats = set(self.settings().format_map) | set(self.FORMATS) + def update_booklist(filename, path, prefix): changed = False - if path_to_ext(filename) in self.FORMATS: + if path_to_ext(filename) in all_formats: try: lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] if lpath.startswith(os.sep): diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 25127d3635..83600c3227 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -204,7 +204,8 @@ class AddAction(InterfaceAction): ] to_device = self.gui.stack.currentIndex() != 0 if to_device: - filters = [(_('Supported books'), self.gui.device_manager.device.FORMATS)] + fmts = self.gui.device_manager.device.settings().format_map + filters = [(_('Supported books'), fmts)] books = choose_files(self.gui, 'add books dialog dir', 'Select books', filters=filters) diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index 7b440db7fc..97c492b550 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -9,15 +9,16 @@ import textwrap from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL, \ QLabel, QLineEdit, QCheckBox -from calibre.gui2 import error_dialog +from calibre.gui2 import error_dialog, question_dialog from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget from calibre.utils.formatter import validation_formatter +from calibre.ebooks import BOOK_EXTENSIONS class ConfigWidget(QWidget, Ui_ConfigWidget): def __init__(self, settings, all_formats, supports_subdirs, must_read_metadata, supports_use_author_sort, - extra_customization_message): + extra_customization_message, device): QWidget.__init__(self) Ui_ConfigWidget.__init__(self) @@ -25,9 +26,15 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): self.settings = settings + all_formats = set(all_formats) + self.calibre_known_formats = device.FORMATS + self.device_name = device.get_gui_name() + if device.USER_CAN_ADD_NEW_FORMATS: + all_formats = set(all_formats) | set(BOOK_EXTENSIONS) + format_map = settings.format_map disabled_formats = list(set(all_formats).difference(format_map)) - for format in format_map + disabled_formats: + for format in format_map + list(sorted(disabled_formats)): item = QListWidgetItem(format, self.columns) item.setData(Qt.UserRole, QVariant(format)) item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable) @@ -110,6 +117,18 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): return self.opt_use_author_sort.isChecked() def validate(self): + formats = set(self.format_map()) + extra = formats - set(self.calibre_known_formats) + if extra: + fmts = sorted([x.upper() for x in extra]) + if not question_dialog(self, _('Unknown formats'), + _('You have enabled the {0} formats for' + ' your {1}. The {1} may not support them.' + ' If you send these formats to your {1} they ' + 'may not work. Are you sure?').format( + (', '.join(fmts)), self.device_name)): + return False + tmpl = unicode(self.opt_save_template.text()) try: validation_formatter.validate(tmpl) From e7ba694110fe85445b9f1976acac599c4e2599b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 09:58:11 -0700 Subject: [PATCH 10/21] Workers World by usrlnx. Fixes #405 (New news feed) --- resources/recipes/workers_world.recipe | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 resources/recipes/workers_world.recipe diff --git a/resources/recipes/workers_world.recipe b/resources/recipes/workers_world.recipe new file mode 100644 index 0000000000..1967b8e76c --- /dev/null +++ b/resources/recipes/workers_world.recipe @@ -0,0 +1,26 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class WorkersWorld(BasicNewsRecipe): + + title = u'Workers World' + description = u'Socialist news and analysis' + __author__ = u'urslnx' + no_stylesheets = True + use_embedded_content = False + remove_javascript = True + oldest_article = 7 + max_articles_per_feed = 100 + encoding = 'utf8' + publisher = 'workers.org' + category = 'news, politics, USA, world' + language = 'en' + publication_type = 'newsportal' + extra_css = ' body{ font-family: Verdana,Arial,Helvetica,sans-serif; } h1{ font-size: x-large; text-align: left; margin-top:0.5em; margin-bottom:0.25em; } h2{ font-size: large; } p{ text-align: left; } .published{ font-size: small; } .byline{ font-size: small; } .copyright{ font-size: small; } ' + remove_tags_before = dict(name='div', attrs={'id':'evernote'}) + remove_tags_after = dict(name='div', attrs={'id':'footer'}) + + masthead_url='http://www.workers.org/graphics/wwlogo300.gif' + cover_url = 'http://www.workers.org/pdf/current.jpg' + feeds = [(u'Headlines', u'http://www.workers.org/rss/nonstandard_rss.xml'), +] + From 3cc6dc480a76d921c0b2e5cfcbf9c017ee5a8ce1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 10:09:21 -0700 Subject: [PATCH 11/21] Add a button ot the search preferences to clear search history. Fixes #8953 (Clear search items) --- src/calibre/gui2/preferences/search.py | 7 +++++++ src/calibre/gui2/preferences/search.ui | 10 ++++++++++ src/calibre/gui2/search_box.py | 3 +++ 3 files changed, 20 insertions(+) diff --git a/src/calibre/gui2/preferences/search.py b/src/calibre/gui2/preferences/search.py index 81bc603df4..749a7c8de0 100644 --- a/src/calibre/gui2/preferences/search.py +++ b/src/calibre/gui2/preferences/search.py @@ -26,12 +26,19 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('limit_search_columns_to', prefs, setting=CommaSeparatedList) fl = gui.library_view.model().db.field_metadata.get_search_terms() self.opt_limit_search_columns_to.update_items_cache(fl) + self.clear_history_button.clicked.connect(self.clear_histories) def refresh_gui(self, gui): gui.search.search_as_you_type(config['search_as_you_type']) gui.library_view.model().set_highlight_only(config['highlight_search_matches']) gui.search.do_search() + def clear_histories(self, *args): + for key, val in config.defaults.iteritems(): + if key.endswith('_search_history') and isinstance(val, list): + config[key] = [] + self.gui.search.clear_history() + if __name__ == '__main__': app = QApplication([]) test_widget('Interface', 'Search') diff --git a/src/calibre/gui2/preferences/search.ui b/src/calibre/gui2/preferences/search.ui index 360059ce56..4a6e799641 100644 --- a/src/calibre/gui2/preferences/search.ui +++ b/src/calibre/gui2/preferences/search.ui @@ -90,6 +90,16 @@ + + + + Clear search histories from all over calibre. Including the book list, e-book viewer, fetch news dialog, etc. + + + Clear search &histories + + + diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 900c882adc..34be6cd276 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -114,6 +114,9 @@ class SearchBox2(QComboBox): # {{{ def text(self): return self.currentText() + def clear_history(self, *args): + QComboBox.clear(self) + def clear(self, emit_search=True): self.normalize_state() self.setEditText('') From 93747dbe94a4f5df860952af9e35fedb9979f8df Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 10:11:56 -0700 Subject: [PATCH 12/21] Fix #8961 (Device request: Kendo/Yifang M7) --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index baefdfc41d..53c73b01a0 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -19,7 +19,7 @@ class ANDROID(USBMS): VENDOR_ID = { # HTC - 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], + 0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226, 0x222], 0x0c01 : [0x100, 0x0227, 0x0226], 0x0ff9 : [0x0100, 0x0227, 0x0226], 0x0c87 : [0x0100, 0x0227, 0x0226], From 247553badc37f9f88576cec5dad316e917e500a8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 10:21:29 -0700 Subject: [PATCH 13/21] Fix #8959 (in bulk edit, assigning series index numbers without entering a series name does nothing) --- src/calibre/gui2/dialogs/metadata_bulk.py | 1 + src/calibre/gui2/dialogs/metadata_bulk.ui | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 9ad61d515b..cdb254ac78 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -912,6 +912,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): def series_changed(self, *args): self.write_series = True + self.autonumber_series.setEnabled(True) def s_r_remove_query(self, *args): if self.query_field.currentIndex() == 0: diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 2ab37bcbc6..ae3445998b 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -303,6 +303,9 @@ + + false + If not checked, the series number for the books will be set to 1. If checked, selected books will be automatically numbered, in the order @@ -1006,8 +1009,8 @@ not multiple and the destination field is multiple 0 0 - 938 - 268 + 197 + 60 From f1bd291140dcaf434355ec74ab4cddbce176aadf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 12:34:20 -0700 Subject: [PATCH 14/21] Implement #8921 (last viewed book(s) history) --- src/calibre/gui2/preferences/search.ui | 6 ++-- src/calibre/gui2/viewer/main.py | 44 ++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/preferences/search.ui b/src/calibre/gui2/preferences/search.ui index 4a6e799641..7d40f723ea 100644 --- a/src/calibre/gui2/preferences/search.ui +++ b/src/calibre/gui2/preferences/search.ui @@ -77,7 +77,7 @@ - + Qt::Vertical @@ -90,7 +90,7 @@ - + Clear search histories from all over calibre. Including the book list, e-book viewer, fetch news dialog, etc. @@ -106,7 +106,7 @@ MultiCompleteLineEdit QLineEdit -
calibre/gui2.complete.h
+
calibre/gui2/complete.h
 diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index c5001659a0..de0f83a5b2 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -17,16 +17,16 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager from calibre.gui2.widgets import ProgressIndicator from calibre.gui2.main_window import MainWindow from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \ - info_dialog, error_dialog, open_url, available_height + info_dialog, error_dialog, open_url, available_height, gprefs from calibre.ebooks.oeb.iterator import EbookIterator from calibre.ebooks import DRMError -from calibre.constants import islinux, isfreebsd, isosx +from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding from calibre.utils.config import Config, StringConfig, dynamic from calibre.gui2.search_box import SearchBox2 from calibre.ebooks.metadata import MetaInformation from calibre.customize.ui import available_input_formats from calibre.gui2.viewer.dictionary import Lookup -from calibre import as_unicode +from calibre import as_unicode, force_unicode, isbytestring class TOCItem(QStandardItem): @@ -160,6 +160,12 @@ class HelpfulLineEdit(QLineEdit): self.setPalette(self.gray) self.setText(self.HELP_TEXT) +class RecentAction(QAction): + + def __init__(self, path, parent): + self.path = path + QAction.__init__(self, os.path.basename(path), parent) + class EbookViewer(MainWindow, Ui_EbookViewer): STATE_VERSION = 1 @@ -284,8 +290,26 @@ class EbookViewer(MainWindow, Ui_EbookViewer): ca = self.view.copy_action ca.setShortcut(QKeySequence.Copy) self.addAction(ca) + self.open_history_menu = QMenu() + self.build_recent_menu() + self.action_open_ebook.setMenu(self.open_history_menu) + self.open_history_menu.triggered[QAction].connect(self.open_recent) + w = self.tool_bar.widgetForAction(self.action_open_ebook) + w.setPopupMode(QToolButton.MenuButtonPopup) + self.restore_state() + def build_recent_menu(self): + m = self.open_history_menu + m.clear() + count = 0 + for path in gprefs.get('viewer_open_history', []): + if count > 9: + break + if os.path.exists(path): + m.addAction(RecentAction(path, m)) + count += 1 + def closeEvent(self, e): self.save_state() return MainWindow.closeEvent(self, e) @@ -425,6 +449,9 @@ class EbookViewer(MainWindow, Ui_EbookViewer): if files: self.load_ebook(files[0]) + def open_recent(self, action): + self.load_ebook(action.path) + def font_size_larger(self, checked): frac = self.view.magnify_fonts() self.action_font_size_larger.setEnabled(self.view.multiplier() < 3) @@ -647,6 +674,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.action_table_of_contents.setChecked(True) else: self.action_table_of_contents.setChecked(False) + if isbytestring(pathtoebook): + pathtoebook = force_unicode(pathtoebook, filesystem_encoding) + vh = gprefs.get('viewer_open_history', []) + try: + vh.remove(pathtoebook) + except: + pass + vh.insert(0, pathtoebook) + gprefs.set('viewer_open_history', vh[:50]) + self.build_recent_menu() + self.action_table_of_contents.setDisabled(not self.iterator.toc) self.current_book_has_toc = bool(self.iterator.toc) self.current_title = title From c6aaa3b7b6eac582cc684fe8b7d798d930f60aca Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 12:38:18 -0700 Subject: [PATCH 15/21] Heuristics processing: Fix bug in italicize regeps that could cuase a scenebreak consisting of multiple underscores to be prefixed by the word None. Fixes #8960 (Multiple hyphens in TXT input causing None to be inserted into text) --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index 95f832c76a..2e26f927f5 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -156,17 +156,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=[\s>])_(?P[^_]+)?_', - r'(?msu)(?<=[\s>])/(?P[^/]+)?/', - r'(?msu)(?<=[\s>])~~(?P[^~]+)?~~', - r'(?msu)(?<=[\s>])\*(?P[^\*]+)?\*', - r'(?msu)(?<=[\s>])~(?P[^~]+)?~', - r'(?msu)(?<=[\s>])_/(?P[^/_]+)?/_', - r'(?msu)(?<=[\s>])_\*(?P[^\*_]+)?\*_', - r'(?msu)(?<=[\s>])\*/(?P[^/\*]+)?/\*', - r'(?msu)(?<=[\s>])_\*/(?P[^\*_]+)?/\*_', - r'(?msu)(?<=[\s>])/:(?P[^:/]+)?:/', - r'(?msu)(?<=[\s>])\|:(?P[^:\|]+)?:\|', + r'(?msu)(?<=[\s>])_(?P[^_]+)_', + r'(?msu)(?<=[\s>])/(?P[^/]+)/', + r'(?msu)(?<=[\s>])~~(?P[^~]+)~~', + r'(?msu)(?<=[\s>])\*(?P[^\*]+)\*', + r'(?msu)(?<=[\s>])~(?P[^~]+)~', + r'(?msu)(?<=[\s>])_/(?P[^/_]+)/_', + r'(?msu)(?<=[\s>])_\*(?P[^\*_]+)\*_', + r'(?msu)(?<=[\s>])\*/(?P[^/\*]+)/\*', + r'(?msu)(?<=[\s>])_\*/(?P[^\*_]+)/\*_', + r'(?msu)(?<=[\s>])/:(?P[^:/]+):/', + r'(?msu)(?<=[\s>])\|:(?P[^:\|]+):\|', ] for word in ITALICIZE_WORDS: From 46a35f5d89b03ca21129f3e1945e6c32d0ac6caf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 12:52:39 -0700 Subject: [PATCH 16/21] Fix regression that caused completion in authors/series/tags fields on OS X to return extra text. Fixes #8963 (In the Edit Dialog, PIcking from the Author's drop down list is broken) --- src/calibre/gui2/complete.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 226fe8b9c7..1da0042bdd 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -158,6 +158,8 @@ class MultiCompleteComboBox(EnComboBox): # item that matches case insensitively c = self.lineEdit().completer() c.setCaseSensitivity(Qt.CaseSensitive) + self.dummy_model = CompleteModel(self) + c.setModel(self.dummy_model) def update_items_cache(self, complete_items): self.lineEdit().update_items_cache(complete_items) From 8f6255f87a7c1fd155a876760b58d95d9a968cb8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 13:08:37 -0700 Subject: [PATCH 17/21] LT covers plugin: has_cover now always returns False as LT is unreliable --- src/calibre/ebooks/metadata/covers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index cbd8fc0e99..3deb54da10 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -145,6 +145,7 @@ class LibraryThingCovers(CoverDownload): # {{{ return url def has_cover(self, mi, ans, timeout=5.): + return False if not mi.isbn or not self.site_customization: return False from calibre.ebooks.metadata.library_thing import get_browser, login From 73119e25979911a0378c6c00b39a670ba871beb2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 14:30:36 -0700 Subject: [PATCH 18/21] Conversion pipeline: Correctly handle align attribute on img tags. MOBI Output: Support the vertical-align CSS property for images --- src/calibre/ebooks/mobi/mobiml.py | 3 +++ src/calibre/ebooks/oeb/transforms/flatcss.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 17a14d9e12..bdf81597b1 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -367,6 +367,9 @@ class MobiMLizer(object): istate.attrib['src'] = elem.attrib['src'] istate.attrib['align'] = 'baseline' cssdict = style.cssdict() + valign = cssdict.get('vertical-align', None) + if valign in ('top', 'bottom', 'middle'): + istate.attrib['align'] = valign for prop in ('width', 'height'): if cssdict[prop] != 'auto': value = style[prop] diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 653aa4533b..db6bdf0a7a 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -207,7 +207,14 @@ class CSSFlattener(object): font_size = self.sbase if self.sbase is not None else \ self.context.source.fbase if 'align' in node.attrib: - cssdict['text-align'] = node.attrib['align'] + if tag != 'img': + cssdict['text-align'] = node.attrib['align'] + else: + val = node.attrib['align'] + if val in ('middle', 'bottom', 'top'): + cssdict['vertical-align'] = val + elif val in ('left', 'right'): + cssdict['text-align'] = val del node.attrib['align'] if node.tag == XHTML('font'): node.tag = XHTML('span') From 6bc6e7c79f0fd16d2fa94722cbfe7e2c33148530 Mon Sep 17 00:00:00 2001 From: Ben Collier Date: Sun, 13 Feb 2011 17:21:17 -0500 Subject: [PATCH 19/21] another minor fix to remove stray header text for "complete coverage" of topics --- resources/recipes/nytimes_sub.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index 7f73664660..4077065d91 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -668,7 +668,7 @@ class NYTimes(BasicNewsRecipe): try: #remove "Related content" bar - runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft ']}) + runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline','articleInline runaroundLeft ','articleInline runaroundLeft lastArticleInline']}) if runAroundsFound: for runAround in runAroundsFound: #find all section headers From 3a2daf39e333e5dff40e214b1e1c04b96859eb62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 13 Feb 2011 15:47:28 -0700 Subject: [PATCH 20/21] Replace LibraryThing cover download plugin with a new plugin to download covers from Amazon --- src/calibre/customize/builtins.py | 4 +- src/calibre/ebooks/metadata/__init__.py | 2 + src/calibre/ebooks/metadata/amazon.py | 105 +++++++++++++++++++++--- src/calibre/ebooks/metadata/covers.py | 60 +++----------- 4 files changed, 110 insertions(+), 61 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 3ccc07040b..1dd575f45b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -511,14 +511,14 @@ from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \ from calibre.ebooks.metadata.douban import DoubanBooks from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ - LibraryThingCovers, DoubanCovers + AmazonCovers, DoubanCovers from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX from calibre.ebooks.epub.fix.unmanifested import Unmanifested from calibre.ebooks.epub.fix.epubcheck import Epubcheck plugins = [HTML2ZIP, PML2PMLZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon, KentDistrictLibrary, DoubanBooks, NiceBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, - Epubcheck, OpenLibraryCovers, LibraryThingCovers, DoubanCovers, + Epubcheck, OpenLibraryCovers, AmazonCovers, DoubanCovers, NiceBooksCovers] plugins += [ ComicInput, diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index fcd4491fd3..6078a0aa94 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -271,6 +271,8 @@ def check_isbn13(isbn): return None def check_isbn(isbn): + if not isbn: + return None isbn = re.sub(r'[^0-9X]', '', isbn.upper()) if len(isbn) == 10: return check_isbn10(isbn) diff --git a/src/calibre/ebooks/metadata/amazon.py b/src/calibre/ebooks/metadata/amazon.py index cf96c9732c..98a2ac6d36 100644 --- a/src/calibre/ebooks/metadata/amazon.py +++ b/src/calibre/ebooks/metadata/amazon.py @@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en' Fetch metadata using Amazon AWS ''' import sys, re +from threading import RLock from lxml import html from lxml.html import soupparser @@ -17,6 +18,10 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.chardet import xml_to_unicode from calibre.library.comments import sanitize_comments_html +asin_cache = {} +cover_url_cache = {} +cache_lock = RLock() + def find_asin(br, isbn): q = 'http://www.amazon.com/s?field-keywords='+isbn raw = br.open_novisit(q).read() @@ -29,6 +34,12 @@ def find_asin(br, isbn): return revs[0] def to_asin(br, isbn): + with cache_lock: + ans = asin_cache.get(isbn, None) + if ans: + return ans + if ans is False: + return None if len(isbn) == 13: try: asin = find_asin(br, isbn) @@ -38,8 +49,11 @@ def to_asin(br, isbn): asin = None else: asin = isbn + with cache_lock: + asin_cache[isbn] = ans if ans else False return asin + def get_social_metadata(title, authors, publisher, isbn): mi = Metadata(title, authors) if not isbn: @@ -58,6 +72,68 @@ def get_social_metadata(title, authors, publisher, isbn): return mi return mi +def get_cover_url(isbn, br): + isbn = check_isbn(isbn) + if not isbn: + return None + with cache_lock: + ans = cover_url_cache.get(isbn, None) + if ans: + return ans + if ans is False: + return None + asin = to_asin(br, isbn) + if asin: + ans = _get_cover_url(br, asin) + if ans: + with cache_lock: + cover_url_cache[isbn] = ans + return ans + from calibre.ebooks.metadata.xisbn import xisbn + for i in xisbn.get_associated_isbns(isbn): + asin = to_asin(br, i) + if asin: + ans = _get_cover_url(br, asin) + if ans: + with cache_lock: + cover_url_cache[isbn] = ans + cover_url_cache[i] = ans + return ans + with cache_lock: + cover_url_cache[isbn] = False + return None + +def _get_cover_url(br, asin): + q = 'http://amzn.com/'+asin + try: + raw = br.open_novisit(q).read() + except Exception, e: + if callable(getattr(e, 'getcode', None)) and \ + e.getcode() == 404: + return None + raise + if '404 - ' in raw: + return None + raw = xml_to_unicode(raw, strip_encoding_pats=True, + resolve_entities=True)[0] + try: + root = soupparser.fromstring(raw) + except: + return False + + imgs = root.xpath('//img[@id="prodImage" and @src]') + if imgs: + src = imgs[0].get('src') + parts = src.split('/') + if len(parts) > 3: + bn = parts[-1] + sparts = bn.split('_') + if len(sparts) > 2: + bn = sparts[0] + sparts[-1] + return ('/'.join(parts[:-1]))+'/'+bn + return None + + def get_metadata(br, asin, mi): q = 'http://amzn.com/'+asin try: @@ -111,18 +187,25 @@ def get_metadata(br, asin, mi): def main(args=sys.argv): - # Test xisbn - print get_social_metadata('Learning Python', None, None, '8324616489') - print + import tempfile, os + tdir = tempfile.gettempdir() + br = browser() + for title, isbn in [ + ('Learning Python', '8324616489'), # Test xisbn + ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting + # Random tests + ('Star Trek: Destiny: Mere Mortals', '9781416551720'), + ('The Great Gatsby', '0743273567'), + ]: + cpath = os.path.join(tdir, title+'.jpg') + curl = get_cover_url(isbn, br) + if curl is None: + print 'No cover found for', title + else: + open(cpath, 'wb').write(br.open_novisit(curl).read()) + print 'Cover for', title, 'saved to', cpath - # Test sophisticated comment formatting - print get_social_metadata('Angels & Demons', None, None, '9781416580829') - print - - # Random tests - print get_social_metadata('Star Trek: Destiny: Mere Mortals', None, None, '9781416551720') - print - print get_social_metadata('The Great Gatsby', None, None, '0743273567') + print get_social_metadata(title, None, None, isbn) return 0 diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index 3deb54da10..15e0a05c1e 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import traceback, socket, re, sys +import traceback, socket, sys from functools import partial from threading import Thread, Event from Queue import Queue, Empty @@ -15,7 +15,6 @@ import mechanize from calibre.customize import Plugin from calibre import browser, prints -from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.constants import preferred_encoding, DEBUG class CoverDownload(Plugin): @@ -112,73 +111,38 @@ class OpenLibraryCovers(CoverDownload): # {{{ # }}} -class LibraryThingCovers(CoverDownload): # {{{ +class AmazonCovers(CoverDownload): # {{{ - name = 'librarything.com covers' - description = _('Download covers from librarything.com') + name = 'amazon.com covers' + description = _('Download covers from amazon.com') author = 'Kovid Goyal' - LIBRARYTHING = 'http://www.librarything.com/isbn/' - - def get_cover_url(self, isbn, br, timeout=5.): - - try: - src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, - timeout=timeout).read().decode('utf-8', 'replace') - except Exception, err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - err = Exception(_('LibraryThing.com timed out. Try again later.')) - raise err - else: - if '/wiki/index.php/HelpThing:Verify' in src: - raise Exception('LibraryThing is blocking calibre.') - s = BeautifulSoup(src) - url = s.find('td', attrs={'class':'left'}) - if url is None: - if s.find('div', attrs={'class':'highloadwarning'}) is not None: - raise Exception(_('Could not fetch cover as server is experiencing high load. Please try again later.')) - raise Exception(_('ISBN: %s not found')%isbn) - url = url.find('img') - if url is None: - raise Exception(_('LibraryThing.com server error. Try again later.')) - url = re.sub(r'_S[XY]\d+', '', url['src']) - return url def has_cover(self, mi, ans, timeout=5.): - return False - if not mi.isbn or not self.site_customization: + if not mi.isbn: return False - from calibre.ebooks.metadata.library_thing import get_browser, login - br = get_browser() - un, _, pw = self.site_customization.partition(':') - login(br, un, pw) + from calibre.ebooks.metadata.amazon import get_cover_url + br = browser() try: - self.get_cover_url(mi.isbn, br, timeout=timeout) + get_cover_url(mi.isbn, br) self.debug('cover for', mi.isbn, 'found') ans.set() except Exception, e: self.debug(e) def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn or not self.site_customization: + if not mi.isbn: return - from calibre.ebooks.metadata.library_thing import get_browser, login - br = get_browser() - un, _, pw = self.site_customization.partition(':') - login(br, un, pw) + from calibre.ebooks.metadata.amazon import get_cover_url + br = browser() try: - url = self.get_cover_url(mi.isbn, br, timeout=timeout) + url = get_cover_url(mi.isbn, br) cover_data = br.open_novisit(url).read() result_queue.put((True, cover_data, 'jpg', self.name)) except Exception, e: result_queue.put((False, self.exception_to_string(e), traceback.format_exc(), self.name)) - def customization_help(self, gui=False): - ans = _('To use librarything.com you must sign up for a %sfree account%s ' - 'and enter your username and password separated by a : below.') - return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>') - # }}} def check_for_cover(mi, timeout=5.): # {{{ From a5cdad2705dbfe5128f0ee4057f85c7196036888 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 13 Feb 2011 16:58:15 -0700 Subject: [PATCH 21/21] ... --- src/calibre/ebooks/txt/input.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 14a21471de..9b3f9c32ab 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -210,9 +210,11 @@ class TXTInput(InputFormatPlugin): oeb = html_input.convert(open(htmlfile.name, 'rb'), options, 'html', log, {}) # Add images from from txtz archive to oeb. - for image, mime in images: - id, href = oeb.manifest.generate(id='image', href=image) - oeb.manifest.add(id, href, mime) + # Disabled as the conversion pipeline adds unmanifested items that are + # referred to in the content automatically + #for image, mime in images: + # id, href = oeb.manifest.generate(id='image', href=image) + # oeb.manifest.add(id, href, mime) options.debug_pipeline = odi os.remove(htmlfile.name)