From 545cdfd00d14f9bdab4967318ea5edcf8d6bb24c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Dec 2010 09:43:11 +0000 Subject: [PATCH 01/11] Enhancement #7931: add tweak to change display in tags pane from author to author_sort. --- resources/default_tweaks.py | 14 ++++++++++++++ src/calibre/gui2/tag_view.py | 29 ++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 750af9efa7..6d4f7a405f 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -41,6 +41,20 @@ series_index_auto_increment = 'next' # selecting 'manage authors', and pressing 'Recalculate all author sort values'. author_sort_copy_method = 'invert' +# Set which author field to display in the tags pane (the list of authors, +# series, publishers etc on the left hand side). The choices are author and +# author_sort. This tweak affects only the tags pane, and only what is displayed +# under the authors category. Please note that if you set this to author_sort, +# it is very possible to see duplicate names in the list becasue although it is +# guaranteed that author names are unique, there is no such guarantee for +# author_sort values. Showing duplicates won't break anything, but it could +# lead to some confusion. When using 'author_sort', the tooltip will show the +# author's name. +# Examples: +# tags_pane_use_field_for_author_name = 'author' +# tags_pane_use_field_for_author_name = 'author_sort' +tags_pane_use_field_for_author_name = 'author' + # Set whether boolean custom columns are two- or three-valued. # Two-values for true booleans diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index f75061da12..a0e26cf77c 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -18,6 +18,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \ from calibre.ebooks.metadata import title_sort from calibre.gui2 import config, NONE from calibre.library.field_metadata import TagsIcons, category_icon_map +from calibre.utils.config import tweaks from calibre.utils.icu import sort_key from calibre.utils.search_query_parser import saved_searches from calibre.gui2 import error_dialog @@ -409,17 +410,31 @@ class TagTreeItem(object): # {{{ return NONE def tag_data(self, role): + tag = self.tag + if tag.category == 'authors' and \ + tweaks['tags_pane_use_field_for_author_name'] == 'author_sort': + name = tag.sort + tt_author = True + else: + name = tag.name + tt_author = False if role == Qt.DisplayRole: - if self.tag.count == 0: - return QVariant('%s'%(self.tag.name)) + if tag.count == 0: + return QVariant('%s'%(name)) else: - return QVariant('[%d] %s'%(self.tag.count, self.tag.name)) + return QVariant('[%d] %s'%(tag.count, name)) if role == Qt.EditRole: - return QVariant(self.tag.name) + return QVariant(tag.name) if role == Qt.DecorationRole: - return self.icon_state_map[self.tag.state] - if role == Qt.ToolTipRole and self.tag.tooltip is not None: - return QVariant(self.tag.tooltip) + return self.icon_state_map[tag.state] + if role == Qt.ToolTipRole: + if tt_author: + if tag.tooltip is not None: + return QVariant('(%s) %s'%(tag.name, tag.tooltip)) + else: + return QVariant(tag.name) + if tag.tooltip is not None: + return QVariant(tag.tooltip) return NONE def toggle(self): From d45398fe478b88930f68a4a2c1052fb5bb9405f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 09:40:45 -0700 Subject: [PATCH 02/11] ... --- src/calibre/manual/conversion.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index fea20a3163..3a7ae16598 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -541,7 +541,9 @@ Use the options to remove headers and footers to mitigate this issue. If the hea removed from the text it can throw off the paragraph unwrapping. Some limitations of PDF input is complex, multi-column, and image based documents are not supported. -Extraction of vector images and tables from within the document is also not supported. +Extraction of vector images and tables from within the document is also not supported. Some PDFs use special glyphs to +represent double ll or doubfle ff or fi,etc. Conversion of these may or may not work depending on jusy how they are +represented internally in the PDF. Comic Book Collections ~~~~~~~~~~~~~~~~~~~~~~~~~ From 3d03f5c4a4a7886834e06281b21e6ed2f780d73b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 10:03:56 -0700 Subject: [PATCH 03/11] Content server: Fix regression that broke browsing by rating --- src/calibre/library/server/browse.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 37f024c08d..485601a311 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -373,13 +373,18 @@ class BrowseServer(object): script='toplevel();', main=main) def browse_sort_categories(self, items, sort): - if sort not in ('rating', 'name', 'popularity'): - sort = 'name' - items.sort(key=lambda x: sort_key(getattr(x, 'sort', x.name))) + def keyg(x): + x = getattr(x, 'sort', x.name) + if isinstance(x, unicode): + return sort_key(x) + return x + if sort == 'popularity': items.sort(key=operator.attrgetter('count'), reverse=True) elif sort == 'rating': items.sort(key=operator.attrgetter('avg_rating'), reverse=True) + else: + items.sort(key=keyg) return sort def browse_category(self, category, sort): From 13e83afab8ddc523087d3c4f92f83e162a289230 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Dec 2010 17:22:26 +0000 Subject: [PATCH 04/11] Fix problem where ratings have non-string sort values --- src/calibre/library/database2.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 35ef8c1f58..2848e8ebb3 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1128,6 +1128,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for l in list: (id, val, sort_val) = (l[0], l[1], l[2]) tids[category][val] = (id, sort_val) + elif cat['datatype'] == 'rating': + for l in list: + (id, val) = (l[0], l[1]) + tids[category][val] = (id, '{0:05.2f}'.format(val)) else: for l in list: (id, val) = (l[0], l[1]) @@ -1256,12 +1260,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # sort the list if sort == 'name': - def get_sort_key(x): - sk = x.s - if isinstance(sk, unicode): - sk = sort_key(sk) - return sk - kf = get_sort_key + kf = lambda x :sort_key(x.s) reverse=False elif sort == 'popularity': kf = lambda x: x.c From 20d7b486e4f6e2761c129a8e3b5dbb6d3ed8fee3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Dec 2010 17:30:12 +0000 Subject: [PATCH 05/11] Revert changes from browse.py --- src/calibre/library/server/browse.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 485601a311..37f024c08d 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -373,18 +373,13 @@ class BrowseServer(object): script='toplevel();', main=main) def browse_sort_categories(self, items, sort): - def keyg(x): - x = getattr(x, 'sort', x.name) - if isinstance(x, unicode): - return sort_key(x) - return x - + if sort not in ('rating', 'name', 'popularity'): + sort = 'name' + items.sort(key=lambda x: sort_key(getattr(x, 'sort', x.name))) if sort == 'popularity': items.sort(key=operator.attrgetter('count'), reverse=True) elif sort == 'rating': items.sort(key=operator.attrgetter('avg_rating'), reverse=True) - else: - items.sort(key=keyg) return sort def browse_category(self, category, sort): From 1ee512bf23272388c97e208719abc10efc52352c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 10:47:16 -0700 Subject: [PATCH 06/11] Fix #7935 (Changed device ID's for SWEEX MM300) --- src/calibre/devices/misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index e27aee4393..68f4dba9a9 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -62,9 +62,9 @@ class SWEEX(USBMS): # Ordered list of supported formats FORMATS = ['epub', 'prc', 'fb2', 'html', 'rtf', 'chm', 'pdf', 'txt'] - VENDOR_ID = [0x0525] - PRODUCT_ID = [0xa4a5] - BCD = [0x0319] + VENDOR_ID = [0x0525, 0x177f] + PRODUCT_ID = [0xa4a5, 0x300] + BCD = [0x0319, 0x110] VENDOR_NAME = 'SWEEX' WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOKREADER' From 66f9313ac94886c45e7dd1cb94a3c38e306d5ba1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 11:19:12 -0700 Subject: [PATCH 07/11] ... --- Changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 7f7afc117a..bcf58ae03d 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -11,7 +11,7 @@ - title: "Page turn animations in the e-book viewer" type: major description: > - "Now when you use the Page Down/Page Up keys or the next/previous page buttons in the viewer, page turning will be animated. The duration of the animation can be controlled in the viewer preferences. Setting it to o disables the animation completely." + "Now when you use the Page Down/Page Up keys or the next/previous page buttons in the viewer, page turning will be animated. The duration of the animation can be controlled in the viewer preferences. Setting it to 0 disables the animation completely." - title: "Conversion pipeline: Add an option to set the minimum line height of all elemnts as a percentage of the computed font size. By default, calibre now sets the line height to 120% of the computed font size." From 2ba5caa9514b1e203ff46971e5be8b3d1fb7ece4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 15:57:32 -0700 Subject: [PATCH 08/11] ... --- setup/installer/windows/notes.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index af4c871dac..b9aef39657 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -36,6 +36,16 @@ Install BeautifulSoup 3.0.x manually into site-packages (3.1.x parses broken HTM Install pywin32 and edit win32com\__init__.py setting _frozen = True and __gen_path__ to a temp dir (otherwise it tries to set it to a dir in the install tree which leads to permission errors) +Note that you should use:: + + import tempfile + __gen_path__ = os.path.join( + tempfile.gettempdir(), "gen_py", + "%d.%d" % (sys.version_info[0], sys.version_info[1])) + +Use gettempdir instead of the win32 api method as gettempdir returns a temp dir that is guaranteed to actually work. + + Also edit win32com\client\gencache.py and change the except IOError on line 57 to catch all exceptions. SQLite From 281058618c072d5e3a8fea24a49d37c3a9b7c325 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 17:52:17 -0700 Subject: [PATCH 09/11] Fix #7955 (Bug with convert in 0.7.34) --- src/calibre/ebooks/rtf/input.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 57903a6711..8c7561f68c 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -245,7 +245,7 @@ class RTFInput(InputFormatPlugin): from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException - self.options = options + self.opts = options self.log = log self.log('Converting RTF to XML...') #Name of the preprocesssed RTF file @@ -290,12 +290,12 @@ class RTFInput(InputFormatPlugin): res = transform.tostring(result) res = res[:100].replace('xmlns:html', 'xmlns') + res[100:] # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines - if not getattr(self.options, 'remove_paragraph_spacing', False): + if not getattr(self.opts, 'remove_paragraph_spacing', False): res = re.sub('\s*', '', res) res = re.sub('(?<=\n)\n{2}', u'

\u00a0

\n'.encode('utf-8'), res) - if self.options.preprocess_html: - preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None)) + if self.opts.preprocess_html: + preprocessor = PreProcessor(self.opts, log=getattr(self, 'log', None)) res = preprocessor(res) f.write(res) self.write_inline_css(inline_class, border_styles) From 2d9f56ffb9f3899b34cf6355d84ac3a4d6388ac0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 18:23:33 -0700 Subject: [PATCH 10/11] Fix New England Journal of Medicine and Journal of Hospital Medicine --- resources/recipes/johm.recipe | 84 ++++++++++++++++------------------- resources/recipes/nejm.recipe | 2 +- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/resources/recipes/johm.recipe b/resources/recipes/johm.recipe index 6930f05a50..0f5625b806 100644 --- a/resources/recipes/johm.recipe +++ b/resources/recipes/johm.recipe @@ -1,78 +1,72 @@ -# -*- coding: utf-8 -*- - +import re from calibre.web.feeds.recipes import BasicNewsRecipe class JournalofHospitalMedicine(BasicNewsRecipe): title = 'Journal of Hospital Medicine' - __author__ = 'Krittika Goyal' + __author__ = 'Kovid Goyal' description = 'Medical news' timefmt = ' [%d %b, %Y]' needs_subscription = True language = 'en' no_stylesheets = True + keep_only_tags = [dict(id=['articleTitle', 'articleMeta', 'fulltext'])] + remove_tags = [dict(attrs={'class':'licensedContent'})] # TO LOGIN def get_browser(self): br = BasicNewsRecipe.get_browser() br.open('http://www3.interscience.wiley.com/cgi-bin/home') - br.select_form(name='siteLogin') - br['LoginName'] = self.username - br['Password'] = self.password + br.select_form(nr=0) + br['j_username'] = self.username + br['j_password'] = self.password response = br.submit() raw = response.read() - if 'userName = ""' in raw: + if '

LOGGED IN

' not in raw: raise Exception('Login failed. Check your username and password') return br #TO GET ARTICLE TOC def johm_get_index(self): - return self.index_to_soup('http://www3.interscience.wiley.com/journal/111081937/home') + return self.index_to_soup('http://onlinelibrary.wiley.com/journal/10.1002/(ISSN)1553-5606/currentissue') # To parse artice toc def parse_index(self): - parse_soup = self.johm_get_index() - - div = parse_soup.find(id='contentCell') - - current_section = None - current_articles = [] + soup = self.johm_get_index() + toc = soup.find(id='issueTocGroups') feeds = [] - for x in div.findAll(True): - if x.name == 'h4': - # Section heading found - if current_articles and current_section: - feeds.append((current_section, current_articles)) - current_section = self.tag_to_string(x) - current_articles = [] - self.log('\tFound section:', current_section) - if current_section is not None and x.name == 'strong': - title = self.tag_to_string(x) - p = x.parent.parent.find('a', href=lambda x: x and '/HTMLSTART' in x) - if p is None: - continue - url = p.get('href', False) - if not url or not title: + for group in toc.findAll('li', id=re.compile(r'group\d+')): + gtitle = group.find(attrs={'class':'subSectionHeading'}) + if gtitle is None: + continue + gtitle = self.tag_to_string(gtitle) + arts = group.find(attrs={'class':'articles'}) + if arts is None: + continue + self.log('Found section:', gtitle) + articles = [] + for art in arts.findAll(attrs={'class':lambda x: x and 'tocArticle' + in x}): + a = art.find('a', href=True) + if a is None: continue + url = a.get('href') if url.startswith('/'): - url = 'http://www3.interscience.wiley.com'+url - url = url.replace('/HTMLSTART', '/main.html,ftx_abs') - self.log('\t\tFound article:', title) - self.log('\t\t\t', url) - #if url.startswith('/'): - #url = 'http://online.wsj.com'+url - current_articles.append({'title': title, 'url':url, - 'description':'', 'date':''}) - - if current_articles and current_section: - feeds.append((current_section, current_articles)) + url = 'http://onlinelibrary.wiley.com' + url + url = url.replace('/abstract', '/full') + title = self.tag_to_string(a) + a.extract() + pm = art.find(attrs={'class':'productMenu'}) + if pm is not None: + pm.extract() + desc = self.tag_to_string(art) + self.log('\tFound article:', title, 'at', url) + articles.append({'title':title, 'url':url, 'description':desc, + 'date':''}) + if articles: + feeds.append((gtitle, articles)) return feeds - def preprocess_html(self, soup): - for img in soup.findAll('img', src=True): - img['src'] = img['src'].replace('tfig', 'nfig') - return soup - diff --git a/resources/recipes/nejm.recipe b/resources/recipes/nejm.recipe index a6580a5232..bc12fbcedf 100644 --- a/resources/recipes/nejm.recipe +++ b/resources/recipes/nejm.recipe @@ -4,7 +4,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe class NYTimes(BasicNewsRecipe): title = 'New England Journal of Medicine' - __author__ = 'Krittika Goyal' + __author__ = 'Kovid Goyal' description = 'Medical news' timefmt = ' [%d %b, %Y]' needs_subscription = True From e03114bed4f0153a2f12ec890065409a0d74dd52 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Dec 2010 18:32:14 -0700 Subject: [PATCH 11/11] Fix #7956 (Motorola Droid 2) --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 8b30631528..f2a5752bd4 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -28,7 +28,7 @@ class ANDROID(USBMS): # Motorola 0x22b8 : { 0x41d9 : [0x216], 0x2d67 : [0x100], 0x41db : [0x216], - 0x4285 : [0x216]}, + 0x4285 : [0x216], 0x42a3 : [0x216] }, # Sony Ericsson 0xfce : { 0xd12e : [0x0100]},